Compare commits

...

59 Commits

Author SHA1 Message Date
nym21 744032f1f1 release: v0.3.2 2026-06-04 18:56:31 +02:00
nym21 99b171bad6 docs: update generated docs 2026-06-04 18:56:00 +02:00
nym21 37e2b6eae2 changelog: updated 2026-06-04 18:50:35 +02:00
nym21 a967fe8f35 oracle: changes + changelog: updated 2026-06-04 18:35:48 +02:00
nym21 a3f3c54675 oracle: v4 2026-06-04 15:38:01 +02:00
nym21 f41874f438 website: redesign part 6 2026-06-03 18:07:11 +02:00
nym21 98bbfec525 website: redesign part 5 2026-06-03 16:50:52 +02:00
nym21 1bcf3235b6 website: redesign part 4 2026-06-03 16:37:00 +02:00
nym21 07734b8bab website: redesign part 3 2026-06-03 16:26:55 +02:00
nym21 a2fd1e03ad website: redesign part 2 2026-06-03 12:41:26 +02:00
nym21 90e8741fb7 website: redesign part 1 2026-06-03 12:34:05 +02:00
nym21 5f5563fece docs: renamed claude to ai 2026-06-02 12:06:50 +02:00
nym21 c7edfce481 changelog: updated 2026-06-02 09:27:56 +02:00
nym21 7b3dd83b93 clients: bump versions 2026-06-01 20:22:13 +02:00
nym21 cae16227fd release: v0.3.1 2026-06-01 19:19:13 +02:00
nym21 dc2ca0ca27 docs: update generated docs 2026-06-01 19:18:42 +02:00
nym21 d161462137 deps: bumped 2026-06-01 18:10:24 +02:00
nym21 be20633945 heatmaps: part 23 2026-06-01 18:03:41 +02:00
nym21 2bbc535b58 heatmaps: part 22 2026-06-01 17:54:42 +02:00
nym21 88c38e74f9 heatmaps: part 21 2026-06-01 14:25:21 +02:00
nym21 a61b76a4a5 heatmaps: part 20 2026-06-01 13:31:00 +02:00
nym21 46b888337c heatmaps: part 19 2026-06-01 13:20:34 +02:00
nym21 4b49a04186 heatmaps: part 18 2026-06-01 13:03:45 +02:00
nym21 15b0cd2445 heatmaps: part 17 2026-06-01 13:03:39 +02:00
nym21 76720434d7 heatmaps: part 16 2026-06-01 12:19:32 +02:00
nym21 200cd1011e heatmaps: part 15 2026-06-01 12:04:44 +02:00
nym21 cb9f277d49 heatmaps: part 14 2026-06-01 12:01:24 +02:00
nym21 102933b406 heatmaps: part 13 2026-06-01 11:17:00 +02:00
nym21 e64ffac8d1 heatmaps: part 12 2026-06-01 10:56:58 +02:00
nym21 a94d31dfdf heatmaps: part 12 2026-06-01 10:30:44 +02:00
nym21 087a3b6fd6 heatmaps: part 11 2026-06-01 01:04:14 +02:00
nym21 7181d59966 heatmaps: part 10 2026-06-01 00:38:12 +02:00
nym21 3b7734a61a heatmaps: part 9 2026-05-31 23:35:19 +02:00
nym21 7860c5a8bd heatmaps: part 8 2026-05-31 18:57:23 +02:00
nym21 5df399d2f7 heatmaps: part 7 2026-05-31 12:05:48 +02:00
nym21 b2345db279 heatmaps: part 6 2026-05-31 01:38:50 +02:00
nym21 7e2fc8b455 heatmaps: part 5 2026-05-30 15:43:59 +02:00
nym21 c1ff095e4b heatmaps: part 5 2026-05-30 13:16:22 +02:00
nym21 cc8fde59e8 heatmaps: part 4 2026-05-30 11:36:49 +02:00
nym21 e43b53b429 heatmaps: part 3 2026-05-30 11:36:46 +02:00
nym21 6938204a24 heatmaps: part 2 2026-05-29 23:17:39 +02:00
nym21 52883bbdba heatmaps: part 1 2026-05-28 21:58:54 +02:00
nym21 100495fdba deps: bumped 2026-05-27 19:41:37 +02:00
nym21 0ad5be6974 global: snap 2026-05-26 15:33:22 +02:00
nym21 66037c862f global: added support for oracle histograms 2026-05-25 16:44:09 +02:00
nym21 ee20175cbf oracle: cleanup + split lib.rs 2026-05-24 18:40:35 +02:00
nym21 7ad0adf659 oracle: start at 340k 2026-05-24 11:34:57 +02:00
nym21 6219d2301d oracle: snap pre 340k patch 2026-05-24 00:07:25 +02:00
nym21 0aaffc6c43 oracle: doc fixes 2026-05-23 12:18:34 +02:00
nym21 9c74881c5d oracle: snapshot + start at 508k 2026-05-23 00:45:37 +02:00
nym21 bf8de73541 oracle: cleanup 2026-05-22 11:02:56 +02:00
nym21 56e8103178 clients: bump versions 2026-05-22 11:02:12 +02:00
nym21 773c0d090b website: cleanup & fixes 2026-05-22 11:01:34 +02:00
nym21 d6f4c0ac19 changelog: updated 2026-05-22 11:01:07 +02:00
nym21 0552ba60d2 release: v0.3.0 2026-05-18 19:43:54 +02:00
nym21 ff056587f7 docs: update generated docs 2026-05-18 19:43:22 +02:00
nym21 0b871e8600 global: snap 2026-05-18 19:38:05 +02:00
nym21 0bdca9086a docs: moved license 2026-05-18 11:40:28 +02:00
nym21 bbab864ed9 actions: remove release pipeline 2026-05-18 11:38:11 +02:00
916 changed files with 36563 additions and 297586 deletions
-15
View File
@@ -1,15 +0,0 @@
name: Check outdated dependencies
on:
schedule:
- cron: "0 9 * * 1"
workflow_dispatch:
jobs:
outdated:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- run: cargo install cargo-outdated
- run: cargo outdated --exit-code 1 --depth 1
-296
View File
@@ -1,296 +0,0 @@
# This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist
#
# Copyright 2022-2024, axodotdev
# SPDX-License-Identifier: MIT or Apache-2.0
#
# CI that:
#
# * checks for a Git Tag that looks like a release
# * builds artifacts with dist (archives, installers, hashes)
# * uploads those artifacts to temporary workflow zip
# * on success, uploads the artifacts to a GitHub Release
#
# Note that the GitHub Release will be created with a generated
# title/body based on your changelogs.
name: Release
permissions:
"contents": "write"
# This task will run whenever you push a git tag that looks like a version
# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc.
# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where
# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION
# must be a Cargo-style SemVer Version (must have at least major.minor.patch).
#
# If PACKAGE_NAME is specified, then the announcement will be for that
# package (erroring out if it doesn't have the given version or isn't dist-able).
#
# If PACKAGE_NAME isn't specified, then the announcement will be for all
# (dist-able) packages in the workspace with that version (this mode is
# intended for workspaces with only one dist-able package, or with all dist-able
# packages versioned/released in lockstep).
#
# If you push multiple tags at once, separate instances of this workflow will
# spin up, creating an independent announcement for each one. However, GitHub
# will hard limit this to 3 tags per commit, as it will assume more tags is a
# mistake.
#
# If there's a prerelease-style suffix to the version, then the release(s)
# will be marked as a prerelease.
on:
pull_request:
push:
tags:
- '**[0-9]+.[0-9]+.[0-9]+*'
jobs:
# Run 'dist plan' (or host) to determine what tasks we need to do
plan:
runs-on: "ubuntu-22.04"
outputs:
val: ${{ steps.plan.outputs.manifest }}
tag: ${{ !github.event.pull_request && github.ref_name || '' }}
tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }}
publishing: ${{ !github.event.pull_request }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install dist
# we specify bash to get pipefail; it guards against the `curl` command
# failing. otherwise `sh` won't catch that `curl` returned non-0
shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.30.2/cargo-dist-installer.sh | sh"
- name: Cache dist
uses: actions/upload-artifact@v4
with:
name: cargo-dist-cache
path: ~/.cargo/bin/dist
# sure would be cool if github gave us proper conditionals...
# so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
# functionality based on whether this is a pull_request, and whether it's from a fork.
# (PRs run on the *source* but secrets are usually on the *target* -- that's *good*
# but also really annoying to build CI around when it needs secrets to work right.)
- id: plan
run: |
dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json
echo "dist ran successfully"
cat plan-dist-manifest.json
echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
- name: "Upload dist-manifest.json"
uses: actions/upload-artifact@v4
with:
name: artifacts-plan-dist-manifest
path: plan-dist-manifest.json
# Build and packages all the platform-specific things
build-local-artifacts:
name: build-local-artifacts (${{ join(matrix.targets, ', ') }})
# Let the initial task tell us to not run (currently very blunt)
needs:
- plan
if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }}
strategy:
fail-fast: false
# Target platforms/runners are computed by dist in create-release.
# Each member of the matrix has the following arguments:
#
# - runner: the github runner
# - dist-args: cli flags to pass to dist
# - install-dist: expression to run to install dist on the runner
#
# Typically there will be:
# - 1 "global" task that builds universal installers
# - N "local" tasks that build each platform's binaries and platform-specific installers
matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
runs-on: ${{ matrix.runner }}
container: ${{ matrix.container && matrix.container.image || null }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
steps:
- name: enable windows longpaths
run: |
git config --global core.longpaths true
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install Rust non-interactively if not already installed
if: ${{ matrix.container }}
run: |
if ! command -v cargo > /dev/null 2>&1; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
fi
- name: Install dist
run: ${{ matrix.install_dist.run }}
# Get the dist-manifest
- name: Fetch local artifacts
uses: actions/download-artifact@v4
with:
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
- name: Install dependencies
run: |
${{ matrix.packages_install }}
- name: Build artifacts
run: |
# Actually do builds and make zips and whatnot
dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
echo "dist ran successfully"
- id: cargo-dist
name: Post-build
# We force bash here just because github makes it really hard to get values up
# to "real" actions without writing to env-vars, and writing to env-vars has
# inconsistent syntax between shell and powershell.
shell: bash
run: |
# Parse out what we just built and upload it to scratch storage
echo "paths<<EOF" >> "$GITHUB_OUTPUT"
dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT"
echo "EOF" >> "$GITHUB_OUTPUT"
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
- name: "Upload artifacts"
uses: actions/upload-artifact@v4
with:
name: artifacts-build-local-${{ join(matrix.targets, '_') }}
path: |
${{ steps.cargo-dist.outputs.paths }}
${{ env.BUILD_MANIFEST_NAME }}
# Build and package all the platform-agnostic(ish) things
build-global-artifacts:
needs:
- plan
- build-local-artifacts
runs-on: "ubuntu-22.04"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install cached dist
uses: actions/download-artifact@v4
with:
name: cargo-dist-cache
path: ~/.cargo/bin/
- run: chmod +x ~/.cargo/bin/dist
# Get all the local artifacts for the global tasks to use (for e.g. checksums)
- name: Fetch local artifacts
uses: actions/download-artifact@v4
with:
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
- id: cargo-dist
shell: bash
run: |
dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json
echo "dist ran successfully"
# Parse out what we just built and upload it to scratch storage
echo "paths<<EOF" >> "$GITHUB_OUTPUT"
jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
echo "EOF" >> "$GITHUB_OUTPUT"
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
- name: "Upload artifacts"
uses: actions/upload-artifact@v4
with:
name: artifacts-build-global
path: |
${{ steps.cargo-dist.outputs.paths }}
${{ env.BUILD_MANIFEST_NAME }}
# Determines if we should publish/announce
host:
needs:
- plan
- build-local-artifacts
- build-global-artifacts
# Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine)
if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
runs-on: "ubuntu-22.04"
outputs:
val: ${{ steps.host.outputs.manifest }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install cached dist
uses: actions/download-artifact@v4
with:
name: cargo-dist-cache
path: ~/.cargo/bin/
- run: chmod +x ~/.cargo/bin/dist
# Fetch artifacts from scratch-storage
- name: Fetch artifacts
uses: actions/download-artifact@v4
with:
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
- id: host
shell: bash
run: |
dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json
echo "artifacts uploaded and released successfully"
cat dist-manifest.json
echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
- name: "Upload dist-manifest.json"
uses: actions/upload-artifact@v4
with:
# Overwrite the previous copy
name: artifacts-dist-manifest
path: dist-manifest.json
# Create a GitHub Release while uploading all files to it
- name: "Download GitHub Artifacts"
uses: actions/download-artifact@v4
with:
pattern: artifacts-*
path: artifacts
merge-multiple: true
- name: Cleanup
run: |
# Remove the granular manifests
rm -f artifacts/*-dist-manifest.json
- name: Create GitHub Release
env:
PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}"
ANNOUNCEMENT_TITLE: "${{ fromJson(steps.host.outputs.manifest).announcement_title }}"
ANNOUNCEMENT_BODY: "${{ fromJson(steps.host.outputs.manifest).announcement_github_body }}"
RELEASE_COMMIT: "${{ github.sha }}"
run: |
# Write and read notes from a file to avoid quoting breaking things
echo "$ANNOUNCEMENT_BODY" > $RUNNER_TEMP/notes.txt
gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/*
announce:
needs:
- plan
- host
# use "always() && ..." to allow us to wait for all publish jobs while
# still allowing individual publish jobs to skip themselves (for prereleases).
# "host" however must run to completion, no skipping allowed!
if: ${{ always() && needs.host.result == 'success' }}
runs-on: "ubuntu-22.04"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
-1
View File
@@ -1 +0,0 @@
Codex will review your output once you are done.
Generated
+123 -134
View File
@@ -103,9 +103,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "autocfg"
version = "1.5.0"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "axum"
@@ -176,11 +176,10 @@ dependencies = [
[[package]]
name = "base58ck"
version = "0.1.0"
version = "0.1.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f"
checksum = "ec5dc7e09f7bb15f0062da7c03086d6b71a2c84e0af4fccbbc7d8c6559847816"
dependencies = [
"bitcoin-internals",
"bitcoin_hashes",
]
@@ -222,20 +221,19 @@ dependencies = [
"quote",
"regex",
"rustc-hash",
"shlex",
"shlex 1.3.0",
"syn",
]
[[package]]
name = "bitcoin"
version = "0.32.9"
version = "0.32.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cf93e61f2dbc3e3c41234ca26a65e2c0b0975c52e0f069ab9893ebbede584d3"
checksum = "39581299241111285f3268ba75ddf372746fd041620918b145c1af9d75e91b6c"
dependencies = [
"base58ck",
"base64 0.21.7",
"bech32",
"bitcoin-internals",
"bitcoin-io",
"bitcoin-units",
"bitcoin_hashes",
@@ -245,36 +243,26 @@ dependencies = [
"serde",
]
[[package]]
name = "bitcoin-internals"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30bdbe14aa07b06e6cfeffc529a1f099e5fbe249524f8125358604df99a4bed2"
dependencies = [
"serde",
]
[[package]]
name = "bitcoin-io"
version = "0.1.4"
version = "0.1.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953"
checksum = "11301df0b06f22dea7bb1916403fdd88a371031e495c49b8f96931b28189e175"
[[package]]
name = "bitcoin-units"
version = "0.1.3"
version = "0.1.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "346568ebaab2918487cea76dd55dae13c27bb618cdb737c952e69eb2017c4118"
checksum = "57bad157b78d0d1b22c4cbb6a35a566211fc4d14866a37f2c780652b50f3b845"
dependencies = [
"bitcoin-internals",
"serde",
]
[[package]]
name = "bitcoin_hashes"
version = "0.14.1"
version = "0.14.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26ec84b80c482df901772e931a9a681e26a1b9ee2302edeff23cb30328745c8b"
checksum = "0c9901a56e133a1fc86eeb1113e2591f45f4682451ca893bff494d2f88918e3f"
dependencies = [
"bitcoin-io",
"hex-conservative",
@@ -295,7 +283,7 @@ checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
[[package]]
name = "blk"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"bitcoin",
"brk_error",
@@ -308,7 +296,7 @@ dependencies = [
[[package]]
name = "brk"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"brk_bencher",
"brk_bindgen",
@@ -333,7 +321,7 @@ dependencies = [
[[package]]
name = "brk_alloc"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"libmimalloc-sys",
"mimalloc",
@@ -341,7 +329,7 @@ dependencies = [
[[package]]
name = "brk_bencher"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"brk_error",
"brk_logger",
@@ -351,14 +339,14 @@ dependencies = [
[[package]]
name = "brk_bencher_visualizer"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"plotters",
]
[[package]]
name = "brk_bindgen"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"brk_cohort",
"brk_query",
@@ -371,7 +359,7 @@ dependencies = [
[[package]]
name = "brk_cli"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"anyhow",
"brk_alloc",
@@ -396,7 +384,7 @@ dependencies = [
[[package]]
name = "brk_client"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"brk_cohort",
"brk_types",
@@ -407,7 +395,7 @@ dependencies = [
[[package]]
name = "brk_cohort"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"brk_error",
"brk_traversable",
@@ -419,7 +407,7 @@ dependencies = [
[[package]]
name = "brk_computer"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"bitcoin",
"brk_alloc",
@@ -448,7 +436,7 @@ dependencies = [
[[package]]
name = "brk_error"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"bitcoin",
"fjall",
@@ -464,7 +452,7 @@ dependencies = [
[[package]]
name = "brk_fetcher"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"brk_error",
"brk_logger",
@@ -476,7 +464,7 @@ dependencies = [
[[package]]
name = "brk_indexer"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"bitcoin",
"brk_alloc",
@@ -502,7 +490,7 @@ dependencies = [
[[package]]
name = "brk_iterator"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"bitcoin",
"brk_error",
@@ -513,7 +501,7 @@ dependencies = [
[[package]]
name = "brk_logger"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"jiff",
"owo-colors",
@@ -524,7 +512,7 @@ dependencies = [
[[package]]
name = "brk_mempool"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"bitcoin",
"brk_error",
@@ -540,7 +528,7 @@ dependencies = [
[[package]]
name = "brk_oracle"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"brk_indexer",
"brk_types",
@@ -550,7 +538,7 @@ dependencies = [
[[package]]
name = "brk_query"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"bitcoin",
"brk_computer",
@@ -575,7 +563,7 @@ dependencies = [
[[package]]
name = "brk_reader"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"bitcoin",
"brk_error",
@@ -591,7 +579,7 @@ dependencies = [
[[package]]
name = "brk_rpc"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"bitcoin",
"brk_error",
@@ -608,7 +596,7 @@ dependencies = [
[[package]]
name = "brk_server"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"aide",
"axum",
@@ -618,6 +606,7 @@ dependencies = [
"brk_indexer",
"brk_logger",
"brk_mempool",
"brk_oracle",
"brk_query",
"brk_reader",
"brk_rpc",
@@ -640,7 +629,7 @@ dependencies = [
[[package]]
name = "brk_store"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"brk_error",
"brk_types",
@@ -651,7 +640,7 @@ dependencies = [
[[package]]
name = "brk_traversable"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"brk_traversable_derive",
"brk_types",
@@ -664,7 +653,7 @@ dependencies = [
[[package]]
name = "brk_traversable_derive"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"proc-macro2",
"quote",
@@ -673,7 +662,7 @@ dependencies = [
[[package]]
name = "brk_types"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"bitcoin",
"brk_error",
@@ -696,7 +685,7 @@ dependencies = [
[[package]]
name = "brk_website"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"axum",
"brk_logger",
@@ -711,9 +700,9 @@ dependencies = [
[[package]]
name = "brotli"
version = "8.0.2"
version = "8.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560"
checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@@ -722,9 +711,9 @@ dependencies = [
[[package]]
name = "brotli-decompressor"
version = "5.0.0"
version = "5.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03"
checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@@ -732,9 +721,9 @@ dependencies = [
[[package]]
name = "bumpalo"
version = "3.20.2"
version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytemuck"
@@ -768,14 +757,14 @@ checksum = "1c53ba0f290bfc610084c05582d9c5d421662128fc69f4bf236707af6fd321b9"
[[package]]
name = "cc"
version = "1.2.62"
version = "1.2.63"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
dependencies = [
"find-msvc-tools",
"jobserver",
"libc",
"shlex",
"shlex 2.0.1",
]
[[package]]
@@ -968,9 +957,9 @@ dependencies = [
[[package]]
name = "corepc-types"
version = "0.13.0"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b96c7869aa8234d10a41cbe3a1697bcb3a2482c48d9eb3541b3a4014a81afdad"
checksum = "f095534efdb8f2f43d48b9c3e9f35aefdf29ec6a5f1895064f575a67bc2a8dfe"
dependencies = [
"bitcoin",
"serde",
@@ -1066,9 +1055,9 @@ checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
[[package]]
name = "dashmap"
version = "6.1.0"
version = "6.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c"
dependencies = [
"cfg-if",
"crossbeam-utils",
@@ -1133,9 +1122,9 @@ dependencies = [
[[package]]
name = "displaydoc"
version = "0.2.5"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
dependencies = [
"proc-macro2",
"quote",
@@ -1186,9 +1175,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "either"
version = "1.15.0"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
[[package]]
name = "enum_dispatch"
@@ -1251,9 +1240,9 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "fjall"
version = "3.0.4"
version = "3.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ebf22b812878dcd767879cb19e03124fd62563dce6410f96538175fba0c132d"
checksum = "b62b25b4d815ae178d7d9e4aa32ee59f072efd5431c736abede1e6ee13c8c453"
dependencies = [
"byteorder-lite",
"byteview",
@@ -1261,7 +1250,7 @@ dependencies = [
"flume",
"log",
"lsm-tree",
"lz4_flex 0.11.6",
"lz4_flex",
"tempfile",
"xxhash-rust",
]
@@ -1533,9 +1522,9 @@ checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd"
[[package]]
name = "http"
version = "1.4.0"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0"
dependencies = [
"bytes",
"itoa",
@@ -1578,9 +1567,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "hyper"
version = "1.9.0"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"
checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
dependencies = [
"atomic-waker",
"bytes",
@@ -1849,23 +1838,23 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "jiff"
version = "0.2.24"
version = "0.2.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d"
checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102"
dependencies = [
"jiff-static",
"log",
"portable-atomic",
"portable-atomic-util",
"serde_core",
"windows-sys 0.61.2",
"windows-link",
]
[[package]]
name = "jiff-static"
version = "0.2.24"
version = "0.2.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7"
checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47"
dependencies = [
"proc-macro2",
"quote",
@@ -1890,9 +1879,9 @@ checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
[[package]]
name = "js-sys"
version = "0.3.98"
version = "0.3.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"
checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
dependencies = [
"cfg-if",
"futures-util",
@@ -1947,9 +1936,9 @@ dependencies = [
[[package]]
name = "libmimalloc-sys"
version = "0.1.47"
version = "0.1.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6"
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
dependencies = [
"cc",
"cty",
@@ -1968,9 +1957,9 @@ dependencies = [
[[package]]
name = "libredox"
version = "0.1.16"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c"
checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3"
dependencies = [
"libc",
]
@@ -2004,15 +1993,15 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.29"
version = "0.4.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5"
[[package]]
name = "lsm-tree"
version = "3.0.4"
version = "3.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9bfd2a6ea0c1d430c13643002f35800a87f200fc8ac4827f18a2db9d9fd0644"
checksum = "e447ac67ff6aef4ec07fc19e507b219336cbba90a697c0dbeb1bf51b91536b67"
dependencies = [
"byteorder-lite",
"byteview",
@@ -2020,7 +2009,7 @@ dependencies = [
"enum_dispatch",
"interval-heap",
"log",
"lz4_flex 0.11.6",
"lz4_flex",
"quick_cache",
"rustc-hash",
"self_cell",
@@ -2030,20 +2019,14 @@ dependencies = [
"xxhash-rust",
]
[[package]]
name = "lz4_flex"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
dependencies = [
"twox-hash",
]
[[package]]
name = "lz4_flex"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
dependencies = [
"twox-hash",
]
[[package]]
name = "matchit"
@@ -2053,9 +2036,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
[[package]]
name = "memchr"
version = "2.8.0"
version = "2.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
[[package]]
name = "memmap2"
@@ -2068,9 +2051,9 @@ dependencies = [
[[package]]
name = "mimalloc"
version = "0.1.50"
version = "0.1.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3627c4272df786b9260cabaa46aec1d59c93ede723d4c3ef646c503816b0640"
checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
dependencies = [
"libmimalloc-sys",
]
@@ -2099,9 +2082,9 @@ dependencies = [
[[package]]
name = "mio"
version = "1.2.0"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
dependencies = [
"libc",
"wasi",
@@ -2110,7 +2093,7 @@ dependencies = [
[[package]]
name = "mmpl"
version = "0.3.0-beta.11"
version = "0.3.2"
dependencies = [
"brk_error",
"brk_mempool",
@@ -2133,9 +2116,9 @@ dependencies = [
[[package]]
name = "num-conv"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
[[package]]
name = "num-traits"
@@ -2387,9 +2370,9 @@ dependencies = [
[[package]]
name = "quickmatch"
version = "0.4.0"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "848244615004bddb7273545dfe909ead495ed734f9faf130c43a7daccca2bf99"
checksum = "d6abd98fde5c9aa23590316caedc9e90be29da035d33905cdae71a54f034aaac"
dependencies = [
"rustc-hash",
]
@@ -2773,9 +2756,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.149"
version = "1.0.150"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
dependencies = [
"indexmap",
"itoa",
@@ -2856,6 +2839,12 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "shlex"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
[[package]]
name = "simd-adler32"
version = "0.3.9"
@@ -2876,9 +2865,9 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
[[package]]
name = "socket2"
version = "0.6.3"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
dependencies = [
"libc",
"windows-sys 0.61.2",
@@ -3150,9 +3139,9 @@ dependencies = [
[[package]]
name = "tower-http"
version = "0.6.10"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51"
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
dependencies = [
"async-compression",
"bitflags 2.11.1",
@@ -3266,9 +3255,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unicode-segmentation"
version = "1.13.2"
version = "1.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
[[package]]
name = "unicode-xid"
@@ -3360,7 +3349,7 @@ dependencies = [
"itoa",
"libc",
"log",
"lz4_flex 0.13.1",
"lz4_flex",
"parking_lot",
"pco",
"rawdb",
@@ -3426,9 +3415,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
version = "0.2.121"
version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"
checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
dependencies = [
"cfg-if",
"once_cell",
@@ -3439,9 +3428,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.121"
version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"
checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -3449,9 +3438,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.121"
version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"
checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
dependencies = [
"bumpalo",
"proc-macro2",
@@ -3462,9 +3451,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.121"
version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"
checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
dependencies = [
"unicode-ident",
]
@@ -3505,9 +3494,9 @@ dependencies = [
[[package]]
name = "web-sys"
version = "0.3.98"
version = "0.3.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"
checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -3857,18 +3846,18 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.48"
version = "0.8.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.48"
version = "0.8.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
dependencies = [
"proc-macro2",
"quote",
+30 -40
View File
@@ -4,7 +4,7 @@ members = ["crates/*"]
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
package.license = "MIT"
package.edition = "2024"
package.version = "0.3.0-beta.11"
package.version = "0.3.2"
package.homepage = "https://bitcoinresearchkit.org"
package.repository = "https://github.com/bitcoinresearchkit/brk"
package.readme = "README.md"
@@ -28,9 +28,6 @@ lto = false
strip = false
inherits = "release"
[profile.dist]
inherits = "release"
[profile.profiling]
inherits = "release"
debug = true
@@ -38,38 +35,38 @@ debug = true
[workspace.dependencies]
aide = { version = "0.16.0-alpha.4", features = ["axum-json", "axum-query"] }
axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] }
bitcoin = { version = "0.32.9", features = ["serde"] }
brk_alloc = { version = "0.3.0-beta.11", path = "crates/brk_alloc" }
brk_bencher = { version = "0.3.0-beta.11", path = "crates/brk_bencher" }
brk_bindgen = { version = "0.3.0-beta.11", path = "crates/brk_bindgen" }
brk_cli = { version = "0.3.0-beta.11", path = "crates/brk_cli" }
brk_client = { version = "0.3.0-beta.11", path = "crates/brk_client" }
brk_cohort = { version = "0.3.0-beta.11", path = "crates/brk_cohort" }
brk_computer = { version = "0.3.0-beta.11", path = "crates/brk_computer" }
brk_error = { version = "0.3.0-beta.11", path = "crates/brk_error" }
brk_fetcher = { version = "0.3.0-beta.11", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.3.0-beta.11", path = "crates/brk_indexer" }
brk_iterator = { version = "0.3.0-beta.11", path = "crates/brk_iterator" }
brk_logger = { version = "0.3.0-beta.11", path = "crates/brk_logger" }
brk_mempool = { version = "0.3.0-beta.11", path = "crates/brk_mempool" }
brk_oracle = { version = "0.3.0-beta.11", path = "crates/brk_oracle" }
brk_query = { version = "0.3.0-beta.11", path = "crates/brk_query", features = ["tokio"] }
brk_reader = { version = "0.3.0-beta.11", path = "crates/brk_reader" }
brk_rpc = { version = "0.3.0-beta.11", path = "crates/brk_rpc" }
brk_server = { version = "0.3.0-beta.11", path = "crates/brk_server" }
brk_store = { version = "0.3.0-beta.11", path = "crates/brk_store" }
brk_traversable = { version = "0.3.0-beta.11", path = "crates/brk_traversable", features = ["pco", "derive"] }
brk_traversable_derive = { version = "0.3.0-beta.11", path = "crates/brk_traversable_derive" }
brk_types = { version = "0.3.0-beta.11", path = "crates/brk_types" }
brk_website = { version = "0.3.0-beta.11", path = "crates/brk_website" }
bitcoin = { version = "0.32.100", features = ["serde"] }
brk_alloc = { version = "0.3.2", path = "crates/brk_alloc" }
brk_bencher = { version = "0.3.2", path = "crates/brk_bencher" }
brk_bindgen = { version = "0.3.2", path = "crates/brk_bindgen" }
brk_cli = { version = "0.3.2", path = "crates/brk_cli" }
brk_client = { version = "0.3.2", path = "crates/brk_client" }
brk_cohort = { version = "0.3.2", path = "crates/brk_cohort" }
brk_computer = { version = "0.3.2", path = "crates/brk_computer" }
brk_error = { version = "0.3.2", path = "crates/brk_error" }
brk_fetcher = { version = "0.3.2", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.3.2", path = "crates/brk_indexer" }
brk_iterator = { version = "0.3.2", path = "crates/brk_iterator" }
brk_logger = { version = "0.3.2", path = "crates/brk_logger" }
brk_mempool = { version = "0.3.2", path = "crates/brk_mempool" }
brk_oracle = { version = "0.3.2", path = "crates/brk_oracle" }
brk_query = { version = "0.3.2", path = "crates/brk_query", features = ["tokio"] }
brk_reader = { version = "0.3.2", path = "crates/brk_reader" }
brk_rpc = { version = "0.3.2", path = "crates/brk_rpc" }
brk_server = { version = "0.3.2", path = "crates/brk_server" }
brk_store = { version = "0.3.2", path = "crates/brk_store" }
brk_traversable = { version = "0.3.2", path = "crates/brk_traversable", features = ["pco", "derive"] }
brk_traversable_derive = { version = "0.3.2", path = "crates/brk_traversable_derive" }
brk_types = { version = "0.3.2", path = "crates/brk_types" }
brk_website = { version = "0.3.2", path = "crates/brk_website" }
byteview = "0.10.1"
color-eyre = "0.6.5"
corepc-jsonrpc = { package = "jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false }
corepc-types = { version = "0.13.0", features = ["std"], default-features = false }
corepc-types = { version = "0.14.0", features = ["std"], default-features = false }
derive_more = { version = "2.1.1", features = ["deref", "deref_mut"] }
fjall = "=3.0.4"
fjall = "3.1.4"
indexmap = { version = "2.14.0", features = ["serde"] }
jiff = { version = "0.2.24", features = ["perf-inline", "tz-system"], default-features = false }
jiff = { version = "0.2.28", features = ["perf-inline", "tz-system"], default-features = false }
owo-colors = "4.3.0"
parking_lot = "0.12.5"
pco = "1.0.2"
@@ -79,10 +76,10 @@ schemars = { version = "1.2.1", features = ["indexmap2"] }
serde = "1.0.228"
serde_bytes = "0.11.19"
serde_derive = "1.0.228"
serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] }
serde_json = { version = "1.0.150", features = ["float_roundtrip", "preserve_order"] }
smallvec = "1.15.1"
tokio = { version = "1.52.3", features = ["rt-multi-thread"] }
tower-http = { version = "0.6.10", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
tower-http = { version = "0.6.11", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
tower-layer = "0.3"
tracing = { version = "0.1", default-features = false, features = ["std"] }
ureq = { version = "3.3.0", features = ["json"] }
@@ -95,10 +92,3 @@ tag-name = "v{{version}}"
pre-release-commit-message = "release: v{{version}}"
tag-message = "release: v{{version}}"
allow-branch = ["main", "next"]
[workspace.metadata.dist]
cargo-dist-version = "0.30.2"
ci = "github"
allow-dirty = ["ci"]
installers = []
targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]
+1 -1
View File
@@ -1,6 +1,6 @@
MIT License
Copyright (c) 2025 bitcoinresearchkit, kibo.money, satonomics
Copyright (c) 2025 Bitcoin Research Kit
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
+2 -2
View File
@@ -8,5 +8,5 @@ homepage.workspace = true
repository.workspace = true
[dependencies]
libmimalloc-sys = { version = "0.1.47", features = ["extended"] }
mimalloc = { version = "0.1.50" }
libmimalloc-sys = { version = "0.1.49", features = ["extended"] }
mimalloc = { version = "0.1.52" }
@@ -51,7 +51,7 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
}
writeln!(
output,
" * @param {{{{ signal?: AbortSignal, onValue?: (value: {}) => void }}}} [options]",
" * @param {{{{ signal?: AbortSignal, onValue?: (value: {}) => void, cache?: boolean }}}} [options]",
return_type
)
.unwrap();
@@ -60,22 +60,22 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
let params = build_method_params(endpoint);
let params_with_opts = if params.is_empty() {
"{ signal, onValue } = {}".to_string()
"{ signal, onValue, cache } = {}".to_string()
} else {
format!("{}, {{ signal, onValue }} = {{}}", params)
format!("{}, {{ signal, onValue, cache }} = {{}}", params)
};
writeln!(output, " async {}({}) {{", method_name, params_with_opts).unwrap();
let path = build_path_template(&endpoint.path, &endpoint.path_params);
let fetch_call: String = if endpoint.returns_binary() {
"this.getBytes(path, { signal, onValue })".to_string()
"this.getBytes(path, { signal, onValue, cache })".to_string()
} else if endpoint.returns_json() {
"this.getJson(path, { signal, onValue })".to_string()
"this.getJson(path, { signal, onValue, cache })".to_string()
} else if endpoint.response_kind.text_is_numeric() {
"Number(await this.getText(path, { signal, onValue: onValue ? (v) => onValue(Number(v)) : undefined }))".to_string()
"Number(await this.getText(path, { signal, cache, onValue: onValue ? (v) => onValue(Number(v)) : undefined }))".to_string()
} else {
"this.getText(path, { signal, onValue })".to_string()
"this.getText(path, { signal, onValue, cache })".to_string()
};
write_path_assignment(output, endpoint, &path);
@@ -83,7 +83,7 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
if endpoint.supports_csv {
writeln!(
output,
" if (format === 'csv') return this.getText(path, {{ signal, onValue }});"
" if (format === 'csv') return this.getText(path, {{ signal, onValue, cache }});"
)
.unwrap();
}
@@ -448,14 +448,17 @@ class BrkClientBase {{
/**
* @param {{string}} path
* @param {{{{ signal?: AbortSignal }}}} [options]
* @param {{{{ signal?: AbortSignal, cache?: boolean }}}} [options]
* @returns {{Promise<Response>}}
*/
async get(path, {{ signal }} = {{}}) {{
async get(path, {{ signal, cache = true }} = {{}}) {{
const url = `${{this.baseUrl}}${{path}}`;
const signals = [AbortSignal.timeout(this.timeout)];
if (signal) signals.push(signal);
const res = await fetch(url, {{ signal: AbortSignal.any(signals) }});
/** @type {{RequestInit}} */
const init = {{ signal: AbortSignal.any(signals) }};
if (!cache) init.cache = 'no-store';
const res = await fetch(url, init);
if (!res.ok) throw new BrkError(`HTTP ${{res.status}}: ${{url}}`, res.status);
return res;
}}
@@ -475,14 +478,21 @@ class BrkClientBase {{
* @template T
* @param {{string}} path
* @param {{(res: Response) => Promise<T>}} parse - Response body reader
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal }}}} [options]
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal, cache?: boolean }}}} [options]
* @returns {{Promise<T>}}
*/
async _getCached(path, parse, {{ onValue, signal }} = {{}}) {{
async _getCached(path, parse, {{ onValue, signal, cache = true }} = {{}}) {{
if (!cache) {{
const res = await this.get(path, {{ signal, cache }});
const value = await parse(res);
if (onValue) onValue(value);
return value;
}}
const url = `${{this.baseUrl}}${{path}}`;
/** @type {{_MemEntry<T> | undefined}} */
const memHit = this._memGet(url);
const browserCache = this._browserCache ?? await this._browserCachePromise;
const browserCache = this._browserCache;
// L1 fast path: deliver from memCache, revalidate via network.
// ETag match → zero parse, zero clone, zero cache write, no second onValue fire.
@@ -497,8 +507,8 @@ class BrkClientBase {{
this._memSet(url, netEtag, value);
if (onValue) onValue(value);
if (cloned && browserCache) {{
const cache = browserCache;
_runIdle(() => cache.put(url, cloned));
const cacheStore = browserCache;
_runIdle(() => cacheStore.put(url, cloned));
}}
return value;
}} catch {{
@@ -531,8 +541,8 @@ class BrkClientBase {{
this._memSet(url, netEtag, value);
if (onValue) onValue(value);
if (cloned && browserCache) {{
const cache = browserCache;
_runIdle(() => cache.put(url, cloned));
const cacheStore = browserCache;
_runIdle(() => cacheStore.put(url, cloned));
}}
return value;
}} catch (e) {{
@@ -546,7 +556,7 @@ class BrkClientBase {{
* Make a GET request expecting a JSON response. Cached and supports `onValue`.
* @template T
* @param {{string}} path
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal }}}} [options]
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal, cache?: boolean }}}} [options]
* @returns {{Promise<T>}}
*/
getJson(path, options) {{
@@ -557,7 +567,7 @@ class BrkClientBase {{
* Make a GET request expecting a text response (text/plain, text/csv, ...).
* Cached and supports `onValue`, same as `getJson`.
* @param {{string}} path
* @param {{{{ onValue?: (value: string) => void, signal?: AbortSignal }}}} [options]
* @param {{{{ onValue?: (value: string) => void, signal?: AbortSignal, cache?: boolean }}}} [options]
* @returns {{Promise<string>}}
*/
getText(path, options) {{
@@ -568,7 +578,7 @@ class BrkClientBase {{
* Make a GET request expecting binary data (application/octet-stream).
* Cached and supports `onValue`, same as `getJson`.
* @param {{string}} path
* @param {{{{ onValue?: (value: Uint8Array) => void, signal?: AbortSignal }}}} [options]
* @param {{{{ onValue?: (value: Uint8Array) => void, signal?: AbortSignal, cache?: boolean }}}} [options]
* @returns {{Promise<Uint8Array>}}
*/
getBytes(path, options) {{
-3
View File
@@ -31,6 +31,3 @@ vecdb = { workspace = true }
[[bin]]
name = "brk"
path = "src/main.rs"
[package.metadata.dist]
dist = true
+4
View File
@@ -30,6 +30,10 @@ Portable build (without native CPU optimizations):
cargo install --locked brk_cli
```
## Update
Updating is the same as installing: re-run the install command above. `cargo install` overwrites the existing binary in place. Indexed data is reused when the on-disk format is unchanged, otherwise it is reset and re-synced automatically on the next run.
## Run
```bash
+62 -17
View File
@@ -3534,7 +3534,7 @@ pub struct SeriesTree {
pub investing: SeriesTree_Investing,
pub market: SeriesTree_Market,
pub pools: SeriesTree_Pools,
pub prices: SeriesTree_Prices,
pub price: SeriesTree_Price,
pub supply: SeriesTree_Supply,
pub cohorts: SeriesTree_Cohorts,
}
@@ -3556,7 +3556,7 @@ impl SeriesTree {
investing: SeriesTree_Investing::new(client.clone(), format!("{base_path}_investing")),
market: SeriesTree_Market::new(client.clone(), format!("{base_path}_market")),
pools: SeriesTree_Pools::new(client.clone(), format!("{base_path}_pools")),
prices: SeriesTree_Prices::new(client.clone(), format!("{base_path}_prices")),
price: SeriesTree_Price::new(client.clone(), format!("{base_path}_price")),
supply: SeriesTree_Supply::new(client.clone(), format!("{base_path}_supply")),
cohorts: SeriesTree_Cohorts::new(client.clone(), format!("{base_path}_cohorts")),
}
@@ -7063,31 +7063,31 @@ impl SeriesTree_Pools_Minor {
}
/// Series tree node.
pub struct SeriesTree_Prices {
pub split: SeriesTree_Prices_Split,
pub ohlc: SeriesTree_Prices_Ohlc,
pub spot: SeriesTree_Prices_Spot,
pub struct SeriesTree_Price {
pub split: SeriesTree_Price_Split,
pub ohlc: SeriesTree_Price_Ohlc,
pub spot: SeriesTree_Price_Spot,
}
impl SeriesTree_Prices {
impl SeriesTree_Price {
pub fn new(client: Arc<BrkClientBase>, base_path: String) -> Self {
Self {
split: SeriesTree_Prices_Split::new(client.clone(), format!("{base_path}_split")),
ohlc: SeriesTree_Prices_Ohlc::new(client.clone(), format!("{base_path}_ohlc")),
spot: SeriesTree_Prices_Spot::new(client.clone(), format!("{base_path}_spot")),
split: SeriesTree_Price_Split::new(client.clone(), format!("{base_path}_split")),
ohlc: SeriesTree_Price_Ohlc::new(client.clone(), format!("{base_path}_ohlc")),
spot: SeriesTree_Price_Spot::new(client.clone(), format!("{base_path}_spot")),
}
}
}
/// Series tree node.
pub struct SeriesTree_Prices_Split {
pub struct SeriesTree_Price_Split {
pub open: CentsSatsUsdPattern3,
pub high: CentsSatsUsdPattern3,
pub low: CentsSatsUsdPattern3,
pub close: CentsSatsUsdPattern3,
}
impl SeriesTree_Prices_Split {
impl SeriesTree_Price_Split {
pub fn new(client: Arc<BrkClientBase>, base_path: String) -> Self {
Self {
open: CentsSatsUsdPattern3::new(client.clone(), "price_open".to_string()),
@@ -7099,13 +7099,13 @@ impl SeriesTree_Prices_Split {
}
/// Series tree node.
pub struct SeriesTree_Prices_Ohlc {
pub struct SeriesTree_Price_Ohlc {
pub usd: SeriesPattern2<OHLCDollars>,
pub cents: SeriesPattern2<OHLCCents>,
pub sats: SeriesPattern2<OHLCSats>,
}
impl SeriesTree_Prices_Ohlc {
impl SeriesTree_Price_Ohlc {
pub fn new(client: Arc<BrkClientBase>, base_path: String) -> Self {
Self {
usd: SeriesPattern2::new(client.clone(), "price_ohlc".to_string()),
@@ -7116,13 +7116,13 @@ impl SeriesTree_Prices_Ohlc {
}
/// Series tree node.
pub struct SeriesTree_Prices_Spot {
pub struct SeriesTree_Price_Spot {
pub usd: SeriesPattern1<Dollars>,
pub cents: SeriesPattern1<Cents>,
pub sats: SeriesPattern1<Sats>,
}
impl SeriesTree_Prices_Spot {
impl SeriesTree_Price_Spot {
pub fn new(client: Arc<BrkClientBase>, base_path: String) -> Self {
Self {
usd: SeriesPattern1::new(client.clone(), "price".to_string()),
@@ -8953,7 +8953,7 @@ pub struct BrkClient {
impl BrkClient {
/// Client version.
pub const VERSION: &'static str = "v0.3.0-beta.9";
pub const VERSION: &'static str = "v0.3.1";
/// Create a new client with the given base URL.
pub fn new(base_url: impl Into<String>) -> Self {
@@ -9856,6 +9856,51 @@ impl BrkClient {
self.base.get_json(&format!("/api/mempool/price"))
}
/// Live BTC/USD price
///
/// Current BTC/USD price in dollars. Same value as `/api/mempool/price`. Confirmed per-height history is available at `/api/vecs/height-to-price`.
///
/// Endpoint: `GET /api/oracle/price`
pub fn get_oracle_price(&self) -> Result<Dollars> {
self.base.get_json(&format!("/api/oracle/price"))
}
/// Live payment output histogram
///
/// Live smoothed histogram of oracle-eligible payment outputs, binned by output value on the oracle log scale. It combines the committed oracle window with the forming mempool block. A flat array of log-scale bins.
///
/// Endpoint: `GET /api/oracle/histogram/payments/live`
pub fn get_oracle_histogram_payments_live(&self) -> Result<Vec<i64>> {
self.base.get_json(&format!("/api/oracle/histogram/payments/live"))
}
/// Payment output histogram at height or day
///
/// Smoothed histogram of oracle-eligible payment outputs for a confirmed point. A block height (`840000`) gives that block's oracle payment histogram; a calendar date (`YYYY-MM-DD`) gives the average of that day's per-block payment histograms. A flat array of log-scale bins.
///
/// Endpoint: `GET /api/oracle/histogram/payments/{point}`
pub fn get_oracle_histogram_payments(&self, point: &str) -> Result<Vec<i64>> {
self.base.get_json(&format!("/api/oracle/histogram/payments/{point}"))
}
/// Live output value histogram
///
/// Live unfiltered output value histogram for the forming mempool block. Every live output is binned by value on the oracle log scale; no oracle payment filters are applied. A flat array of log-scale bins, all zero when no mempool is configured.
///
/// Endpoint: `GET /api/oracle/histogram/outputs/live`
pub fn get_oracle_histogram_outputs_live(&self) -> Result<Vec<i64>> {
self.base.get_json(&format!("/api/oracle/histogram/outputs/live"))
}
/// Output value histogram at height or day
///
/// Unfiltered output value histogram for a confirmed point. A block height (`840000`) gives every output in that block, coinbase included, binned by value on the oracle log scale; a calendar date (`YYYY-MM-DD`) sums every block that day. A flat array of log-scale bins.
///
/// Endpoint: `GET /api/oracle/histogram/outputs/{point}`
pub fn get_oracle_histogram_outputs(&self, point: &str) -> Result<Vec<i64>> {
self.base.get_json(&format!("/api/oracle/histogram/outputs/{point}"))
}
/// Txid by index
///
/// Retrieve the transaction ID (txid) at a given global transaction index. Returns the txid as plain text.
+2 -2
View File
@@ -3,14 +3,14 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, distribution, mining, prices, supply};
use crate::{blocks, distribution, mining, price, supply};
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
blocks: &blocks::Vecs,
mining: &mining::Vecs,
supply_vecs: &supply::Vecs,
@@ -5,14 +5,14 @@ use vecdb::Exit;
use super::super::{activity, cap, supply};
use super::Vecs;
use crate::{distribution, prices};
use crate::{distribution, price};
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
distribution: &distribution::Vecs,
activity: &activity::Vecs,
supply: &supply::Vecs,
@@ -4,14 +4,14 @@ use brk_types::StoredF64;
use vecdb::Exit;
use super::{super::value, Vecs};
use crate::{blocks, internal::algo::ComputeRollingMedianFromStarts, prices};
use crate::{blocks, internal::algo::ComputeRollingMedianFromStarts, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
value: &value::Vecs,
exit: &Exit,
) -> Result<()> {
@@ -4,13 +4,13 @@ use vecdb::Exit;
use super::super::activity;
use super::Vecs;
use crate::{distribution, prices};
use crate::{distribution, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
distribution: &distribution::Vecs,
activity: &activity::Vecs,
exit: &Exit,
@@ -5,13 +5,13 @@ use vecdb::Exit;
use super::super::activity;
use super::Vecs;
use crate::{distribution, prices};
use crate::{distribution, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
distribution: &distribution::Vecs,
activity: &activity::Vecs,
exit: &Exit,
@@ -45,7 +45,7 @@ use super::{
count::AddrCountFundedTotalVecs,
supply::{AddrSupplyShareVecs, AddrSupplyVecs},
};
use crate::{indexes, prices};
use crate::{indexes, price};
mod state;
@@ -104,7 +104,7 @@ impl ExposedAddrVecs {
pub(crate) fn compute_rest(
&mut self,
starting_lengths: &Lengths,
prices: &prices::Vecs,
prices: &price::Vecs,
all_supply_sats: &impl ReadableVec<Height, Sats>,
type_supply_sats: &ByAddrType<&impl ReadableVec<Height, Sats>>,
exit: &Exit,
@@ -35,7 +35,7 @@ use super::{
use crate::{
indexes, inputs,
internal::{WindowStartVec, Windows},
outputs, prices,
outputs, price,
};
mod state;
@@ -112,7 +112,7 @@ impl ReusedAddrVecs {
starting_lengths: &Lengths,
outputs_by_type: &outputs::ByTypeVecs,
inputs_by_type: &inputs::ByTypeVecs,
prices: &prices::Vecs,
prices: &price::Vecs,
all_supply_sats: &impl ReadableVec<Height, Sats>,
type_supply_sats: &ByAddrType<&impl ReadableVec<Height, Sats>>,
exit: &Exit,
@@ -13,7 +13,7 @@ use crate::{
distribution::DynCohortVecs,
indexes,
internal::{WindowStartVec, Windows},
prices,
price,
};
use super::{super::traits::CohortVecs, vecs::AddrCohortVecs};
@@ -95,7 +95,7 @@ impl AddrCohorts {
/// First phase of post-processing: compute index transforms.
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -108,7 +108,7 @@ impl AddrCohorts {
/// Second phase of post-processing: compute relative metrics.
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -12,7 +12,7 @@ use crate::{
distribution::state::{AddrCohortState, MinimalRealizedState},
indexes,
internal::{PerBlockWithDeltas, WindowStartVec, Windows},
prices,
price,
};
use crate::distribution::metrics::{ImportConfig, MinimalCohortMetrics};
@@ -174,7 +174,7 @@ impl DynCohortVecs for AddrCohortVecs {
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -229,7 +229,7 @@ impl CohortVecs for AddrCohortVecs {
fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -3,7 +3,7 @@ use brk_indexer::Lengths;
use brk_types::{Cents, Height, Sats, StoredU64, Version};
use vecdb::{Exit, ReadableVec};
use crate::prices;
use crate::price;
/// Dynamic dispatch trait for cohort vectors.
///
@@ -31,7 +31,7 @@ pub trait DynCohortVecs: Send + Sync {
/// First phase of post-processing computations.
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()>;
@@ -61,7 +61,7 @@ pub trait CohortVecs: DynCohortVecs {
/// Second phase of post-processing computations.
fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -27,7 +27,7 @@ use crate::{
},
indexes,
internal::{ValuePerBlockCumulativeRolling, WindowStartVec, Windows},
prices,
price,
};
use super::{fenwick::CostBasisFenwick, vecs::UTXOCohortVecs};
@@ -483,7 +483,7 @@ impl UTXOCohorts<Rw> {
/// First phase of post-processing: compute index transforms.
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -546,7 +546,7 @@ impl UTXOCohorts<Rw> {
pub(crate) fn compute_rest_part2(
&mut self,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
exit: &Exit,
@@ -6,7 +6,7 @@ use vecdb::{Exit, ReadableVec};
use crate::{
distribution::{cohorts::traits::DynCohortVecs, metrics::CoreCohortMetrics},
prices,
price,
};
use super::UTXOCohortVecs;
@@ -56,7 +56,7 @@ impl DynCohortVecs for UTXOCohortVecs<CoreCohortMetrics> {
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -6,7 +6,7 @@ use vecdb::{Exit, ReadableVec};
use crate::{
distribution::{cohorts::traits::DynCohortVecs, metrics::MinimalCohortMetrics},
prices,
price,
};
use super::UTXOCohortVecs;
@@ -49,7 +49,7 @@ impl DynCohortVecs for UTXOCohortVecs<MinimalCohortMetrics> {
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -55,7 +55,7 @@ use crate::{
metrics::{CohortMetricsBase, CohortMetricsState},
state::UTXOCohortState,
},
prices,
price,
};
#[derive(Traversable)]
@@ -186,7 +186,7 @@ impl<M: CohortMetricsBase + Traversable> DynCohortVecs for UTXOCohortVecs<M> {
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -5,7 +5,7 @@ use brk_types::{Cents, Height, Version};
use vecdb::{Exit, ReadableVec};
use crate::{
distribution::cohorts::traits::DynCohortVecs, distribution::metrics::TypeCohortMetrics, prices,
distribution::cohorts::traits::DynCohortVecs, distribution::metrics::TypeCohortMetrics, price,
};
use super::UTXOCohortVecs;
@@ -55,7 +55,7 @@ impl DynCohortVecs for UTXOCohortVecs<TypeCohortMetrics> {
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -24,51 +24,60 @@ pub struct RecoveredState {
/// Returns Height::ZERO if any validation fails (triggers fresh start).
pub(crate) fn recover_state(
height: Height,
chain_state_rollback: vecdb::Result<Stamp>,
chain_state_rollback: Option<vecdb::Result<Stamp>>,
any_addr_indexes: &mut AnyAddrIndexesVecs,
addrs_data: &mut AddrsDataVecs,
utxo_cohorts: &mut UTXOCohorts,
addr_cohorts: &mut AddrCohorts,
) -> Result<RecoveredState> {
let stamp = Stamp::from(height);
// `None`: clean resume, already at the checkpoint, nothing to undo.
// `Some`: reorg, undo state past the resume point.
let consistent_height = match chain_state_rollback {
None => height,
Some(chain_state_rollback) => {
let stamp = Stamp::from(height);
// Rollback address state vectors
let addr_indexes_rollback = any_addr_indexes.rollback_before(stamp);
let addr_data_rollback = addrs_data.rollback_before(stamp);
// Rollback address state vectors
let addr_indexes_rollback = any_addr_indexes.rollback_before(stamp);
let addr_data_rollback = addrs_data.rollback_before(stamp);
// Verify rollback consistency - all must agree on the same height
let consistent_height = rollback_states(
chain_state_rollback,
addr_indexes_rollback,
addr_data_rollback,
);
// Verify rollback consistency - all must agree on the same height
let consistent_height = rollback_states(
chain_state_rollback,
addr_indexes_rollback,
addr_data_rollback,
);
// If rollbacks are inconsistent, start fresh
if consistent_height.is_zero() {
warn!("Rollback consistency check failed: inconsistent heights");
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
// If rollbacks are inconsistent, start fresh
if consistent_height.is_zero() {
warn!("Rollback consistency check failed: inconsistent heights");
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
// Rollback can land at an earlier height (multi-block change file), which is fine.
// But if it lands AHEAD of target, that means rollback failed (missing change files).
if consistent_height > height {
warn!(
"Rollback failed: still at {} but target was {}, falling back to fresh start",
consistent_height, height
);
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
// Rollback can land at an earlier height (multi-block change file), which is fine.
// But if it lands AHEAD of target, that means rollback failed (missing change files).
if consistent_height > height {
warn!(
"Rollback failed: still at {} but target was {}, falling back to fresh start",
consistent_height, height
);
return Ok(RecoveredState {
starting_height: Height::ZERO,
});
}
if consistent_height != height {
debug!(
"Rollback landed at {} instead of {}, will resume from there",
consistent_height, height
);
}
if consistent_height != height {
debug!(
"Rollback landed at {} instead of {}, will resume from there",
consistent_height, height
);
}
consistent_height
}
};
// Import UTXO cohort states - all must succeed
debug!(
@@ -11,7 +11,7 @@ use crate::{
state::{CohortState, CostBasisOps, RealizedOps},
},
internal::{PerBlockCumulativeRolling, ValuePerBlockCumulativeRolling},
prices,
price,
};
use super::ActivityMinimal;
@@ -98,7 +98,7 @@ impl ActivityCore {
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -12,7 +12,7 @@ use crate::{
metrics::ImportConfig,
state::{CohortState, CostBasisOps, RealizedOps},
},
prices,
price,
};
use super::ActivityCore;
@@ -89,7 +89,7 @@ impl ActivityFull {
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -10,7 +10,7 @@ use crate::{
state::{CohortState, CostBasisOps, RealizedOps},
},
internal::ValuePerBlockCumulativeRolling,
prices,
price,
};
#[derive(Traversable)]
@@ -63,7 +63,7 @@ impl ActivityMinimal {
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -13,7 +13,7 @@ use vecdb::Exit;
use crate::{
distribution::state::{CohortState, CostBasisOps, RealizedOps},
prices,
price,
};
pub trait ActivityLike: Send + Sync {
@@ -30,7 +30,7 @@ pub trait ActivityLike: Send + Sync {
) -> Result<()>;
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()>;
@@ -62,7 +62,7 @@ impl ActivityLike for ActivityCore {
}
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -96,7 +96,7 @@ impl ActivityLike for ActivityFull {
}
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -11,7 +11,7 @@ use crate::{
ActivityFull, AdjustedSopr, CohortMetricsBase, CostBasis, ImportConfig, OutputsBase,
RealizedFull, RelativeForAll, SupplyCore, UnrealizedFull,
},
prices,
price,
};
/// All-cohort metrics: extended realized + adjusted (as composable add-on),
@@ -100,7 +100,7 @@ impl AllCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
under_1h_value_created: &impl ReadableVec<Height, Cents>,
@@ -10,7 +10,7 @@ use crate::{
ActivityCore, CohortMetricsBase, ImportConfig, OutputsBase, RealizedCore, SupplyCore,
UnrealizedCore,
},
prices,
price,
};
/// Basic cohort metrics: no extensions, used by age_range cohorts.
@@ -61,7 +61,7 @@ impl BasicCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -10,7 +10,7 @@ use crate::{
ActivityCore, CohortMetricsBase, ImportConfig, OutputsBase, RealizedCore, SupplyCore,
UnrealizedCore,
},
prices,
price,
};
#[derive(Traversable)]
@@ -102,7 +102,7 @@ impl CoreCohortMetrics {
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -122,7 +122,7 @@ impl CoreCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -12,7 +12,7 @@ use crate::{
ActivityFull, CohortMetricsBase, CostBasis, ImportConfig, OutputsBase, RealizedFull,
RelativeWithExtended, SupplyCore, UnrealizedFull,
},
prices,
price,
};
/// Cohort metrics with extended realized + extended cost basis (no adjusted).
@@ -90,7 +90,7 @@ impl ExtendedCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
all_supply_sats: &impl ReadableVec<Height, Sats>,
@@ -10,7 +10,7 @@ use crate::{
distribution::metrics::{
ActivityFull, AdjustedSopr, CohortMetricsBase, ImportConfig, RealizedFull, UnrealizedFull,
},
prices,
price,
};
use super::ExtendedCohortMetrics;
@@ -62,7 +62,7 @@ impl ExtendedAdjustedCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
under_1h_value_created: &impl ReadableVec<Height, Cents>,
@@ -9,7 +9,7 @@ use crate::{
distribution::metrics::{
ActivityMinimal, ImportConfig, OutputsBase, RealizedMinimal, SupplyBase, UnrealizedMinimal,
},
prices,
price,
};
/// MinimalCohortMetrics: supply, outputs, realized cap/price/mvrv/profit/loss + value_created/destroyed.
@@ -97,7 +97,7 @@ impl MinimalCohortMetrics {
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -111,7 +111,7 @@ impl MinimalCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -9,7 +9,7 @@ use crate::{
distribution::metrics::{
ActivityMinimal, ImportConfig, OutputsBase, RealizedMinimal, SupplyCore, UnrealizedBasic,
},
prices,
price,
};
/// TypeCohortMetrics: supply(core), outputs(base), realized(minimal), unrealized(basic).
@@ -59,7 +59,7 @@ impl TypeCohortMetrics {
pub(crate) fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -73,7 +73,7 @@ impl TypeCohortMetrics {
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
all_supply_sats: &impl ReadableVec<Height, Sats>,
all_utxo_count: &impl ReadableVec<Height, StoredU64>,
@@ -149,7 +149,7 @@ use crate::{
CohortState, CoreRealizedState, CostBasisData, CostBasisOps, CostBasisRaw,
MinimalRealizedState, RealizedOps, RealizedState, WithCapital, WithoutCapital,
},
prices,
price,
};
pub trait CohortMetricsState {
@@ -270,7 +270,7 @@ pub trait CohortMetricsBase:
/// First phase of computed metrics (indexes from height).
fn compute_rest_part1(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -10,7 +10,7 @@ use crate::{
internal::{
PerBlock, RatioPerBlock, ValuePerBlock, ValuePerBlockWithDeltas, WindowStartVec, Windows,
},
prices,
price,
};
#[derive(Traversable)]
@@ -115,7 +115,7 @@ impl ProfitabilityBucket {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
is_profit: bool,
exit: &Exit,
@@ -176,7 +176,7 @@ impl ProfitabilityBucket {
pub(crate) fn compute_from_ranges(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
is_profit: bool,
sources: &[&ProfitabilityBucket],
@@ -293,7 +293,7 @@ impl ProfitabilityMetrics {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -16,7 +16,7 @@ use crate::{
FiatPerBlockCumulativeWithSumsAndDeltas, LazyPerBlock, NegCentsUnsignedToDollars,
PerBlockCumulativeRolling, RatioCents64, RollingWindow24hPerBlock, Windows,
},
prices,
price,
};
use crate::distribution::metrics::ImportConfig;
@@ -166,7 +166,7 @@ impl RealizedCore {
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_supply: &impl ReadableVec<Height, Bitcoin>,
transfer_volume_sum_24h_cents: &impl ReadableVec<Height, Cents>,
@@ -18,7 +18,7 @@ use crate::{
RatioPerBlockStdDevBands, RatioSma, RollingWindows, RollingWindowsFrom1w,
ValuePerBlockCumulativeRolling,
},
prices,
price,
};
use crate::distribution::metrics::ImportConfig;
@@ -240,7 +240,7 @@ impl RealizedFull {
pub(crate) fn compute_rest_part2(
&mut self,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_supply: &impl ReadableVec<Height, Bitcoin>,
height_to_market_cap: &impl ReadableVec<Height, Dollars>,
@@ -13,7 +13,7 @@ use crate::{
FiatPerBlockCumulativeWithSums, FiatPerBlockWithDeltas, Identity, LazyPerBlock,
PriceWithRatioPerBlock,
},
prices,
price,
};
use crate::distribution::metrics::ImportConfig;
@@ -104,7 +104,7 @@ impl RealizedMinimal {
pub(crate) fn compute_rest_part2(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
height_to_supply: &impl ReadableVec<Height, Bitcoin>,
exit: &Exit,
@@ -3,7 +3,7 @@ use brk_traversable::Traversable;
use brk_types::{Height, Sats, StoredU64, Version};
use vecdb::{AnyStoredVec, Database, Exit, ReadableVec, Rw, StorageMode, WritableVec};
use crate::{indexes, internal::ValuePerBlock, prices};
use crate::{indexes, internal::ValuePerBlock, price};
/// Average amount held per UTXO and per funded address.
///
@@ -53,7 +53,7 @@ impl AvgAmountMetrics {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
supply_sats: &impl ReadableVec<Height, Sats>,
utxo_count: &impl ReadableVec<Height, StoredU64>,
funded_addr_count: &impl ReadableVec<Height, StoredU64>,
@@ -6,7 +6,7 @@ use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, Rw, StorageMode, WritableVe
use crate::{
distribution::state::{CohortState, CostBasisOps, RealizedOps},
prices,
price,
};
use crate::internal::{
@@ -64,7 +64,7 @@ impl SupplyBase {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
max_from: Height,
exit: &Exit,
) -> Result<()> {
@@ -5,7 +5,7 @@ use brk_types::{Height, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{AnyStoredVec, AnyVec, Exit, Rw, StorageMode, WritableVec};
use crate::{distribution::state::UnrealizedState, prices};
use crate::{distribution::state::UnrealizedState, price};
use crate::internal::{
HalveCents, HalveDollars, HalveSats, HalveSatsToBitcoin, LazyValuePerBlock, ValuePerBlock,
@@ -72,7 +72,7 @@ impl SupplyCore {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
max_from: Height,
exit: &Exit,
) -> Result<()> {
@@ -7,7 +7,7 @@ use vecdb::{AnyStoredVec, AnyVec, BytesVec, Exit, ReadableVec, Rw, StorageMode,
use crate::distribution::state::UnrealizedState;
use crate::internal::{CentsSubtractToCentsSigned, FiatPerBlock};
use crate::{distribution::metrics::ImportConfig, prices};
use crate::{distribution::metrics::ImportConfig, price};
use super::UnrealizedCore;
@@ -99,7 +99,7 @@ impl UnrealizedFull {
pub(crate) fn compute_rest_all(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
supply_in_profit_sats: &(impl ReadableVec<Height, Sats> + Sync),
supply_in_loss_sats: &(impl ReadableVec<Height, Sats> + Sync),
@@ -13,7 +13,7 @@ use brk_indexer::Lengths;
use brk_types::{Height, Sats};
use vecdb::{Exit, ReadableVec};
use crate::{distribution::state::UnrealizedState, prices};
use crate::{distribution::state::UnrealizedState, price};
pub trait UnrealizedLike: Send + Sync {
fn as_core(&self) -> &UnrealizedCore;
@@ -22,7 +22,7 @@ pub trait UnrealizedLike: Send + Sync {
fn push_state(&mut self, state: &UnrealizedState);
fn compute_rest(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
supply_in_profit_sats: &(impl ReadableVec<Height, Sats> + Sync),
supply_in_loss_sats: &(impl ReadableVec<Height, Sats> + Sync),
@@ -46,7 +46,7 @@ impl UnrealizedLike for UnrealizedCore {
}
fn compute_rest(
&mut self,
_prices: &prices::Vecs,
_prices: &price::Vecs,
starting_lengths: &Lengths,
_supply_in_profit_sats: &(impl ReadableVec<Height, Sats> + Sync),
_supply_in_loss_sats: &(impl ReadableVec<Height, Sats> + Sync),
@@ -72,7 +72,7 @@ impl UnrealizedLike for UnrealizedFull {
}
fn compute_rest(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
supply_in_profit_sats: &(impl ReadableVec<Height, Sats> + Sync),
supply_in_loss_sats: &(impl ReadableVec<Height, Sats> + Sync),
+8 -7
View File
@@ -29,7 +29,7 @@ use crate::{
PerBlockCumulativeRolling, WindowStartVec, Windows, WithAddrTypes,
db_utils::{finalize_db, open_db},
},
outputs, prices, transactions,
outputs, price, transactions,
};
use super::{
@@ -316,7 +316,7 @@ impl Vecs {
outputs: &outputs::Vecs,
transactions: &transactions::Vecs,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
self.db.sync_bg_tasks()?;
@@ -341,12 +341,13 @@ impl Vecs {
// Try to resume from checkpoint, fall back to fresh start if needed
let recovered_height = match start_mode {
StartMode::Resume(height) => {
let stamp = Stamp::from(height);
// Roll back only on a reorg. A clean resume has nothing to undo, and an
// interrupted run wrote no rollback metadata (periodic flushes use
// with_changes=false; only the final write creates the `changes/` dir),
// so `rollback_before` would fail with `NotFound`.
let chain_state_rollback = (height < current_height)
.then(|| self.supply_state.rollback_before(Stamp::from(height)));
// Rollback BytesVec state and capture results for validation
let chain_state_rollback = self.supply_state.rollback_before(stamp);
// Validate all rollbacks and imports are consistent
let recovered = recover_state(
height,
chain_state_rollback,
@@ -6,7 +6,7 @@ use brk_traversable::Traversable;
use brk_types::Version;
use vecdb::{Database, Exit, Rw, StorageMode};
use crate::{distribution, indexes, prices};
use crate::{distribution, indexes, price};
pub use inner::RarityMeterInner;
@@ -37,7 +37,7 @@ impl RarityMeter {
&mut self,
indexer: &Indexer,
distribution: &distribution::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
let realized = &distribution.utxo_cohorts.all.metrics.realized;
@@ -6,7 +6,7 @@ use derive_more::{Deref, DerefMut};
use vecdb::{Database, EagerVec, Exit, PcoVec, ReadableVec, Rw, StorageMode};
use crate::internal::{LazyPerBlock, PerBlock, Price};
use crate::{indexes, prices};
use crate::{indexes, price};
use super::{RatioPerBlock, RatioPerBlockPercentiles};
@@ -63,7 +63,7 @@ impl PriceWithRatioPerBlock {
/// Compute price via closure (in cents), then compute ratio.
pub(crate) fn compute_all<F>(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
mut compute_price: F,
@@ -101,7 +101,7 @@ impl PriceWithRatioExtendedPerBlock {
/// Compute ratio and percentiles from already-computed price cents.
pub(crate) fn compute_rest(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
) -> Result<()> {
@@ -120,7 +120,7 @@ impl PriceWithRatioExtendedPerBlock {
/// Compute price via closure (in cents), then compute ratio and percentiles.
pub(crate) fn compute_all<F>(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
starting_lengths: &Lengths,
exit: &Exit,
mut compute_price: F,
@@ -10,7 +10,7 @@ use crate::{
CentsUnsignedToDollars, LazyPerBlock, NumericValue, PerBlock, SatsSignedToBitcoin,
SatsToBitcoin, SatsToCents,
},
prices,
price,
};
/// Trait that associates a sats type with its transform to Bitcoin.
@@ -69,7 +69,7 @@ impl ValuePerBlock {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
max_from: Height,
exit: &Exit,
) -> Result<()> {
@@ -8,7 +8,7 @@ use vecdb::{
use crate::{
internal::{CentsUnsignedToDollars, SatsToBitcoin, SatsToCents},
prices,
price,
};
/// Raw per-block amount data: sats + cents (stored), btc + usd (lazy), no resolutions.
@@ -44,7 +44,7 @@ impl ValueBlock {
pub(crate) fn compute_cents(
&mut self,
max_from: Height,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
self.cents.compute_binary::<Sats, Cents, SatsToCents>(
@@ -6,7 +6,7 @@ use vecdb::{Database, EagerVec, Exit, PcoVec, Rw, StorageMode};
use crate::{
indexes,
internal::{ValueBlock, ValuePerBlock},
prices,
price,
};
#[derive(Traversable)]
@@ -39,7 +39,7 @@ impl ValuePerBlockCumulative {
pub(crate) fn compute(
&mut self,
prices: &prices::Vecs,
prices: &price::Vecs,
max_from: Height,
exit: &Exit,
) -> Result<()> {
@@ -61,7 +61,7 @@ impl ValuePerBlockCumulative {
pub(crate) fn compute_with(
&mut self,
max_from: Height,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
compute_sats: impl FnOnce(&mut EagerVec<PcoVec<Height, Sats>>) -> Result<()>,
) -> Result<()> {
@@ -10,7 +10,7 @@ use crate::{
LazyRollingAvgsAmountFromHeight, LazyRollingSumsAmountFromHeight, ValuePerBlockCumulative,
WindowStartVec, Windows,
},
prices,
price,
};
#[derive(Deref, DerefMut, Traversable)]
@@ -63,7 +63,7 @@ impl ValuePerBlockCumulativeRolling {
pub(crate) fn compute(
&mut self,
max_from: Height,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
compute_sats: impl FnOnce(&mut EagerVec<PcoVec<Height, Sats>>) -> Result<()>,
) -> Result<()> {
@@ -74,7 +74,7 @@ impl ValuePerBlockCumulativeRolling {
pub(crate) fn compute_rest(
&mut self,
max_from: Height,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
self.inner.compute(prices, max_from, exit)
@@ -10,7 +10,7 @@ use crate::{
RollingDistributionValuePerBlock, ValuePerBlockCumulativeRolling, WindowStartVec,
WindowStarts, Windows,
},
prices,
price,
};
#[derive(Deref, DerefMut, Traversable)]
@@ -49,7 +49,7 @@ impl ValuePerBlockFull {
&mut self,
max_from: Height,
windows: &WindowStarts<'_>,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
compute_sats: impl FnOnce(&mut EagerVec<PcoVec<Height, Sats>>) -> Result<()>,
) -> Result<()> {
@@ -11,7 +11,7 @@ use rayon::prelude::*;
use schemars::JsonSchema;
use vecdb::{AnyStoredVec, AnyVec, Database, EagerVec, Exit, PcoVec, WritableVec};
use crate::{indexes, prices};
use crate::{indexes, price};
use super::{
BpsType, NumericValue, PerBlock, PerBlockCumulativeRolling, PercentPerBlock, ValuePerBlock,
@@ -229,7 +229,7 @@ impl WithAddrTypes<ValuePerBlock> {
pub(crate) fn compute_rest(
&mut self,
max_from: Height,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
self.all.compute(prices, max_from, exit)?;
+2 -2
View File
@@ -4,7 +4,7 @@ use brk_types::{BasisPointsSigned32, Bitcoin, Cents, Date, Day1, Dollars, Sats};
use vecdb::{AnyVec, Exit, ReadableOptionVec, ReadableVec, VecIndex};
use super::{ByDcaPeriod, Vecs};
use crate::{blocks, indexes, internal::RatioDiffCentsBps32, market, prices};
use crate::{blocks, indexes, internal::RatioDiffCentsBps32, market, price};
const DCA_AMOUNT: Dollars = Dollars::mint(100.0);
@@ -13,7 +13,7 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
blocks: &blocks::Vecs,
lookback: &market::lookback::Vecs,
exit: &Exit,
+20 -20
View File
@@ -22,7 +22,7 @@ mod market;
mod mining;
mod outputs;
mod pools;
pub mod prices;
pub mod price;
mod supply;
mod transactions;
@@ -38,7 +38,7 @@ pub struct Computer<M: StorageMode = Rw> {
pub investing: Box<investing::Vecs<M>>,
pub market: Box<market::Vecs<M>>,
pub pools: Box<pools::Vecs<M>>,
pub prices: Box<prices::Vecs<M>>,
pub price: Box<price::Vecs<M>>,
#[traversable(flatten)]
pub distribution: Box<distribution::Vecs<M>>,
pub supply: Box<supply::Vecs<M>>,
@@ -66,14 +66,14 @@ impl Computer {
)?))
})?;
let (constants, prices) = timed("Imported prices/constants", || -> Result<_> {
let (constants, price) = timed("Imported price/constants", || -> Result<_> {
let constants = Box::new(constants::Vecs::new(VERSION, &indexes));
let prices = Box::new(prices::Vecs::forced_import(
let price = Box::new(price::Vecs::forced_import(
&computed_path,
VERSION,
&indexes,
)?);
Ok((constants, prices))
Ok((constants, price))
})?;
let blocks = timed("Imported blocks", || -> Result<_> {
@@ -223,7 +223,7 @@ impl Computer {
cointime,
indexes,
inputs,
prices,
price,
outputs,
};
@@ -244,7 +244,7 @@ impl Computer {
investing::DB_NAME,
market::DB_NAME,
pools::DB_NAME,
prices::DB_NAME,
price::DB_NAME,
distribution::DB_NAME,
supply::DB_NAME,
inputs::DB_NAME,
@@ -297,8 +297,8 @@ impl Computer {
})
},
|| {
timed("Computed prices", || {
self.prices.compute(indexer, &self.indexes, exit)
timed("Computed price", || {
self.price.compute(indexer, &self.indexes, exit)
})
},
);
@@ -310,7 +310,7 @@ impl Computer {
let market = scope.spawn(|| {
timed("Computed market", || {
self.market
.compute(indexer, &self.prices, &self.indexes, &self.blocks, exit)
.compute(indexer, &self.price, &self.indexes, &self.blocks, exit)
})
});
@@ -321,7 +321,7 @@ impl Computer {
&self.indexes,
&self.blocks,
&self.inputs,
&self.prices,
&self.price,
exit,
)
})?;
@@ -331,7 +331,7 @@ impl Computer {
&self.indexes,
&self.blocks,
&self.transactions,
&self.prices,
&self.price,
exit,
)
})
@@ -343,7 +343,7 @@ impl Computer {
&self.indexes,
&self.inputs,
&self.blocks,
&self.prices,
&self.price,
exit,
)
})?;
@@ -360,7 +360,7 @@ impl Computer {
indexer,
&self.indexes,
&self.blocks,
&self.prices,
&self.price,
&self.mining,
exit,
)
@@ -372,7 +372,7 @@ impl Computer {
self.investing.compute(
indexer,
&self.indexes,
&self.prices,
&self.price,
&self.blocks,
&self.market.lookback,
exit,
@@ -388,7 +388,7 @@ impl Computer {
&self.outputs,
&self.transactions,
&self.blocks,
&self.prices,
&self.price,
exit,
)
})?;
@@ -421,7 +421,7 @@ impl Computer {
&self.blocks,
&self.mining,
&self.transactions,
&self.prices,
&self.price,
&self.distribution,
exit,
)
@@ -430,7 +430,7 @@ impl Computer {
timed("Computed cointime", || {
self.cointime.compute(
indexer,
&self.prices,
&self.price,
&self.blocks,
&self.mining,
&self.supply,
@@ -445,7 +445,7 @@ impl Computer {
self.indicators
.rarity_meter
.compute(indexer, &self.distribution, &self.prices, exit)?;
.compute(indexer, &self.distribution, &self.price, exit)?;
info!("Total compute time: {:?}", compute_start.elapsed());
Ok(())
@@ -498,7 +498,7 @@ impl_iter_named!(
investing,
market,
pools,
prices,
price,
distribution,
supply,
inputs,
@@ -4,13 +4,13 @@ use brk_types::{StoredF32, Timestamp};
use vecdb::{Exit, ReadableVec, VecIndex};
use super::Vecs;
use crate::{indexes, prices};
use crate::{indexes, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
indexes: &indexes::Vecs,
exit: &Exit,
) -> Result<()> {
+2 -2
View File
@@ -2,7 +2,7 @@ use brk_error::Result;
use brk_indexer::Indexer;
use vecdb::Exit;
use crate::{blocks, indexes, prices};
use crate::{blocks, indexes, price};
use super::Vecs;
@@ -10,7 +10,7 @@ impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
exit: &Exit,
@@ -3,14 +3,14 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, prices};
use crate::{blocks, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
@@ -3,14 +3,14 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, prices};
use crate::{blocks, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_lengths = indexer.safe_lengths();
@@ -4,13 +4,13 @@ use brk_types::{BasisPoints16, StoredF32};
use vecdb::{Exit, ReadableVec, VecIndex};
use super::Vecs;
use crate::{blocks, prices};
use crate::{blocks, price};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
blocks: &blocks::Vecs,
exit: &Exit,
) -> Result<()> {
@@ -5,14 +5,14 @@ use vecdb::Exit;
use super::Vecs;
use crate::{
blocks, internal::RatioDiffDollarsBps32, investing::ByDcaPeriod, market::lookback, prices,
blocks, internal::RatioDiffDollarsBps32, investing::ByDcaPeriod, market::lookback, price,
};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
blocks: &blocks::Vecs,
lookback: &lookback::Vecs,
exit: &Exit,
@@ -10,7 +10,7 @@ use super::{
use crate::{
blocks,
internal::{RatioDollarsBp32, WindowsTo1m},
prices,
price,
};
impl Vecs {
@@ -19,7 +19,7 @@ impl Vecs {
&mut self,
indexer: &Indexer,
returns: &returns::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
blocks: &blocks::Vecs,
moving_average: &moving_average::Vecs,
exit: &Exit,
@@ -3,14 +3,14 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::MacdChain;
use crate::{blocks, prices};
use crate::{blocks, price};
#[allow(clippy::too_many_arguments)]
pub(super) fn compute(
chain: &mut MacdChain,
indexer: &Indexer,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
fast_days: usize,
slow_days: usize,
signal_days: usize,
+2 -2
View File
@@ -3,7 +3,7 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes, prices, transactions};
use crate::{blocks, indexes, price, transactions};
impl Vecs {
#[allow(clippy::too_many_arguments)]
@@ -13,7 +13,7 @@ impl Vecs {
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
transactions: &transactions::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
self.db.sync_bg_tasks()?;
@@ -7,7 +7,7 @@ use super::Vecs;
use crate::{
blocks, indexes,
internal::{RatioDollarsBp32, RatioSatsBp16},
prices, transactions,
price, transactions,
};
impl Vecs {
@@ -18,7 +18,7 @@ impl Vecs {
indexes: &indexes::Vecs,
lookback: &blocks::LookbackVecs,
transactions: &transactions::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
+2 -2
View File
@@ -3,7 +3,7 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes, inputs, prices};
use crate::{blocks, indexes, inputs, price};
impl Vecs {
#[allow(clippy::too_many_arguments)]
@@ -13,7 +13,7 @@ impl Vecs {
indexes: &indexes::Vecs,
inputs: &inputs::Vecs,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
self.db.sync_bg_tasks()?;
@@ -4,13 +4,13 @@ use brk_types::{Height, OutputType, Sats, TxOutIndex};
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use super::Vecs;
use crate::prices;
use crate::price;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_lengths = indexer.safe_lengths();
+2 -2
View File
@@ -11,7 +11,7 @@ use crate::{
MaskSats, PercentRollingWindows, RatioU64Bp16, ValuePerBlockCumulativeRolling,
WindowStartVec, Windows,
},
mining, prices,
mining, price,
};
use super::minor;
@@ -63,7 +63,7 @@ impl Vecs {
indexer: &Indexer,
pool: &impl ReadableVec<Height, PoolSlug>,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
mining: &mining::Vecs,
exit: &Exit,
) -> Result<()> {
+2 -2
View File
@@ -22,7 +22,7 @@ use crate::{
WindowStartVec, Windows,
db_utils::{finalize_db, open_db},
},
mining, prices,
mining, price,
};
pub const DB_NAME: &str = "pools";
@@ -90,7 +90,7 @@ impl Vecs {
indexer: &Indexer,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
mining: &mining::Vecs,
exit: &Exit,
) -> Result<()> {
@@ -2,7 +2,9 @@ use std::ops::Range;
use brk_error::Result;
use brk_indexer::{Indexer, Lengths};
use brk_oracle::{Config, Histogram, Oracle, START_HEIGHT, bin_to_cents, cents_to_bin};
use brk_oracle::{
bin_to_cents, cents_to_bin, Config, Oracle, PaymentFilter, START_HEIGHT_FAST, START_HEIGHT_SLOW,
};
use brk_types::{Cents, OutputType, Sats, TxIndex, TxOutIndex};
use tracing::info;
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, StorageMode, VecIndex, WritableVec};
@@ -61,8 +63,8 @@ impl Vecs {
fn compute_prices(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let source_version = indexer.vecs.outputs.value.version()
+ indexer.vecs.outputs.output_type.version();
let source_version =
indexer.vecs.outputs.value.version() + indexer.vecs.outputs.output_type.version();
self.spot
.cents
.height
@@ -71,7 +73,7 @@ impl Vecs {
let total_heights = indexer.vecs.blocks.timestamp.len();
if total_heights <= START_HEIGHT {
if total_heights <= START_HEIGHT_SLOW {
return Ok(());
}
@@ -83,17 +85,12 @@ impl Vecs {
.inner
.truncate_if_needed_at(truncate_to)?;
if self.spot.cents.height.len() < START_HEIGHT {
for line in brk_oracle::PRICES
.lines()
.skip(self.spot.cents.height.len())
{
if self.spot.cents.height.len() >= START_HEIGHT {
if self.spot.cents.height.len() < START_HEIGHT_SLOW {
for cents in brk_oracle::pre_oracle_prices_from(self.spot.cents.height.len()) {
if self.spot.cents.height.len() >= START_HEIGHT_SLOW {
break;
}
let dollars: f64 = line.parse().unwrap_or(0.0);
let cents = (dollars * 100.0).round() as u64;
self.spot.cents.height.inner.push(Cents::new(cents));
self.spot.cents.height.inner.push(cents);
}
}
@@ -101,8 +98,8 @@ impl Vecs {
return Ok(());
}
let config = Config::default();
let committed = self.spot.cents.height.len();
let config = Config::for_height(committed);
let prev_cents = self
.spot
.cents
@@ -110,9 +107,9 @@ impl Vecs {
.collect_one_at(committed - 1)
.unwrap();
let seed_bin = cents_to_bin(prev_cents.inner() as f64);
let warmup = config.window_size.min(committed - START_HEIGHT);
let warmup = config.window_size.min(committed - START_HEIGHT_SLOW);
let mut oracle = Oracle::from_checkpoint(seed_bin, config, |o| {
Self::feed_blocks(o, indexer, (committed - warmup)..committed, None);
Self::feed_blocks_for_warmup(o, indexer, (committed - warmup)..committed, None);
});
let num_new = total_heights - committed;
@@ -121,19 +118,48 @@ impl Vecs {
committed, total_heights
);
let ref_bins =
Self::feed_blocks(&mut oracle, indexer, committed..total_heights, None);
// Slow cold-start EMA up to START_HEIGHT_FAST, then switch to the fast
// mature-market EMA. Steady-state runs start past START_HEIGHT_FAST and skip
// the slow segment entirely.
{
let mut processed = 0usize;
let mut push_ref_bin = |ref_bin| {
self.spot
.cents
.height
.inner
.push(Cents::new(bin_to_cents(ref_bin)));
for (i, ref_bin) in ref_bins.into_iter().enumerate() {
self.spot
.cents
.height
.inner
.push(Cents::new(bin_to_cents(ref_bin)));
processed += 1;
let progress = (processed * 100 / num_new) as u8;
if processed > 1 && progress > (((processed - 1) * 100 / num_new) as u8) {
info!("Oracle price computation: {}%", progress);
}
};
let progress = ((i + 1) * 100 / num_new) as u8;
if i > 0 && progress > ((i * 100 / num_new) as u8) {
info!("Oracle price computation: {}%", progress);
if committed < START_HEIGHT_FAST {
let slow_end = START_HEIGHT_FAST.min(total_heights);
Self::feed_blocks_with(
&mut oracle,
indexer,
committed..slow_end,
None,
|_, _, ref_bin| push_ref_bin(ref_bin),
);
if slow_end == START_HEIGHT_FAST {
oracle.reconfigure(Config::default());
}
}
let fast_start = committed.max(START_HEIGHT_FAST);
if fast_start < total_heights {
Self::feed_blocks_with(
&mut oracle,
indexer,
fast_start..total_heights,
None,
|_, _, ref_bin| push_ref_bin(ref_bin),
);
}
}
@@ -153,10 +179,6 @@ impl Vecs {
/// Feed a range of blocks from the indexer into an Oracle (skipping coinbase),
/// returning per-block ref_bin values.
///
/// A transaction carrying an `OP_RETURN` output is protocol machinery, not a
/// dollar-denominated payment, so all of its outputs are dropped from the
/// histogram. This needs per-transaction grouping of a block's outputs.
///
/// Pass `cap = None` from compute paths, when the indexer is quiescent and
/// raw vec lengths are authoritative. Pass `cap = Some(&safe_lengths)` from
/// reader paths so concurrent writer pushes past the cap are invisible.
@@ -166,6 +188,33 @@ impl Vecs {
range: Range<usize>,
cap: Option<&Lengths>,
) -> Vec<f64> {
let mut ref_bins = Vec::with_capacity(range.len());
Self::feed_blocks_with(oracle, indexer, range, cap, |_, _, ref_bin| {
ref_bins.push(ref_bin);
});
ref_bins
}
/// Feed blocks into an Oracle when callers only need the warmed EMA/window state.
pub fn feed_blocks_for_warmup<IM: StorageMode>(
oracle: &mut Oracle,
indexer: &Indexer<IM>,
range: Range<usize>,
cap: Option<&Lengths>,
) {
Self::feed_blocks_with(oracle, indexer, range, cap, |_, _, _| {});
}
/// Feed a range of blocks into an Oracle and call `on_block` after each
/// processed block. This lets callers observe derived state such as EMA
/// without duplicating the histogram extraction path.
pub fn feed_blocks_with<IM: StorageMode>(
oracle: &mut Oracle,
indexer: &Indexer<IM>,
range: Range<usize>,
cap: Option<&Lengths>,
mut on_block: impl FnMut(usize, &Oracle, f64),
) {
let (total_txs, total_outputs, height_len) = match cap {
Some(c) => (
c.tx_index.to_usize(),
@@ -193,8 +242,6 @@ impl Vecs {
.first_txout_index
.collect_range_at(range.start, collect_end);
let mut ref_bins = Vec::with_capacity(range.len());
// Cursor avoids per-block PcoVec page decompression for the
// tx-indexed first_txout_index lookup. Accessed tx_index values
// are strictly increasing across blocks, so it only advances forward.
@@ -239,26 +286,21 @@ impl Vecs {
&mut output_types,
);
let mut hist = Histogram::zeros();
for tx in 0..tx_count {
let tx_outputs = (0..tx_count).map(|tx| {
let lo = tx_starts[tx] - out_start;
let hi = tx_starts
.get(tx + 1)
.map(|s| s - out_start)
.unwrap_or(out_end - out_start);
if output_types[lo..hi].contains(&OutputType::OpReturn) {
continue;
}
for i in lo..hi {
if let Some(bin) = oracle.output_to_bin(values[i], output_types[i]) {
hist.increment(bin);
}
}
}
values[lo..hi]
.iter()
.copied()
.zip(output_types[lo..hi].iter().copied())
});
let hist = PaymentFilter::for_height(range.start + idx).histogram(tx_outputs);
ref_bins.push(oracle.process_histogram(&hist));
let ref_bin = oracle.process_histogram(&hist);
on_block(range.start + idx, oracle, ref_bin);
}
ref_bins
}
}
@@ -21,7 +21,7 @@ use crate::{
use by_unit::{OhlcByUnit, PriceByUnit, SplitByUnit, SplitCloseByUnit, SplitIndexesByUnit};
use ohlcs::{LazyOhlcVecs, OhlcVecs};
pub const DB_NAME: &str = "prices";
pub const DB_NAME: &str = "price";
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
@@ -4,7 +4,7 @@ use brk_types::Sats;
use vecdb::{Exit, VecIndex};
use super::Vecs;
use crate::{mining, outputs, prices};
use crate::{mining, outputs, price};
impl Vecs {
pub(crate) fn compute(
@@ -12,7 +12,7 @@ impl Vecs {
indexer: &Indexer,
outputs: &outputs::Vecs,
mining: &mining::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
+2 -2
View File
@@ -7,7 +7,7 @@ use vecdb::Exit;
const INITIAL_SUBSIDY: f64 = Sats::ONE_BTC_U64 as f64 * 50.0;
use super::Vecs;
use crate::{blocks, distribution, mining, outputs, prices, transactions};
use crate::{blocks, distribution, mining, outputs, price, transactions};
impl Vecs {
#[allow(clippy::too_many_arguments)]
@@ -18,7 +18,7 @@ impl Vecs {
blocks: &blocks::Vecs,
mining: &mining::Vecs,
transactions: &transactions::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
distribution: &distribution::Vecs,
exit: &Exit,
) -> Result<()> {
@@ -3,7 +3,7 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes, inputs, prices};
use crate::{blocks, indexes, inputs, price};
impl Vecs {
#[allow(clippy::too_many_arguments)]
@@ -13,7 +13,7 @@ impl Vecs {
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
inputs: &inputs::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
exit: &Exit,
) -> Result<()> {
self.db.sync_bg_tasks()?;
@@ -5,7 +5,7 @@ use vecdb::Exit;
use super::Vecs;
use crate::transactions::{count, fees};
use crate::{indexes, internal::Windows, prices};
use crate::{indexes, internal::Windows, price};
impl Vecs {
#[allow(clippy::too_many_arguments)]
@@ -13,7 +13,7 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
prices: &prices::Vecs,
prices: &price::Vecs,
count_vecs: &count::Vecs,
fees_vecs: &fees::Vecs,
exit: &Exit,
+16 -8
View File
@@ -1,6 +1,6 @@
//! Mempool info + price-blending output histogram.
use brk_oracle::Histogram;
use brk_oracle::HistogramRaw;
use brk_types::MempoolInfo;
use crate::Mempool;
@@ -11,13 +11,21 @@ impl Mempool {
self.read().info.clone()
}
/// Snapshot of pre-bucketed oracle bins across all live mempool tx
/// outputs. The total is maintained incrementally by `TxStore` on
/// every insert/remove, so this hot path is `O(NUM_BINS)` regardless
/// of pool size. Used by `live_price` to blend the mempool into the
/// committed oracle without re-parsing scripts per request.
/// Snapshot of pre-bucketed round-dollar-eligible bins across all live
/// mempool tx outputs. Maintained incrementally by `TxStore` on every
/// insert/remove, so this hot path is `O(NUM_BINS)` regardless of pool
/// size. Used by `live_price` to blend the mempool into the committed
/// oracle without re-parsing scripts per request.
#[must_use]
pub fn live_histogram(&self) -> Histogram {
self.read().txs.live_histogram()
pub fn live_eligible_histogram(&self) -> HistogramRaw {
self.read().txs.live_eligible_histogram()
}
/// Snapshot of the raw histogram: every live mempool output binned by
/// value with no payment filtering. Backs the `histogram/raw/live`
/// endpoint.
#[must_use]
pub fn live_raw_histogram(&self) -> HistogramRaw {
self.read().txs.live_raw_histogram()
}
}
@@ -0,0 +1,56 @@
use brk_oracle::{sats_to_bin, HistogramRaw, PaymentFilter};
use brk_types::Transaction;
use crate::stores::tx_store::TxRecord;
/// The two live per-bin histograms the pool maintains incrementally as txs
/// enter and leave: `eligible` applies the round-dollar payment filter (it
/// feeds the oracle blend), `raw` bins every output by value with no filtering.
/// Add and remove run through the same code so the two stay symmetric.
#[derive(Default)]
pub struct LiveHistograms {
eligible: HistogramRaw,
raw: HistogramRaw,
}
impl LiveHistograms {
/// Fold a record's outputs into both histograms.
pub fn add(&mut self, record: &TxRecord) {
Self::eligible_bins(&record.tx, |bin| self.eligible[bin as usize] += 1);
for bin in Self::raw_bins(&record.tx) {
self.raw[bin] += 1;
}
}
/// Reverse a previous `add` for the same record.
pub fn remove(&mut self, record: &TxRecord) {
Self::eligible_bins(&record.tx, |bin| self.eligible[bin as usize] -= 1);
for bin in Self::raw_bins(&record.tx) {
self.raw[bin] -= 1;
}
}
/// Round-dollar-eligible bins, blended into the oracle by `live_price`.
pub fn eligible(&self) -> HistogramRaw {
self.eligible.clone()
}
/// Every live output binned by value, no payment filtering.
pub fn raw(&self) -> HistogramRaw {
self.raw.clone()
}
/// Round-dollar-eligible bins, applying the oracle payment filter. Calls
/// `emit(bin)` per eligible output. Deterministic over a tx's outputs,
/// which are never mutated after insert, so add and remove recompute it
/// identically rather than caching.
fn eligible_bins(tx: &Transaction, emit: impl FnMut(u16)) {
PaymentFilter::MODERN.for_each_bin(tx.output.iter().map(|o| (o.value, o.type_())), emit);
}
/// Raw bin index per output, dropping only values outside the bin domain
/// (zero / out-of-range).
fn raw_bins(tx: &Transaction) -> impl Iterator<Item = usize> + '_ {
tx.output.iter().filter_map(|o| sats_to_bin(o.value))
}
}
+2 -2
View File
@@ -4,13 +4,13 @@
//! one lock-order discipline.
mod addr_tracker;
mod live_histograms;
mod outpoint_spends;
mod output_bins;
mod tx_graveyard;
mod tx_store;
pub use addr_tracker::AddrTracker;
pub use live_histograms::LiveHistograms;
pub use outpoint_spends::OutpointSpends;
pub use output_bins::OutputBins;
pub use tx_graveyard::{TxGraveyard, TxTombstone};
pub use tx_store::TxStore;
@@ -1,23 +0,0 @@
use brk_oracle::default_eligible_bin;
use brk_types::Transaction;
use smallvec::SmallVec;
/// Pre-bucketed oracle bins for a tx's eligible outputs. Computed once on
/// insert so `Mempool::live_histogram` can bin all live outputs without
/// re-parsing scripts or recomputing eligibility per request.
pub struct OutputBins(SmallVec<[u16; 4]>);
impl OutputBins {
pub fn from_tx(tx: &Transaction) -> Self {
Self(
tx.output
.iter()
.filter_map(|o| default_eligible_bin(o.value, o.type_()))
.collect(),
)
}
pub fn iter(&self) -> impl Iterator<Item = u16> + '_ {
self.0.iter().copied()
}
}
+63 -34
View File
@@ -1,28 +1,21 @@
use brk_oracle::Histogram;
use brk_oracle::HistogramRaw;
use brk_types::{MempoolRecentTx, Transaction, TxOut, Txid, TxidPrefix, Vin};
use rustc_hash::{FxHashMap, FxHashSet};
use crate::{state::TxEntry, stores::OutputBins};
use crate::{state::TxEntry, stores::LiveHistograms};
const RECENT_CAP: usize = 10;
/// Per-tx record: live tx body, its mempool entry, and the pre-bucketed
/// oracle bins for its outputs. Kept under one key so a single map probe
/// returns everything readers need.
/// Per-tx record: live tx body and its mempool entry, kept under one key
/// so a single map probe returns everything readers need.
pub struct TxRecord {
pub tx: Transaction,
pub entry: TxEntry,
pub output_bins: OutputBins,
}
impl TxRecord {
pub fn new(tx: Transaction, entry: TxEntry) -> Self {
let output_bins = OutputBins::from_tx(&tx);
Self {
tx,
entry,
output_bins,
}
Self { tx, entry }
}
}
@@ -32,15 +25,15 @@ impl TxRecord {
/// set of prefixes whose tx still has at least one `prevout: None`,
/// maintained on every `insert` / `remove_by_prefix` / `apply_fills`
/// so the post-update prevout filler can early-exit when empty.
/// `live_histogram` mirrors the union of each record's `OutputBins`,
/// kept in sync on `insert` / `remove_by_prefix` so the oracle-blend
/// read path is a single array clone, not a full pool walk.
/// `histograms` holds the eligible (oracle-blend) and raw per-bin output
/// histograms, kept in sync on `insert` / `remove_by_prefix` so each read
/// path is a single array clone, not a full pool walk.
#[derive(Default)]
pub struct TxStore {
records: FxHashMap<TxidPrefix, TxRecord>,
recent: Vec<MempoolRecentTx>,
unresolved: FxHashSet<TxidPrefix>,
live_histogram: Histogram,
histograms: LiveHistograms,
}
impl TxStore {
@@ -92,9 +85,7 @@ impl TxStore {
self.unresolved.insert(prefix);
}
let record = TxRecord::new(tx, entry);
for bin in record.output_bins.iter() {
self.live_histogram[bin as usize] += 1;
}
self.histograms.add(&record);
self.records.insert(prefix, record);
}
@@ -112,16 +103,21 @@ impl TxStore {
pub fn remove_by_prefix(&mut self, prefix: &TxidPrefix) -> Option<TxRecord> {
let record = self.records.remove(prefix)?;
self.unresolved.remove(prefix);
for bin in record.output_bins.iter() {
self.live_histogram[bin as usize] -= 1;
}
self.histograms.remove(&record);
Some(record)
}
/// Snapshot the live oracle-bin histogram. Maintained incrementally
/// on insert/remove, so this is `O(NUM_BINS)`, not `O(live_outputs)`.
pub fn live_histogram(&self) -> Histogram {
self.live_histogram.clone()
/// Snapshot the round-dollar-eligible histogram that feeds the oracle
/// blend. Maintained incrementally, so this is `O(NUM_BINS)`, not
/// `O(live_outputs)`.
pub fn live_eligible_histogram(&self) -> HistogramRaw {
self.histograms.eligible()
}
/// Snapshot the raw histogram: every live output binned by value with no
/// payment filtering. Maintained incrementally alongside the eligible one.
pub fn live_raw_histogram(&self) -> HistogramRaw {
self.histograms.raw()
}
/// Set of prefixes with at least one unfilled prevout. Used by the
@@ -263,7 +259,10 @@ mod tests {
assert_eq!(applied[0].value, new_prevout.value);
let record = store.record_by_prefix(&prefix).expect("record present");
assert_eq!(record.tx.input[0].prevout.as_ref().unwrap().value, new_prevout.value);
assert_eq!(
record.tx.input[0].prevout.as_ref().unwrap().value,
new_prevout.value
);
assert_eq!(
record.tx.input[1].prevout.as_ref().unwrap().value,
prev_present.value
@@ -277,7 +276,10 @@ mod tests {
let stray_prefix = TxidPrefix::from(&fake_txid(0xFF));
let applied = store.apply_fills(
&stray_prefix,
vec![(Vin::from(0u32), TxOut::from((ScriptBuf::new(), Sats::from(1u64))))],
vec![(
Vin::from(0u32),
TxOut::from((ScriptBuf::new(), Sats::from(1u64))),
)],
);
assert!(applied.is_empty());
}
@@ -319,10 +321,7 @@ mod tests {
let tx_a = fake_tx(
20,
&[Some(TxOut::from((p2wpkh_script(8), Sats::from(1_234u64))))],
&[
(p2wpkh_script(9), 2_345),
(p2wpkh_script(10), 3_456),
],
&[(p2wpkh_script(9), 2_345), (p2wpkh_script(10), 3_456)],
);
let tx_b = fake_tx(
21,
@@ -335,11 +334,41 @@ mod tests {
store.insert(tx_a, entry_a);
store.insert(tx_b, entry_b);
let total_after_both: u32 = store.live_histogram().iter().sum();
let total_after_both: u32 = store.live_eligible_histogram().iter().sum();
assert_eq!(total_after_both, 3, "two outputs + one output");
store.remove_by_prefix(&prefix_a);
let total_after_remove: u32 = store.live_histogram().iter().sum();
let total_after_remove: u32 = store.live_eligible_histogram().iter().sum();
assert_eq!(total_after_remove, 1);
}
#[test]
fn raw_histogram_bins_outputs_the_eligible_filter_drops() {
let mut store = TxStore::default();
// 2_345 sats is a round-dollar-eligible payment; 100_000_000 sats (1 BTC)
// is a round-BTC value the eligible filter drops but raw still bins.
let tx = fake_tx(
30,
&[Some(TxOut::from((p2wpkh_script(1), Sats::from(50_000u64))))],
&[(p2wpkh_script(2), 2_345), (p2wpkh_script(3), 100_000_000)],
);
let entry = entry_for(&tx, 100, 100);
let prefix = entry.txid_prefix();
store.insert(tx, entry);
assert_eq!(
store.live_eligible_histogram().iter().sum::<u32>(),
1,
"round-BTC output filtered out of the eligible histogram"
);
assert_eq!(
store.live_raw_histogram().iter().sum::<u32>(),
2,
"raw histogram bins every output"
);
store.remove_by_prefix(&prefix);
assert_eq!(store.live_eligible_histogram().iter().sum::<u32>(), 0);
assert_eq!(store.live_raw_histogram().iter().sum::<u32>(), 0);
}
}
+61 -37
View File
@@ -1,8 +1,8 @@
# brk_oracle
**Version 2**
**Version 3**
Pure on-chain BTC/USD price oracle. No exchange feeds, no external APIs. Derives the bitcoin price from transaction data alone. Tracks block by block from height 525,000 (May 2018) onward.
Pure on-chain BTC/USD price oracle. No exchange feeds, no external APIs. Derives the bitcoin price from transaction data alone. Tracks block by block from height 340,000 (January 2015) onward.
Inspired by [UTXOracle](https://utxo.live/oracle/) by [@SteveSimple](https://x.com/SteveSimple), which proved the concept. brk_oracle takes the same core insight and redesigns the algorithm for per-block resolution and rolling operation. See [comparison](#comparison-with-utxoracle) below.
@@ -42,13 +42,13 @@ The spacing between spikes is constant (set by the ratios between dollar amounts
## How it works
The oracle tracks the price incrementally, block by block, starting from a known seed price. Each new block nudges the estimate. The search window is narrow (about ±10 bins, or ±12%), so the oracle can only follow gradual movement — it cannot jump to an arbitrary price from scratch. This is by design: it makes the algorithm resistant to noise.
The oracle tracks the price incrementally, block by block, starting from a known seed price. Each new block nudges the estimate. The search window is narrow (about 12 bins, or +15% / -12% in price), so the oracle can only follow gradual movement, not jump to an arbitrary price from scratch. This is by design: it makes the algorithm resistant to noise.
For each new block:
### 1. Filter outputs
Skip the coinbase transaction, and skip every output of a transaction carrying an `OP_RETURN`: that transaction is protocol machinery, not a dollar-denominated payment, so its payout amounts are not price signal. Then exclude noisy outputs: script types dominated by protocol activity (P2TR by default), dust below 1,000 sats, and round BTC amounts (0.01, 0.1, 1.0 BTC, etc.) that create false spikes unrelated to dollar purchases.
Skip the coinbase transaction, and skip every output of a transaction carrying an `OP_RETURN`: that transaction is protocol machinery, not a dollar-denominated payment, so its payout amounts are not price signal. Below height 630,000, also skip every output of a transaction with more than 100 outputs: a large fan-out is a batch payout (exchange sweep, mixer), not a round-dollar payment, and the thin early signal needs it removed. At and above height 630,000, the transaction fan-out cap relaxes to 250 outputs so dense-chain payment activity remains visible while very large fan-outs cannot dominate one EMA slot. Then exclude noisy outputs: script types dominated by protocol activity (P2TR by default), dust below 1,000 sats, and round BTC amounts (0.01, 0.1, 1.0 BTC, etc.) that create false spikes unrelated to dollar purchases.
### 2. Build a log-scale histogram
@@ -87,7 +87,7 @@ The fixed ratios between round-dollar amounts ($1, $2, $3, $5, ... $10,000) crea
The oracle slides this stencil across the EMA histogram within the search window. At each candidate position:
1. **Read** the EMA value at all 19 expected spike locations
2. **Normalize** each value by dividing by that offset's peak within the search window this gives rare amounts like $3 equal voting weight to common amounts like $100
2. **Normalize** each value by dividing by that offset's peak within the search window: this gives rare amounts like $3 equal voting weight to common amounts like $100
3. **Sum** the 19 normalized values into a single score
The position with the highest score is where the fingerprint best matches the histogram.
@@ -102,7 +102,7 @@ A $100 purchase at price P produces `$100 / P × 10⁸` sats, which lands in bin
= (10 log₁₀(P)) × 200
```
So the stencil's winning position the bin where $100 purchases land directly encodes the price:
So the stencil's winning position, the bin where $100 purchases land, directly encodes the price:
```
price = 10^(10 bin / 200) dollars
@@ -122,9 +122,9 @@ Parabolic interpolation between the best bin and its two neighbors refines the e
The oracle consumes one pre-built histogram per block via `process_histogram(&hist)`, a `[u32; 2400]` bin-count array, and returns the updated reference bin.
The caller does the filtering when it builds the histogram. For each block it skips the coinbase, drops every output of a transaction carrying an `OP_RETURN`, then bins the rest. `default_eligible_bin(sats, output_type)` (or `Oracle::output_to_bin` for a non-default `Config`) applies the per-output rules: excluded script types, dust, and round-BTC values. It returns the bin index, or `None` for a filtered output.
The caller filters as it builds the histogram, applying the [step 1](#1-filter-outputs) rules. `PaymentFilter::for_height(height).histogram(txs)` builds a fresh block histogram from non-coinbase transaction outputs. Incremental live callers use `PaymentFilter::MODERN.for_each_bin(outputs, emit)`, which applies the modern fan-out cap without requiring a height. `PaymentFilter::eligible_bin(sats, output_type)` returns an individual output's bin index, or `None` if filtered. The transaction-level rules include the OP_RETURN drop, the >100 transaction-output fan-out cap below height 630,000, and the >250 cap from height 630,000 onward.
The initial seed must be close to the real price at the starting height. The crate includes a `PRICES` constant with exchange prices for every height up to 630,000 to derive a seed from.
The initial seed must be close to the real price at the starting height. The crate includes typed pre-oracle helpers for exchange prices at heights 0..340,000. `Oracle::from_seed()` uses the last baked price, height 339,999 (one below `START_HEIGHT_SLOW`), and the slow cold-start config to seed the oracle's first on-chain computation at height 340,000.
## Configuration
@@ -134,10 +134,12 @@ All parameters via `Config` with sensible defaults:
|-----------|---------|---------|
| `alpha` | 2/7 | EMA decay rate (~6-block span) |
| `window_size` | 12 | Ring buffer depth in blocks |
| `search_below` / `search_above` | 9 / 11 | Search window around previous estimate (bins) |
| `min_sats` | 1,000 | Dust threshold |
| `exclude_common_round_values` | true | Filter d × 10ⁿ (d ∈ {1,2,3,5,6}) to prevent false stencil matches |
| `excluded_output_types` | P2TR | Script types dominated by protocol activity |
| `search_below` / `search_above` | 12 / 11 | Search window around previous estimate (bins) |
| `shape_weight` | 0 | Shape-anchoring restoring-force weight. 0 disables it. `Config::slow()` sets 8 for the cold-start |
The output-filtering rules (1,000-sat dust floor, excluded P2TR, round-BTC exclusion) are not `Config` parameters: they are constants in the `filter` module so the indexer, per-request reconstruction, and mempool all bin identically. See [Input](#input).
Between heights 340,000 and 508,000 the oracle runs a slower cold-start configuration (`Config::slow()`: `alpha` = 0.10, ~19-block span, `window_size` = 40, `shape_weight` = 8). In the thin pre-2018 output mix the fast default octave-locks onto the round-dollar half-price pattern, so the slow EMA and the shape-anchoring restoring force resist that drift. At 508,000 `Oracle::reconfigure` switches to the defaults above (`shape_weight` back to 0), and `Config::for_height` returns the right one for any height.
## Comparison with UTXOracle
@@ -145,35 +147,36 @@ All parameters via `Config` with sensible defaults:
| | brk_oracle | UTXOracle |
|---|---|---|
| Resolution | Per-block (~10 min) + daily candles | Per-run consensus price + per-output intraday scatter |
| Resolution | Per-block (~10 min); daily OHLC built downstream | Per-run consensus price + per-output intraday scatter |
| Operation | Rolling: EMA over ring buffer, updates each block | Batch: processes a full day from scratch, stateless |
| Algorithm | Single-pass stencil scoring with per-offset normalization | Multi-step: dual stencil → rough estimate → output-to-USD mapping → iterative convergence |
| Steps to compute price | 7 (filter+bin → ring insert → EMA → per-offset peaks → score → argmax+parabolic → bin→price) | 10 (filter+bin → clip → smooth round BTC → sum → normalize → cap extremes → dual-stencil slide → neighbor weight-avg → output-to-USD map → iterative central price) |
| Stencil | 19 round-USD offsets ($1 to $10k), each normalized to its own peak | 803-point Gaussian + weighted spike template targeting 17 round-USD amounts |
| Round BTC handling | Excluded from histogram entirely | Histogram bins smoothed by averaging neighbors |
| Output filtering | Per-tx OP_RETURN drop, then per-output: script type, dust threshold, round BTC | Per-tx: exactly 2 outputs, ≤5 inputs, no same-day inputs, ≤500-byte witness |
| Validated from | Height 525,000 (May 2018) | December 2023 |
| Output filtering | Per-tx OP_RETURN drop, then per-output: script type, dust threshold, round BTC | Per-tx: not coinbase, no OP_RETURN, exactly 2 outputs, ≤5 inputs, no same-day inputs, ≤500-byte witness |
| Validated from | Height 340,000 (January 2015) | Dec 15, 2023 |
| Language | Rust | Python |
| Dependencies | None (pure computation, caller provides block data) | Bitcoin Core RPC |
| Dependencies | None (pure computation, caller provides block data) | bitcoin-cli + direct blk file reads |
| Bins per decade | 200 | 200 |
## Accuracy
Tested over 411,251 blocks (heights 525,000 to 949,800, as of May 2026) against exchange OHLC data. Error is measured per block as distance from the oracle estimate to the exchange high/low range at that height. If the oracle falls within the range, the error is zero.
Tested over 596,251 exchange-covered blocks after running the oracle from height 340,000 through height 952,314. Error is measured per block as distance from the oracle estimate to the exchange high/low range at that height. If the oracle falls within the range, the error is zero.
### Per-block
| Metric | Value |
|--------|-------|
| Median error | 0.11% |
| 95th percentile | 0.67% |
| 99th percentile | 1.7% |
| 99.9th percentile | 5.4% |
| RMSE | 0.50% |
| Max error | 33.4% |
| Bias | +0.00 bins (essentially zero) |
| Blocks > 5% error | 472 (0.11%) |
| Blocks > 10% error | 177 |
| Blocks > 20% error | 3 |
| Median error | 0.15% |
| 95th percentile | 1.2% |
| 99th percentile | 3.4% |
| 99.9th percentile | 15.6% |
| RMSE | 0.97% |
| Max error | 33.8% |
| Bias | +0.06 bins (essentially zero) |
| Blocks > 5% error | 3,233 (0.542%) |
| Blocks > 10% error | 1,323 |
| Blocks > 20% error | 154 |
### Daily candles
@@ -181,36 +184,57 @@ Oracle daily OHLC built from per-block prices vs exchange daily OHLC:
| | Median | RMSE | Max |
|-------|--------|------|-----|
| Open | 0.21% | 0.65% | 15.3% |
| High | 0.53% | 1.12% | 28.0% |
| Low | 0.51% | 1.38% | 19.7% |
| Close | 0.24% | 0.73% | 15.4% |
| Open | 0.24% | 1.07% | 29.1% |
| High | 0.58% | 1.47% | 27.3% |
| Low | 0.53% | 1.95% | 55.1% |
| Close | 0.27% | 1.18% | 29.2% |
### By year
| Year | Blocks | Median | RMSE | Max | >5% | >10% | >20% | Price range |
|------|--------|--------|------|-----|-----|------|------|-------------|
| 2018 | 31,492 | 0.21% | 1.11% | 33.4% | 169 | 109 | 3 | $3,129$8,488 |
| 2019 | 54,272 | 0.16% | 0.69% | 17.4% | 165 | 53 | 0 | $3,338$13,868 |
| 2020 | 53,102 | 0.10% | 0.44% | 12.6% | 70 | 6 | 0 | $3,858$29,322 |
| 2015 | 51,249 | 0.26% | 1.67% | 33.8% | 916 | 449 | 25 | $198$500 |
| 2016 | 54,753 | 0.33% | 0.80% | 16.9% | 150 | 33 | 0 | $351$989 |
| 2017 | 55,959 | 0.45% | 2.05% | 28.6% | 1,527 | 606 | 67 | $0$19,892 |
| 2018 | 54,531 | 0.18% | 1.31% | 31.6% | 411 | 207 | 62 | $3,129$17,178 |
| 2019 | 54,272 | 0.16% | 0.59% | 17.4% | 100 | 16 | 0 | $3,338$13,868 |
| 2020 | 53,102 | 0.10% | 0.42% | 11.6% | 61 | 3 | 0 | $3,858$29,322 |
| 2021 | 52,733 | 0.07% | 0.47% | 14.4% | 42 | 9 | 0 | $27,678$69,000 |
| 2022 | 53,230 | 0.07% | 0.32% | 6.8% | 10 | 0 | 0 | $15,460$48,240 |
| 2023 | 54,032 | 0.10% | 0.25% | 6.6% | 5 | 0 | 0 | $16,490$44,700 |
| 2024 | 53,367 | 0.10% | 0.28% | 6.7% | 7 | 0 | 0 | $38,555$108,298 |
| 2025 | 53,113 | 0.11% | 0.25% | 5.8% | 4 | 0 | 0 | $74,409$126,198 |
| 2026 | 5,910 | 0.11% | 0.27% | 3.2% | 0 | 0 | 0 | $60,000$97,900 |
| 2026 | 5,910 | 0.10% | 0.27% | 3.2% | 0 | 0 | 0 | $60,000$97,900 |
The oracle is only as good as the signal it reads. The largest errors cluster in late 2018: the November price crash fell faster than the narrow search window could follow (33% max error), and on-chain volume was lower then, so the round-dollar pattern was weaker (1.1% RMSE for the year). By 2020 the signal is strong enough for 0.1% median accuracy, and since 2022 no block exceeds 10% error.
The oracle is only as good as the signal it reads. The largest errors cluster in the early cold-start, where thin 2015 on-chain volume gives a weaker round-dollar pattern: the 33.8% max error sits at height 341,498 (oracle ~$287 vs exchange ~$213) during the first weeks of warm-up. A second cluster sits just below the 508,000 regime switch, where the slow EMA lagged the fast early-2018 rally (~31.6% at height 507,278, oracle ~$6,685 vs exchange ~$8,800) before handing off to the fast default. The thin pre-2018 mix means 2015, 2017, and 2018 carry the bulk of the error (1.67%, 2.05%, and 1.31% RMSE). From 2019 the signal strengthens: by 2020 the oracle reaches 0.1% median accuracy, and since 2022 no block exceeds 10% error.
### Why no outlier smoothing?
Post-hoc smoothing for example, correcting any block whose price deviates more than 5% from both its neighbors would improve the aggregate numbers. This is deliberately not done, for two reasons:
Post-hoc smoothing, for example correcting any block whose price deviates more than 5% from both its neighbors, would improve the aggregate numbers. This is deliberately not done, for two reasons:
1. **Simplicity**: The oracle is a single forward pass with no lookback corrections. Adding smoothing means defining thresholds, neighbor windows, and replacement strategies, all of which add complexity for marginal gain.
2. **Finality**: Each block's price is produced once and never revised (unless the block itself is reorged). Downstream consumers can treat the oracle output as append-only. Smoothing would require retroactively changing already-published prices, breaking that property.
## Changelog
### v4
Changes from v3:
- **Modern fan-out cap**: below height 630,000 the oracle keeps the strict >100-output transaction drop introduced in v3. At and above 630,000 the cap now relaxes to 250 outputs instead of being fully lifted. This preserves dense-chain payment signal while preventing very large modern fan-outs from dominating a single EMA slot and creating a transient false round-dollar ladder.
`VERSION` is bumped to 4 so downstream consumers invalidate prices computed by an earlier algorithm.
### v3
Changes from v2:
- **Earlier start with a cold-start regime**: on-chain tracking begins at height 340,000 (January 2015) instead of 525,000, adding about 185,000 blocks of history. Below height 508,000 the oracle runs a slower EMA (`Config::slow()`, ~19-block span, window 40) paired with a shape-anchoring restoring force (`shape_weight` 8) that pulls candidate scores toward a slowly-adapted profile of the round-dollar arm shape, resisting the half-price octave drift the fast default locks onto in the thinner pre-2018 output mix. At height 508,000 it switches to the fast default via `Oracle::reconfigure`, which restores `shape_weight` to 0 and turns the force off.
- **Max-outputs filter**: a transaction with more than 100 outputs is dropped from the histogram below height 630,000. Large fan-outs (exchange sweeps, mixer payouts) are batch machinery, not round-dollar payments, and the thin 2018-2020 signal needs them removed to stay locked onto the pattern. Above 630,000 on-chain volume is dense enough that the cap removes more genuine signal than noise, so it is lifted.
- **Wider up-reach**: `search_below` raised from 9 to 12 bins. The sharp 2018 reversal candles need extra room to follow a fast move upward in price.
`VERSION` is bumped to 3 so downstream consumers invalidate prices computed by an earlier algorithm.
### v2
Changes from v1:
@@ -1,295 +0,0 @@
//! Compare specific digit filter configurations across multiple start heights.
//!
//! Run with: cargo run -p brk_oracle --example compare_digits --release
use std::path::PathBuf;
use std::time::Instant;
use brk_indexer::Indexer;
use brk_oracle::{Config, Histogram, NUM_BINS, Oracle, PRICES, cents_to_bin, sats_to_bin};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
const BINS_5PCT: f64 = 4.24;
const BINS_10PCT: f64 = 8.28;
const BINS_20PCT: f64 = 15.84;
fn bins_to_pct(bins: f64) -> f64 {
(10.0_f64.powf(bins / 200.0) - 1.0) * 100.0
}
fn seed_bin(start_height: usize) -> f64 {
let price: f64 = PRICES
.lines()
.nth(start_height - 1)
.expect("prices.txt too short")
.parse()
.expect("Failed to parse seed price");
cents_to_bin(price * 100.0)
}
fn leading_digit(sats: u64) -> u8 {
let log = (sats as f64).log10();
let magnitude = 10.0_f64.powf(log.floor());
let d = (sats as f64 / magnitude).round() as u8;
if d >= 10 { 1 } else { d }
}
fn is_round(sats: u64) -> bool {
let log = (sats as f64).log10();
let magnitude = 10.0_f64.powf(log.floor());
let leading = (sats as f64 / magnitude).round();
let round_val = leading * magnitude;
(sats as f64 - round_val).abs() <= round_val * 0.001
}
struct Stats {
total_sq_err: f64,
total_bias: f64,
max_err: f64,
total_blocks: u64,
gt_5pct: u64,
gt_10pct: u64,
gt_20pct: u64,
}
impl Stats {
fn new() -> Self {
Self {
total_sq_err: 0.0,
total_bias: 0.0,
max_err: 0.0,
total_blocks: 0,
gt_5pct: 0,
gt_10pct: 0,
gt_20pct: 0,
}
}
fn update(&mut self, err: f64) {
self.total_sq_err += err * err;
self.total_bias += err;
self.total_blocks += 1;
let abs_err = err.abs();
if abs_err > self.max_err {
self.max_err = abs_err;
}
if abs_err > BINS_5PCT {
self.gt_5pct += 1;
}
if abs_err > BINS_10PCT {
self.gt_10pct += 1;
}
if abs_err > BINS_20PCT {
self.gt_20pct += 1;
}
}
fn rmse_pct(&self) -> f64 {
bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt())
}
fn max_pct(&self) -> f64 {
bins_to_pct(self.max_err)
}
fn bias(&self) -> f64 {
self.total_bias / self.total_blocks as f64
}
}
fn main() {
let t0 = Instant::now();
let data_dir = std::env::var("BRK_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| {
let home = std::env::var("HOME").unwrap();
PathBuf::from(home).join(".brk")
});
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
let total_heights = indexer.vecs.blocks.timestamp.len();
let manifest_dir = env!("CARGO_MANIFEST_DIR");
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
.expect("Failed to read height_price_ohlc.json"),
)
.expect("Failed to parse height OHLC");
let height_bands: Vec<(f64, f64)> = height_ohlc
.iter()
.map(|ohlc| {
let high = ohlc[1];
let low = ohlc[2];
if high > 0.0 && low > 0.0 {
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
} else {
(0.0, 0.0)
}
})
.collect();
// Configs to compare.
// 987654321
let masks: &[(u16, &str)] = &[
(0b0_0111_0111, "{1,2,3,5,6,7}"),
(0b0_0011_0111, "{1,2,3,5,6}"),
(0b0_0001_1111, "{1,2,3,4,5}"),
(0b0_0001_0111, "{1,2,3,5}"),
];
let start_heights: &[usize] = &[575_000, 600_000, 630_000];
// (mask_idx, start_idx) -> (Oracle, Stats)
let n = masks.len() * start_heights.len();
let mut oracles: Vec<Option<Oracle>> = (0..n).map(|_| None).collect();
let mut stats: Vec<Stats> = (0..n).map(|_| Stats::new()).collect();
let idx = |m: usize, s: usize| -> usize { m * start_heights.len() + s };
let total_txs = indexer.vecs.transactions.txid.len();
let total_outputs = indexer.vecs.outputs.value.len();
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
let ref_config = Config::default();
let earliest_start = *start_heights.iter().min().unwrap();
for h in earliest_start..total_heights {
let ft = first_tx_index[h];
let next_ft = first_tx_index
.get(h + 1)
.copied()
.unwrap_or(TxIndex::from(total_txs));
let out_start = if ft.to_usize() + 1 < next_ft.to_usize() {
indexer
.vecs
.transactions
.first_txout_index
.collect_one(ft + 1)
.unwrap()
.to_usize()
} else {
out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize()
};
let out_end = out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
let values: Vec<Sats> = indexer
.vecs
.outputs
.value
.collect_range_at(out_start, out_end);
let output_types: Vec<OutputType> = indexer
.vecs
.outputs
.output_type
.collect_range_at(out_start, out_end);
// Build full histogram and per-digit histograms.
let mut full_hist = Histogram::zeros();
let mut digit_hist: [Histogram; 9] = std::array::from_fn(|_| Histogram::zeros());
for (sats, output_type) in values.into_iter().zip(output_types) {
if ref_config.excluded_output_types.contains(&output_type) {
continue;
}
if *sats < ref_config.min_sats {
continue;
}
if let Some(bin) = sats_to_bin(sats) {
full_hist.increment(bin);
if is_round(*sats) {
let d = leading_digit(*sats);
if (1..=9).contains(&d) {
digit_hist[(d - 1) as usize].increment(bin);
}
}
}
}
// Feed each (mask, start_height) combo.
for (mi, &(mask, _)) in masks.iter().enumerate() {
// Build filtered histogram for this mask.
let mut hist = full_hist.clone();
(0..9usize).for_each(|d| {
if mask & (1 << d) != 0 {
for bin in 0..NUM_BINS {
hist[bin] -= digit_hist[d][bin];
}
}
});
for (si, &sh) in start_heights.iter().enumerate() {
if h < sh {
continue;
}
let i = idx(mi, si);
if oracles[i].is_none() {
oracles[i] = Some(Oracle::new(
seed_bin(sh),
Config {
exclude_common_round_values: false,
..Default::default()
},
));
}
let ref_bin = oracles[i].as_mut().unwrap().process_histogram(&hist);
if h < height_bands.len() {
let (high_bin, low_bin) = height_bands[h];
if high_bin > 0.0 && low_bin > 0.0 {
let err = if ref_bin < high_bin {
ref_bin - high_bin
} else if ref_bin > low_bin {
ref_bin - low_bin
} else {
0.0
};
stats[i].update(err);
}
}
}
}
}
// Print results grouped by start height.
for (si, &sh) in start_heights.iter().enumerate() {
println!();
println!("@ {}k:", sh / 1000);
println!(
" {:<16} {:>8} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}",
"Digits", "Blocks", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias"
);
println!(" {}", "-".repeat(72));
for (mi, &(_, label)) in masks.iter().enumerate() {
let s = &stats[idx(mi, si)];
println!(
" {:<16} {:>8} {:>7.3}% {:>7.1}% {:>6} {:>6} {:>6} {:>+8.2}",
label,
s.total_blocks,
s.rmse_pct(),
s.max_pct(),
s.gt_5pct,
s.gt_10pct,
s.gt_20pct,
s.bias()
);
}
}
println!("\nDone in {:.1}s", t0.elapsed().as_secs_f64());
}
+131 -139
View File
@@ -1,10 +1,11 @@
//! Verify oracle determinism: oracles started from different heights converge
//! to identical ref_bin values after the ring buffer fills.
//! Verify the production restart property: an oracle restored via
//! `from_checkpoint` (seeded from the previous block's stored cents price,
//! replayed over the last `window_size` blocks) produces bit-exact `ref_bin`
//! values matching a continuously-running oracle from the restart height
//! onward.
//!
//! Creates a reference oracle at height 575k and test oracles every 1000 blocks
//! up to 630k. After window_size blocks, each test oracle should produce the
//! same ref_bin as the reference, proving the truncated EMA provides
//! start-point independence.
//! Mirrors the production transaction filter exactly, so it exercises the same code path
//! `brk_computer::price::compute::feed_blocks` uses at runtime.
//!
//! Run with: cargo run -p brk_oracle --example determinism --release
@@ -12,26 +13,35 @@ use std::path::PathBuf;
use brk_indexer::Indexer;
use brk_oracle::{
Config, Histogram, Oracle, PRICES, START_HEIGHT, cents_to_bin, default_eligible_bin,
bin_to_cents, cents_to_bin, Config, HistogramRaw, Oracle, PaymentFilter, START_HEIGHT_FAST,
START_HEIGHT_SLOW,
};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
fn seed_bin(height: usize) -> f64 {
let price: f64 = PRICES
.lines()
.nth(height - 1)
.expect("prices.txt too short")
.parse()
.expect("Failed to parse seed price");
cents_to_bin(price * 100.0)
struct Block {
height: usize,
values: Vec<Sats>,
output_types: Vec<OutputType>,
tx_starts: Vec<usize>,
out_start: usize,
out_end: usize,
}
struct TestRun {
start_height: usize,
oracle: Option<Oracle>,
converged_at: Option<usize>,
diverged_after: bool,
fn build_histogram(block: &Block) -> HistogramRaw {
let tx_outputs = (0..block.tx_starts.len()).map(|tx| {
let lo = block.tx_starts[tx] - block.out_start;
let hi = block
.tx_starts
.get(tx + 1)
.map(|s| s - block.out_start)
.unwrap_or(block.out_end - block.out_start);
block.values[lo..hi]
.iter()
.copied()
.zip(block.output_types[lo..hi].iter().copied())
});
PaymentFilter::for_height(block.height).histogram(tx_outputs)
}
fn main() {
@@ -45,62 +55,54 @@ fn main() {
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
let total_heights = indexer.vecs.blocks.timestamp.len();
let config = Config::default();
let window_size = config.window_size;
let fast_config = Config::default();
let window_size = fast_config.window_size;
let restart_offset = 1000;
let end_offset = restart_offset + window_size * 4;
let end_height = (START_HEIGHT_FAST + end_offset).min(total_heights);
let restart_at = START_HEIGHT_FAST + restart_offset;
let warmup_start = restart_at - window_size;
let load_start = START_HEIGHT_SLOW;
assert!(
end_height > restart_at,
"indexer has {total_heights} blocks; need at least {} to test restart at {restart_at}",
restart_at + 1
);
println!(
"Loading {} blocks ({load_start}..{end_height})...",
end_height - load_start
);
let total_txs = indexer.vecs.transactions.txid.len();
let total_outputs = indexer.vecs.outputs.value.len();
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor();
// Reference oracle at 575k.
let ref_start = START_HEIGHT;
let mut ref_oracle = Oracle::new(seed_bin(ref_start), Config::default());
// Test oracles every 1000 blocks from 576k to 630k.
let mut runs: Vec<TestRun> = (576_000..=630_000)
.step_by(1000)
.map(|h| TestRun {
start_height: h,
oracle: None,
converged_at: None,
diverged_after: false,
})
.collect();
let last_start = runs.last().map(|r| r.start_height).unwrap_or(ref_start);
// Process enough blocks for all oracles to converge + verification margin.
let end_height = (last_start + window_size + 100).min(total_heights);
for h in START_HEIGHT..end_height {
let mut blocks: Vec<Block> = Vec::with_capacity(end_height - load_start);
for h in load_start..end_height {
let ft = first_tx_index[h];
let next_ft = first_tx_index
.get(h + 1)
.copied()
.unwrap_or(TxIndex::from(total_txs));
let out_start = if ft.to_usize() + 1 < next_ft.to_usize() {
indexer
.vecs
.transactions
.first_txout_index
.collect_one(ft + 1)
.unwrap()
.to_usize()
} else {
out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize()
};
let block_first_tx = ft.to_usize() + 1;
let tx_count = next_ft.to_usize() - block_first_tx;
let out_end = out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
txout_cursor.advance(block_first_tx - txout_cursor.position());
let mut tx_starts: Vec<usize> = Vec::with_capacity(tx_count);
for _ in 0..tx_count {
tx_starts.push(txout_cursor.next().unwrap().to_usize());
}
let out_start = tx_starts.first().copied().unwrap_or(out_end);
let values: Vec<Sats> = indexer
.vecs
.outputs
@@ -112,95 +114,85 @@ fn main() {
.output_type
.collect_range_at(out_start, out_end);
let mut hist = Histogram::zeros();
for (sats, output_type) in values.into_iter().zip(output_types) {
if let Some(bin) = default_eligible_bin(sats, output_type) {
hist.increment(bin as usize);
}
}
let ref_bin = ref_oracle.process_histogram(&hist);
for run in &mut runs {
if h < run.start_height {
continue;
}
if run.oracle.is_none() {
run.oracle = Some(Oracle::new(seed_bin(run.start_height), Config::default()));
}
let test_bin = run.oracle.as_mut().unwrap().process_histogram(&hist);
if run.converged_at.is_some() {
if test_bin != ref_bin {
run.diverged_after = true;
}
} else if test_bin == ref_bin {
run.converged_at = Some(h);
}
}
blocks.push(Block {
height: h,
values,
output_types,
tx_starts,
out_start,
out_end,
});
}
// Print results.
println!();
println!("{:<12} {:>16} {:>8}", "Start", "Converged at", "Blocks");
println!("{}", "-".repeat(40));
let mut max_blocks = 0usize;
let mut failed = Vec::new();
let mut diverged = Vec::new();
for run in &runs {
if let Some(converged) = run.converged_at {
let blocks = converged - run.start_height;
if blocks > max_blocks {
max_blocks = blocks;
let mut continuous = Oracle::from_seed();
let continuous_bins: Vec<f64> = blocks
.iter()
.map(|b| {
if b.height == START_HEIGHT_FAST {
continuous.reconfigure(fast_config);
}
println!("{:<12} {:>16} {:>8}", run.start_height, converged, blocks);
if run.diverged_after {
diverged.push(run.start_height);
}
} else {
println!("{:<12} {:>16} {:>8}", run.start_height, "NEVER", "-");
failed.push(run.start_height);
}
}
println!();
continuous.process_histogram(&build_histogram(b))
})
.collect();
println!(
"{}/{} converged, max {} blocks to converge (window_size={})",
runs.len() - failed.len(),
runs.len(),
max_blocks,
window_size,
"Continuous oracle: {} blocks processed",
continuous_bins.len()
);
if !diverged.is_empty() {
println!("DIVERGED after convergence: {:?}", diverged);
}
if !failed.is_empty() {
println!("NEVER converged: {:?}", failed);
let prev_bin = continuous_bins[restart_at - load_start - 1];
let seed_bin = cents_to_bin(bin_to_cents(prev_bin) as f64);
println!(
"Restart at {restart_at}: prev_bin={prev_bin:.4} -> cents -> seed_bin={seed_bin:.4} (delta {:.6})",
seed_bin - prev_bin
);
let warmup_slice = &blocks[warmup_start - load_start..restart_at - load_start];
let mut restored = Oracle::from_checkpoint(seed_bin, fast_config, |o| {
for b in warmup_slice {
o.process_histogram(&build_histogram(b));
}
});
let restored_bins: Vec<f64> = blocks[restart_at - load_start..]
.iter()
.map(|b| restored.process_histogram(&build_histogram(b)))
.collect();
println!("Restored oracle: {} blocks processed", restored_bins.len());
let mut mismatches: Vec<(usize, f64, f64)> = Vec::new();
for (i, &r) in restored_bins.iter().enumerate() {
let c = continuous_bins[restart_at - load_start + i];
if r != c {
mismatches.push((restart_at + i, c, r));
}
}
// Assertions.
assert!(
failed.is_empty(),
"{} oracles never converged: {:?}",
failed.len(),
failed
);
assert!(
diverged.is_empty(),
"{} oracles diverged after convergence: {:?}",
diverged.len(),
diverged
);
assert!(
max_blocks <= window_size * 2,
"Convergence took {} blocks, expected <= {} (2 * window_size)",
max_blocks,
window_size * 2
println!();
if mismatches.is_empty() {
println!(
"All {} blocks from {restart_at} onward match exactly.",
restored_bins.len()
);
} else {
println!(
"{} of {} blocks differ (showing up to 5):",
mismatches.len(),
restored_bins.len()
);
for (h, c, r) in mismatches.iter().take(5) {
println!(
" h={h}: continuous={c:.6}, restored={r:.6}, delta={:.6}",
r - c
);
}
}
assert_eq!(
mismatches.len(),
0,
"restored oracle diverged from continuous oracle"
);
println!();
println!("All assertions passed!");
println!("Assertion passed: from_checkpoint restart is bit-exact.");
}
+132
View File
@@ -0,0 +1,132 @@
//! Dump the RAW per-output data over a height range for fully offline analysis.
//! Nothing is filtered or binned, so any downstream filter (round-BTC tolerance,
//! dust floor, type exclusion, OP_RETURN / batch-payout tx drops, log-bin
//! resolution) can be reconstructed in analysis WITHOUT re-dumping.
//!
//! For every non-coinbase output in [ORACLE_START, ORACLE_END) (default
//! 500000..510000) one row is written:
//! oracle_outputs_{start}_{end}.csv height,tx,sats,otype
//! where `tx` is the 0-based index of the (non-coinbase) transaction within the
//! block (so OP_RETURN-tx and >N-output-tx drops can be reapplied by grouping),
//! `sats` is the exact output value, and `otype` is `OutputType as u8`.
//!
//! Plus per-block metadata:
//! oracle_meta_{start}_{end}.csv height,timestamp,ex_low,ex_high,ex_close
//!
//! Run: cargo run -p brk_oracle --example dump_hist --release
use std::{
fs::File,
io::{BufWriter, Write},
path::PathBuf,
};
use brk_indexer::Indexer;
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
fn main() {
let data_dir = std::env::var("BRK_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from(std::env::var("HOME").unwrap()).join(".brk"));
let out_dir = std::env::var("DUMP_DIR").unwrap_or_else(|_| "/tmp".to_string());
let start: usize = std::env::var("ORACLE_START")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(500_000);
let end: usize = std::env::var("ORACLE_END")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(510_000);
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
let total_heights = indexer.vecs.blocks.timestamp.len();
let end = end.min(total_heights);
let manifest_dir = env!("CARGO_MANIFEST_DIR");
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
.expect("read height_price_ohlc.json"),
)
.expect("parse height OHLC");
let timestamps: Vec<brk_types::Timestamp> = indexer.vecs.blocks.timestamp.collect();
let total_txs = indexer.vecs.transactions.txid.len();
let total_outputs = indexer.vecs.outputs.value.len();
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor();
let mut tx_starts: Vec<usize> = Vec::new();
let out_path = format!("{out_dir}/oracle_outputs_{start}_{end}.csv");
let meta_path = format!("{out_dir}/oracle_meta_{start}_{end}.csv");
let mut out_w = BufWriter::new(File::create(&out_path).expect("create outputs csv"));
let mut meta_w = BufWriter::new(File::create(&meta_path).expect("create meta csv"));
writeln!(out_w, "height,tx,sats,otype").unwrap();
writeln!(meta_w, "height,timestamp,ex_low,ex_high,ex_close").unwrap();
eprintln!(
"otype legend: OpReturn={} P2TR={}",
OutputType::OpReturn as u8,
OutputType::P2TR as u8
);
let mut rows: u64 = 0;
for h in start..end {
let ft = first_tx_index[h];
let next_ft = first_tx_index
.get(h + 1)
.copied()
.unwrap_or(TxIndex::from(total_txs));
let block_first_tx = ft.to_usize() + 1; // skip coinbase
let tx_count = next_ft.to_usize() - block_first_tx;
let out_end = out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
txout_cursor.advance(block_first_tx - txout_cursor.position());
tx_starts.clear();
for _ in 0..tx_count {
tx_starts.push(txout_cursor.next().unwrap().to_usize());
}
let out_start = tx_starts.first().copied().unwrap_or(out_end);
let values: Vec<Sats> = indexer
.vecs
.outputs
.value
.collect_range_at(out_start, out_end);
let output_types: Vec<OutputType> = indexer
.vecs
.outputs
.output_type
.collect_range_at(out_start, out_end);
for tx in 0..tx_count {
let lo = tx_starts[tx] - out_start;
let hi = tx_starts
.get(tx + 1)
.map(|s| s - out_start)
.unwrap_or(out_end - out_start);
for i in lo..hi {
writeln!(out_w, "{h},{tx},{},{}", *values[i], output_types[i] as u8).unwrap();
rows += 1;
}
}
let o = height_ohlc.get(h).copied().unwrap_or([0.0; 4]);
writeln!(
meta_w,
"{h},{},{:.2},{:.2},{:.2}",
*timestamps[h], o[2], o[1], o[3]
)
.unwrap();
}
out_w.flush().unwrap();
meta_w.flush().unwrap();
eprintln!("wrote {out_path} ({rows} output rows)");
eprintln!("wrote {meta_path}");
}
+690
View File
@@ -0,0 +1,690 @@
//! Compare oracle filter/EMA variants against the historical OHLC set.
//!
//! This is a diagnostic harness, not production code. It mirrors the production
//! state machine closely enough to compare candidate changes in one pass over
//! the indexed chain while recording the recent bad-lock heights.
//!
//! Run:
//! cargo run -p brk_oracle --example experiment --release
use std::{cmp::Ordering, env, path::PathBuf};
use brk_indexer::Indexer;
use brk_oracle::{
bin_to_cents, cents_to_bin, seed_bin as oracle_seed_bin, Config, PaymentFilter,
BINS_PER_DECADE, NUM_BINS, START_HEIGHT_FAST, START_HEIGHT_SLOW,
};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
const GENESIS_DAY: u32 = 14252;
const BINS_5PCT: f64 = 4.24;
const BINS_10PCT: f64 = 8.28;
const BINS_20PCT: f64 = 15.84;
const STENCIL_OFFSETS: [i32; 19] = [
-400, -340, -305, -260, -200, -165, -140, -120, -105, -60, 0, 35, 60, 95, 140, 200, 260, 340,
400,
];
const N_ARMS: usize = STENCIL_OFFSETS.len();
const TARGET_HEIGHTS: &[usize] = &[952_286, 952_287, 952_288, 952_289, 952_290];
fn bins_to_pct(bins: f64) -> f64 {
(10.0_f64.powf(bins / BINS_PER_DECADE as f64) - 1.0) * 100.0
}
fn timestamp_to_year(ts: u32) -> u16 {
let years_since_1970 = ts as f64 / 31_557_600.0;
(1970.0 + years_since_1970) as u16
}
#[derive(Clone)]
struct ShapeAnchor {
weight: f64,
profile: [f64; N_ARMS],
}
impl ShapeAnchor {
fn new(weight: f64) -> Self {
Self {
weight,
profile: [1.0 / N_ARMS as f64; N_ARMS],
}
}
fn score(&self, state: &OracleState, center: i64) -> f64 {
if self.weight == 0.0 {
return 0.0;
}
self.weight
* normalized_arms_at(state, center)
.map(|arms| {
1.0 - (0..N_ARMS)
.map(|i| (arms[i] - self.profile[i]).abs())
.sum::<f64>()
})
.unwrap_or(0.0)
}
fn update(&mut self, state: &OracleState, pick: i64) {
const BETA: f64 = 0.004;
if self.weight == 0.0 {
return;
}
if let Some(arms) = normalized_arms_at(state, pick) {
for (p, arm) in self.profile.iter_mut().zip(arms) {
*p = (1.0 - BETA) * *p + BETA * arm;
}
}
}
}
#[derive(Clone)]
struct OracleState {
config: Config,
ring: Vec<Vec<f64>>,
nonzero: Vec<Vec<usize>>,
weights: Vec<f64>,
cursor: usize,
filled: usize,
ref_bin: f64,
warmup: bool,
shape: ShapeAnchor,
}
impl OracleState {
fn new(ref_bin: f64, config: Config) -> Self {
let weights = weights(config.window_size, config.alpha);
Self {
ring: vec![vec![0.0; NUM_BINS]; config.window_size],
nonzero: vec![Vec::new(); config.window_size],
weights,
cursor: 0,
filled: 0,
ref_bin,
warmup: false,
shape: ShapeAnchor::new(config.shape_weight),
config,
}
}
fn reconfigure(&mut self, config: Config) {
let kept = self.recent(config.window_size);
let mut next = Self::new(self.ref_bin, config);
next.warmup = true;
for hist in kept {
next.push_existing(hist);
}
next.warmup = false;
*self = next;
}
fn recent(&self, n: usize) -> Vec<Vec<f64>> {
(0..self.filled.min(n))
.rev()
.map(|age| self.ring[self.index_at_age(age)].clone())
.collect()
}
fn index_at_age(&self, age: usize) -> usize {
(self.cursor + self.ring.len() - 1 - age) % self.ring.len()
}
fn start_block(&mut self) {
for bin in self.nonzero[self.cursor].drain(..) {
self.ring[self.cursor][bin] = 0.0;
}
}
fn add(&mut self, bin: usize, weight: f64) {
let slot = &mut self.ring[self.cursor];
if slot[bin] == 0.0 {
self.nonzero[self.cursor].push(bin);
}
slot[bin] += weight;
}
fn push_existing(&mut self, hist: Vec<f64>) {
self.start_block();
for (bin, value) in hist
.into_iter()
.enumerate()
.filter(|(_, value)| *value != 0.0)
{
self.add(bin, value);
}
self.finish_block();
}
fn finish_block(&mut self) {
self.cursor = (self.cursor + 1) % self.ring.len();
self.filled = (self.filled + 1).min(self.ring.len());
if self.warmup {
return;
}
self.ref_bin = find_best_bin(self);
let mut shape = self.shape.clone();
shape.update(self, self.ref_bin.round() as i64);
self.shape = shape;
}
fn value_at(&self, bin: i64) -> f64 {
if bin < 0 || bin as usize >= NUM_BINS {
return 0.0;
}
let bin = bin as usize;
(0..self.filled)
.map(|age| self.weights[age] * self.ring[self.index_at_age(age)][bin])
.sum()
}
}
fn weights(window_size: usize, alpha: f64) -> Vec<f64> {
let decay = 1.0 - alpha;
(0..window_size)
.map(|i| alpha * decay.powi(i as i32))
.collect()
}
fn normalized_arms_at(state: &OracleState, center: i64) -> Option<[f64; N_ARMS]> {
let mut arms = STENCIL_OFFSETS.map(|offset| state.value_at(center + offset as i64));
let sum: f64 = arms.iter().sum();
if sum <= 0.0 {
return None;
}
for arm in &mut arms {
*arm /= sum;
}
Some(arms)
}
fn find_best_bin(state: &OracleState) -> f64 {
let center = state.ref_bin.round() as usize;
let search_start = center.saturating_sub(state.config.search_below);
let search_end = (center + state.config.search_above + 1).min(NUM_BINS);
if search_start >= search_end {
return state.ref_bin;
}
let mut arm_peaks = [0.0f64; N_ARMS];
for (i, &offset) in STENCIL_OFFSETS.iter().enumerate() {
for bin in search_start..search_end {
arm_peaks[i] = arm_peaks[i].max(state.value_at(bin as i64 + offset as i64));
}
}
let score = |bin: usize| -> f64 {
let mut total = 0.0;
for (i, &offset) in STENCIL_OFFSETS.iter().enumerate() {
if arm_peaks[i] > 0.0 {
total += state.value_at(bin as i64 + offset as i64) / arm_peaks[i];
}
}
total + state.shape.score(state, bin as i64)
};
let mut best_bin = search_start;
let mut best_score = score(search_start);
for bin in (search_start + 1)..search_end {
let candidate = score(bin);
if candidate > best_score {
best_score = candidate;
best_bin = bin;
}
}
let score_center = best_score;
let score_left = if best_bin > search_start {
score(best_bin - 1)
} else {
score_center
};
let score_right = if best_bin + 1 < search_end {
score(best_bin + 1)
} else {
score_center
};
let denom = score_left - 2.0 * score_center + score_right;
let sub_bin = if denom.abs() > 1e-10 {
(0.5 * (score_left - score_right) / denom).clamp(-0.5, 0.5)
} else {
0.0
};
best_bin as f64 + sub_bin
}
#[derive(Clone)]
struct VariantCfg {
name: String,
fast_alpha: f64,
fast_window: usize,
max_outputs: Option<usize>,
max_outputs_until: usize,
max_outputs_after: Option<usize>,
}
struct Variant {
cfg: VariantCfg,
state: OracleState,
overall: YearStats,
years: Vec<YearStats>,
bias: f64,
target_prices: Vec<(usize, f64)>,
}
fn cap_label(cap: Option<usize>) -> String {
cap.map(|cap| cap.to_string())
.unwrap_or_else(|| "none".to_string())
}
fn target_price(target_prices: &[(usize, f64)], height: usize) -> Option<f64> {
target_prices
.iter()
.find(|(h, _)| *h == height)
.map(|(_, price)| *price)
}
fn fixes_bad_lock(target_prices: &[(usize, f64)]) -> bool {
target_price(target_prices, 952_287).is_some_and(|price| price > 62_000.0)
&& target_price(target_prices, 952_288).is_some_and(|price| price > 62_000.0)
}
impl Variant {
fn new(cfg: VariantCfg, seed_bin: f64) -> Self {
Self {
cfg,
state: OracleState::new(seed_bin, Config::slow()),
overall: YearStats::new(0),
years: Vec::new(),
bias: 0.0,
target_prices: Vec::new(),
}
}
fn fast_config(&self) -> Config {
Config {
alpha: self.cfg.fast_alpha,
window_size: self.cfg.fast_window,
..Config::default()
}
}
fn maybe_reconfigure(&mut self, height: usize) {
if height == START_HEIGHT_FAST {
self.state.reconfigure(self.fast_config());
}
}
fn should_drop_tx(&self, height: usize, output_count: usize) -> bool {
if height < self.cfg.max_outputs_until {
self.cfg.max_outputs.is_some_and(|max| output_count > max)
} else {
self.cfg
.max_outputs_after
.is_some_and(|max| output_count > max)
}
}
fn add_tx(&mut self, bins: &[u16], height: usize, output_count: usize) {
if bins.is_empty() || self.should_drop_tx(height, output_count) {
return;
}
for &bin in bins {
self.state.add(bin as usize, 1.0);
}
}
fn finish_block(&mut self, height: usize) {
self.state.finish_block();
if TARGET_HEIGHTS.contains(&height) {
self.target_prices
.push((height, bin_to_cents(self.state.ref_bin) as f64 / 100.0));
}
}
fn update_stats(
&mut self,
height: usize,
height_bands: &[(f64, f64)],
height_ohlc: &[[f64; 4]],
height_years: &[u16],
) {
if height >= height_bands.len() {
return;
}
let (high_bin, low_bin) = height_bands[height];
if high_bin <= 0.0 || low_bin <= 0.0 {
return;
}
let err = if self.state.ref_bin < high_bin {
self.state.ref_bin - high_bin
} else if self.state.ref_bin > low_bin {
self.state.ref_bin - low_bin
} else {
0.0
};
let exchange_high = height_ohlc[height][1];
let exchange_low = height_ohlc[height][2];
self.overall.update(err, exchange_high, exchange_low);
self.bias += err;
let year = height_years[height];
if self.years.last().is_none_or(|stats| stats.year != year) {
self.years.push(YearStats::new(year));
}
self.years
.last_mut()
.unwrap()
.update(err, exchange_high, exchange_low);
}
}
struct YearStats {
year: u16,
total_sq_err: f64,
max_err: f64,
total_blocks: u64,
gt_5pct: u64,
gt_10pct: u64,
gt_20pct: u64,
errors: Vec<f64>,
}
impl YearStats {
fn new(year: u16) -> Self {
Self {
year,
total_sq_err: 0.0,
max_err: 0.0,
total_blocks: 0,
gt_5pct: 0,
gt_10pct: 0,
gt_20pct: 0,
errors: Vec::new(),
}
}
fn update(&mut self, err: f64, _exchange_high: f64, _exchange_low: f64) {
let abs_err = err.abs();
self.total_sq_err += err * err;
self.total_blocks += 1;
self.errors.push(bins_to_pct(abs_err));
self.max_err = self.max_err.max(abs_err);
if abs_err > BINS_5PCT {
self.gt_5pct += 1;
}
if abs_err > BINS_10PCT {
self.gt_10pct += 1;
}
if abs_err > BINS_20PCT {
self.gt_20pct += 1;
}
}
fn rmse_pct(&self) -> f64 {
if self.total_blocks == 0 {
return 0.0;
}
bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt())
}
fn max_pct(&self) -> f64 {
bins_to_pct(self.max_err)
}
fn percentile(&self, p: f64) -> f64 {
if self.errors.is_empty() {
return 0.0;
}
let mut errors = self.errors.clone();
errors.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal));
let idx = ((p / 100.0) * (errors.len() - 1) as f64).round() as usize;
errors[idx.min(errors.len() - 1)]
}
}
fn main() {
let data_dir = std::env::var("BRK_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from(std::env::var("HOME").unwrap()).join(".brk"));
let end_override = std::env::var("ORACLE_END")
.ok()
.and_then(|s| s.parse::<usize>().ok());
let stats_start = std::env::var("ORACLE_STATS_START")
.ok()
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or(START_HEIGHT_SLOW)
.max(START_HEIGHT_SLOW);
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
let total_heights = indexer.vecs.blocks.timestamp.len();
let end = end_override.unwrap_or(total_heights).min(total_heights);
let manifest_dir = env!("CARGO_MANIFEST_DIR");
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
.expect("read height_price_ohlc.json"),
)
.expect("parse height OHLC");
let height_bands: Vec<(f64, f64)> = height_ohlc
.iter()
.map(|ohlc| {
let high = ohlc[1];
let low = ohlc[2];
if high > 0.0 && low > 0.0 {
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
} else {
(0.0, 0.0)
}
})
.collect();
let timestamps: Vec<brk_types::Timestamp> = indexer.vecs.blocks.timestamp.collect();
let height_years: Vec<u16> = timestamps
.iter()
.map(|ts| timestamp_to_year(**ts))
.collect();
let _height_day1s: Vec<usize> = timestamps
.iter()
.map(|ts| (**ts / 86_400).saturating_sub(GENESIS_DAY) as usize)
.collect();
let seed_bin = oracle_seed_bin();
let current_alpha = 2.0 / 7.0;
let current_window = 12;
let mut cfgs = Vec::<VariantCfg>::new();
let mut add_cfg = |name: String,
max_outputs: Option<usize>,
max_outputs_until: usize,
max_outputs_after: Option<usize>| {
cfgs.push(VariantCfg {
name,
fast_alpha: current_alpha,
fast_window: current_window,
max_outputs,
max_outputs_until,
max_outputs_after,
});
};
for post in [200, 250] {
add_cfg(
format!("pre100_post{post}"),
Some(100),
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT,
Some(post),
);
}
cfgs.dedup_by(|a, b| a.name == b.name);
if let Ok(only) = env::var("BRK_ORACLE_EXPERIMENT_ONLY") {
let names = only
.split(',')
.map(str::trim)
.filter(|name| !name.is_empty())
.collect::<Vec<_>>();
cfgs.retain(|cfg| names.iter().any(|name| *name == cfg.name));
}
let mut variants: Vec<Variant> = cfgs
.into_iter()
.map(|cfg| Variant::new(cfg, seed_bin))
.collect();
let total_txs = indexer.vecs.transactions.txid.len();
let total_outputs = indexer.vecs.outputs.value.len();
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
let mut txout_cursor = indexer.vecs.transactions.first_txout_index.cursor();
let mut tx_starts: Vec<usize> = Vec::new();
let mut values: Vec<Sats> = Vec::new();
let mut output_types: Vec<OutputType> = Vec::new();
let mut bins: Vec<u16> = Vec::new();
eprintln!(
"running {} variants over heights {START_HEIGHT_SLOW}..{end}; stats from {stats_start}",
variants.len()
);
for h in START_HEIGHT_SLOW..end {
if h % 25_000 == 0 {
eprintln!("height {h}");
}
for variant in &mut variants {
variant.maybe_reconfigure(h);
variant.state.start_block();
}
let ft = first_tx_index[h];
let next_ft = first_tx_index
.get(h + 1)
.copied()
.unwrap_or(TxIndex::from(total_txs));
let block_first_tx = ft.to_usize() + 1;
let tx_count = next_ft.to_usize() - block_first_tx;
let out_end = out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
txout_cursor.advance(block_first_tx - txout_cursor.position());
tx_starts.clear();
for _ in 0..tx_count {
tx_starts.push(txout_cursor.next().unwrap().to_usize());
}
let out_start = tx_starts.first().copied().unwrap_or(out_end);
indexer
.vecs
.outputs
.value
.collect_range_into_at(out_start, out_end, &mut values);
indexer.vecs.outputs.output_type.collect_range_into_at(
out_start,
out_end,
&mut output_types,
);
for tx in 0..tx_count {
let lo = tx_starts[tx] - out_start;
let hi = tx_starts
.get(tx + 1)
.map(|s| s - out_start)
.unwrap_or(out_end - out_start);
if output_types[lo..hi].contains(&OutputType::OpReturn) {
continue;
}
bins.clear();
for i in lo..hi {
if let Some(bin) = PaymentFilter::eligible_bin(values[i], output_types[i]) {
bins.push(bin);
}
}
for variant in &mut variants {
variant.add_tx(&bins, h, hi - lo);
}
}
for variant in &mut variants {
variant.finish_block(h);
if h >= stats_start {
variant.update_stats(h, &height_bands, &height_ohlc, &height_years);
}
}
}
variants.sort_by(|a, b| {
fixes_bad_lock(&b.target_prices)
.cmp(&fixes_bad_lock(&a.target_prices))
.then_with(|| {
a.overall
.rmse_pct()
.partial_cmp(&b.overall.rmse_pct())
.unwrap_or(Ordering::Equal)
})
.then_with(|| a.overall.gt_5pct.cmp(&b.overall.gt_5pct))
});
println!(
"variant\tpre_cap\tpost_cap\tfixed\tmedian\tp95\tp99\tp999\trmse\tmax\tbias_bins\tgt5\tgt10\tgt20\tp952287\tp952288\ttarget_prices\trmse_by_year\tgt5_by_year"
);
for variant in &variants {
let overall = &variant.overall;
let bias = if overall.total_blocks > 0 {
variant.bias / overall.total_blocks as f64
} else {
0.0
};
let rmse_by_year = (2015..=2026)
.map(|year| {
let rmse = variant
.years
.iter()
.find(|stats| stats.year == year)
.map(YearStats::rmse_pct)
.unwrap_or(0.0);
format!("{year}:{rmse:.3}")
})
.collect::<Vec<_>>()
.join(",");
let gt5_by_year = (2015..=2026)
.map(|year| {
let gt5 = variant
.years
.iter()
.find(|stats| stats.year == year)
.map(|stats| stats.gt_5pct)
.unwrap_or(0);
format!("{year}:{gt5}")
})
.collect::<Vec<_>>()
.join(",");
println!(
"{}\t{}\t{}\t{}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{}\t{}\t{}\t{:.2}\t{:.2}\t{}\t{}\t{}",
variant.cfg.name,
cap_label(variant.cfg.max_outputs),
cap_label(variant.cfg.max_outputs_after),
fixes_bad_lock(&variant.target_prices),
overall.percentile(50.0),
overall.percentile(95.0),
overall.percentile(99.0),
overall.percentile(99.9),
overall.rmse_pct(),
overall.max_pct(),
bias,
overall.gt_5pct,
overall.gt_10pct,
overall.gt_20pct,
target_price(&variant.target_prices, 952_287).unwrap_or(0.0),
target_price(&variant.target_prices, 952_288).unwrap_or(0.0),
variant
.target_prices
.iter()
.map(|(height, price)| format!("{height}:{price:.2}"))
.collect::<Vec<_>>()
.join(","),
rmse_by_year,
gt5_by_year
);
}
}
-272
View File
@@ -1,272 +0,0 @@
//! Diagnostic: sweep oracle start heights and clamp-top-N strategies.
//!
//! Run with: cargo run -p brk_oracle --example noise --release
use std::path::PathBuf;
use std::time::Instant;
use brk_indexer::Indexer;
use brk_oracle::{Config, Histogram, Oracle, PRICES, cents_to_bin, default_eligible_bin};
use brk_types::{Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
const BINS_5PCT: f64 = 4.24;
const BINS_10PCT: f64 = 8.28;
const BINS_20PCT: f64 = 15.84;
const BPD: f64 = 200.0;
fn bins_to_pct(bins: f64) -> f64 {
(10.0_f64.powf(bins / BPD) - 1.0) * 100.0
}
fn seed_bin(start_height: usize) -> f64 {
let price: f64 = PRICES
.lines()
.nth(start_height - 1)
.expect("prices.txt too short")
.parse()
.expect("Failed to parse seed price");
cents_to_bin(price * 100.0)
}
/// Clamp the top N bins in `src` down to the (N+1)th highest value, writing into `dst`.
fn clamp_top_n(src: &Histogram, dst: &mut Histogram, n: usize) {
let mut top: Vec<u32> = src.iter().copied().filter(|&v| v > 0).collect();
top.sort_unstable_by(|a, b| b.cmp(a));
let clamp_to = if top.len() > n { top[n] } else { 0 };
for (i, &v) in src.iter().enumerate() {
dst[i] = v.min(clamp_to.max(v.min(clamp_to)));
}
}
fn main() {
let t0 = Instant::now();
let data_dir = std::env::var("BRK_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| {
let home = std::env::var("HOME").unwrap();
PathBuf::from(home).join(".brk")
});
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
let total_heights = indexer.vecs.blocks.timestamp.len();
let manifest_dir = env!("CARGO_MANIFEST_DIR");
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
.expect("Failed to read height_price_ohlc.json"),
)
.expect("Failed to parse height OHLC");
let height_bands: Vec<(f64, f64)> = height_ohlc
.iter()
.map(|ohlc| {
let high = ohlc[1];
let low = ohlc[2];
if high > 0.0 && low > 0.0 {
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
} else {
(0.0, 0.0)
}
})
.collect();
// Start heights: 630k, 600k, 575k, then 570k down to 500k by 5k.
let mut start_heights: Vec<usize> = vec![630_000, 600_000, 575_000];
let mut h = 570_000;
while h >= 500_000 {
start_heights.push(h);
h -= 5_000;
}
let lowest = *start_heights.iter().min().unwrap();
// Clamp-top-N values to test: 0 (no clamp), 2, 3, 5, 10.
let clamp_values: Vec<usize> = vec![0, 2, 3, 5, 10];
// Build per-block RAW histograms from the lowest start height.
eprintln!("Building histograms from height {}...", lowest);
let total_txs = indexer.vecs.transactions.txid.len();
let total_outputs = indexer.vecs.outputs.value.len();
let first_txout_index_reader = indexer.vecs.transactions.first_txout_index.reader();
let value_reader = indexer.vecs.outputs.value.reader();
let output_type_reader = indexer.vecs.outputs.output_type.reader();
let config = Config::default();
let total_blocks = total_heights - lowest;
struct BlockData {
hist: Histogram,
high_bin: f64,
low_bin: f64,
}
let mut blocks: Vec<BlockData> = Vec::with_capacity(total_blocks);
for h in lowest..total_heights {
let first_tx_index: TxIndex = indexer
.vecs
.transactions
.first_tx_index
.collect_one_at(h)
.unwrap();
let next_first_tx_index: TxIndex = indexer
.vecs
.transactions
.first_tx_index
.collect_one_at(h + 1)
.unwrap_or(TxIndex::from(total_txs));
let out_start = if first_tx_index.to_usize() + 1 < next_first_tx_index.to_usize() {
first_txout_index_reader
.get(first_tx_index.to_usize() + 1)
.to_usize()
} else {
indexer
.vecs
.outputs
.first_txout_index
.collect_one_at(h + 1)
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize()
};
let out_end: usize = indexer
.vecs
.outputs
.first_txout_index
.collect_one_at(h + 1)
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
let mut hist = Histogram::zeros();
for i in out_start..out_end {
let sats: Sats = value_reader.get(i);
let output_type = output_type_reader.get(i);
if let Some(bin) = default_eligible_bin(sats, output_type) {
hist.increment(bin as usize);
}
}
let (high_bin, low_bin) = if h < height_bands.len() {
height_bands[h]
} else {
(0.0, 0.0)
};
blocks.push(BlockData {
hist,
high_bin,
low_bin,
});
if (h - lowest).is_multiple_of(50_000) {
eprint!(
"\r {}/{} ({:.0}%)",
h - lowest,
total_blocks,
(h - lowest) as f64 / total_blocks as f64 * 100.0
);
}
}
eprintln!(
"\r {} blocks built in {:.1}s",
blocks.len(),
t0.elapsed().as_secs_f64()
);
// For each clamp value, run all start heights.
for &clamp_n in &clamp_values {
println!();
let label = if clamp_n == 0 {
"no clamp".to_string()
} else {
format!("clamp top {}", clamp_n)
};
println!("=== {} ===", label);
println!(
"{:>8} {:>8} {:>8} {:>8} {:>6} {:>6} {:>6} {:>8}",
"Start", "Blocks", "RMSE%", "Worst%", ">5%", ">10%", ">20%", "Worst@"
);
println!("{}", "-".repeat(72));
for &start_height in &start_heights {
let mut oracle = Oracle::new(seed_bin(start_height), config.clone());
let block_offset = start_height - lowest;
let mut worst_err: f64 = 0.0;
let mut worst_height: usize = 0;
let mut gt_5: u64 = 0;
let mut gt_10: u64 = 0;
let mut gt_20: u64 = 0;
let mut total_sq_err: f64 = 0.0;
let mut total_measured: u64 = 0;
let mut clamped_hist = Histogram::zeros();
for (i, bd) in blocks[block_offset..].iter().enumerate() {
if clamp_n > 0 {
clamp_top_n(&bd.hist, &mut clamped_hist, clamp_n);
oracle.process_histogram(&clamped_hist);
} else {
oracle.process_histogram(&bd.hist);
}
let height = start_height + i;
let ref_bin = oracle.ref_bin();
if bd.high_bin <= 0.0 || bd.low_bin <= 0.0 {
continue;
}
let err = if ref_bin < bd.high_bin {
ref_bin - bd.high_bin
} else if ref_bin > bd.low_bin {
ref_bin - bd.low_bin
} else {
0.0
};
total_measured += 1;
total_sq_err += err * err;
let abs_err = err.abs();
if abs_err > BINS_5PCT {
gt_5 += 1;
}
if abs_err > BINS_10PCT {
gt_10 += 1;
}
if abs_err > BINS_20PCT {
gt_20 += 1;
}
if abs_err > worst_err {
worst_err = abs_err;
worst_height = height;
}
}
let rmse = if total_measured > 0 {
bins_to_pct((total_sq_err / total_measured as f64).sqrt())
} else {
0.0
};
println!(
"{:>8} {:>8} {:>7.3}% {:>7.1}% {:>6} {:>6} {:>6} {}",
format!("{}k", start_height / 1000),
total_measured,
rmse,
bins_to_pct(worst_err),
gt_5,
gt_10,
gt_20,
worst_height,
);
}
}
println!("\nTotal time: {:.1}s", t0.elapsed().as_secs_f64());
}
+24 -28
View File
@@ -6,8 +6,7 @@ use std::path::PathBuf;
use brk_indexer::Indexer;
use brk_oracle::{
Config, Histogram, Oracle, PRICES, START_HEIGHT, bin_to_cents, cents_to_bin,
default_eligible_bin,
bin_to_cents, cents_to_bin, Config, Oracle, PaymentFilter, START_HEIGHT_FAST, START_HEIGHT_SLOW,
};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
@@ -94,7 +93,11 @@ impl YearStats {
fn median_pct(&mut self) -> f64 {
self.errors.sort_by(|a, b| a.partial_cmp(b).unwrap());
let n = self.errors.len();
if n == 0 { 0.0 } else { self.errors[n / 2] }
if n == 0 {
0.0
} else {
self.errors[n / 2]
}
}
fn percentile(&self, p: f64) -> f64 {
@@ -172,15 +175,7 @@ fn main() {
.map(|ts| (**ts / 86400).saturating_sub(GENESIS_DAY) as usize)
.collect();
let start_price: f64 = PRICES
.lines()
.nth(START_HEIGHT - 1)
.expect("prices.txt too short")
.parse()
.expect("Failed to parse seed price");
let config = Config::default();
let mut oracle = Oracle::new(cents_to_bin(start_price * 100.0), config);
let mut oracle = Oracle::from_seed();
let total_txs = indexer.vecs.transactions.txid.len();
let total_outputs = indexer.vecs.outputs.value.len();
@@ -201,7 +196,11 @@ fn main() {
let mut oracle_candles: Vec<DayCandle> = Vec::new();
let mut current_di: Option<usize> = None;
for h in START_HEIGHT..total_heights {
for h in START_HEIGHT_SLOW..total_heights {
if h == START_HEIGHT_FAST {
oracle.reconfigure(Config::default());
}
let ft = first_tx_index[h];
let next_ft = first_tx_index
.get(h + 1)
@@ -235,23 +234,18 @@ fn main() {
.output_type
.collect_range_at(out_start, out_end);
// Drop every output of a tx carrying an OP_RETURN (protocol machinery).
let mut hist = Histogram::zeros();
for tx in 0..tx_count {
let tx_outputs = (0..tx_count).map(|tx| {
let lo = tx_starts[tx] - out_start;
let hi = tx_starts
.get(tx + 1)
.map(|s| s - out_start)
.unwrap_or(out_end - out_start);
if output_types[lo..hi].contains(&OutputType::OpReturn) {
continue;
}
for i in lo..hi {
if let Some(bin) = default_eligible_bin(values[i], output_types[i]) {
hist.increment(bin as usize);
}
}
}
values[lo..hi]
.iter()
.copied()
.zip(output_types[lo..hi].iter().copied())
});
let hist = PaymentFilter::for_height(h).histogram(tx_outputs);
let ref_bin = oracle.process_histogram(&hist);
let oracle_price = bin_to_cents(ref_bin) as f64 / 100.0;
@@ -373,10 +367,12 @@ fn main() {
println!(" brk_oracle accuracy report");
println!(" ══════════════════════════");
println!();
println!(" Config: w12, alpha=2/7, search -9/+11, noisy/dust/round-btc filtered");
println!(
" Test range: height {} .. {} ({} blocks)",
START_HEIGHT,
" Config: slow w40 alpha=0.10 until {START_HEIGHT_FAST}, then w12 alpha=2/7; shared payment filter"
);
println!(
" Test range: height {} .. {} ({} exchange-covered blocks)",
START_HEIGHT_SLOW,
total_heights - 1,
overall.total_blocks
);
File diff suppressed because it is too large Load Diff
-416
View File
@@ -1,416 +0,0 @@
//! Sweep round-value digit filter to find optimal configuration.
//!
//! Tests all 512 subsets of leading digits {1,...,9} to find which
//! digits to filter out for best oracle accuracy.
//!
//! Phase 1: single pass over indexer, precompute per-block histograms.
//! Phase 2: run 512 configs in parallel across CPU cores.
//!
//! Run with: cargo run -p brk_oracle --example sweep_digits --release
use std::path::PathBuf;
use std::time::Instant;
use brk_indexer::Indexer;
use brk_oracle::{Config, Histogram, Oracle, PRICES, cents_to_bin, sats_to_bin};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
const BINS_5PCT: f64 = 4.24;
const BINS_10PCT: f64 = 8.28;
const BINS_20PCT: f64 = 15.84;
fn bins_to_pct(bins: f64) -> f64 {
(10.0_f64.powf(bins / 200.0) - 1.0) * 100.0
}
fn seed_bin(start_height: usize) -> f64 {
let price: f64 = PRICES
.lines()
.nth(start_height - 1)
.expect("prices.txt too short")
.parse()
.expect("Failed to parse seed price");
cents_to_bin(price * 100.0)
}
fn leading_digit(sats: u64) -> u8 {
let log = (sats as f64).log10();
let magnitude = 10.0_f64.powf(log.floor());
let d = (sats as f64 / magnitude).round() as u8;
if d >= 10 { 1 } else { d }
}
fn is_round(sats: u64) -> bool {
let log = (sats as f64).log10();
let magnitude = 10.0_f64.powf(log.floor());
let leading = (sats as f64 / magnitude).round();
let round_val = leading * magnitude;
(sats as f64 - round_val).abs() <= round_val * 0.001
}
fn mask_label(mask: u16) -> String {
let digits: String = (1..=9u8)
.filter(|&d| mask & (1 << (d - 1)) != 0)
.map(|d| char::from_digit(d as u32, 10).unwrap())
.collect();
if digits.is_empty() {
"none".to_string()
} else {
digits
}
}
struct Stats {
total_sq_err: f64,
total_bias: f64,
max_err: f64,
total_blocks: u64,
gt_5pct: u64,
gt_10pct: u64,
gt_20pct: u64,
}
impl Stats {
fn new() -> Self {
Self {
total_sq_err: 0.0,
total_bias: 0.0,
max_err: 0.0,
total_blocks: 0,
gt_5pct: 0,
gt_10pct: 0,
gt_20pct: 0,
}
}
fn update(&mut self, err: f64) {
self.total_sq_err += err * err;
self.total_bias += err;
self.total_blocks += 1;
let abs_err = err.abs();
if abs_err > self.max_err {
self.max_err = abs_err;
}
if abs_err > BINS_5PCT {
self.gt_5pct += 1;
}
if abs_err > BINS_10PCT {
self.gt_10pct += 1;
}
if abs_err > BINS_20PCT {
self.gt_20pct += 1;
}
}
fn rmse_pct(&self) -> f64 {
bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt())
}
fn max_pct(&self) -> f64 {
bins_to_pct(self.max_err)
}
fn bias(&self) -> f64 {
self.total_bias / self.total_blocks as f64
}
}
struct BlockData {
full_hist: Histogram,
/// (bin_index, leading_digit) for outputs that are round values.
round_outputs: Vec<(u16, u8)>,
high_bin: f64,
low_bin: f64,
}
fn main() {
let t0 = Instant::now();
let data_dir = std::env::var("BRK_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| {
let home = std::env::var("HOME").unwrap();
PathBuf::from(home).join(".brk")
});
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
let total_heights = indexer.vecs.blocks.timestamp.len();
let manifest_dir = env!("CARGO_MANIFEST_DIR");
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
.expect("Failed to read height_price_ohlc.json"),
)
.expect("Failed to parse height OHLC");
let height_bands: Vec<(f64, f64)> = height_ohlc
.iter()
.map(|ohlc| {
let high = ohlc[1];
let low = ohlc[2];
if high > 0.0 && low > 0.0 {
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
} else {
(0.0, 0.0)
}
})
.collect();
let sweep_start: usize = 575_000;
// Phase 1: precompute per-block data in a single pass over the indexer.
eprintln!("Phase 1: precomputing block data...");
let total_txs = indexer.vecs.transactions.txid.len();
let total_outputs = indexer.vecs.outputs.value.len();
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
let ref_config = Config::default();
let total_blocks = total_heights - sweep_start;
let mut blocks: Vec<BlockData> = Vec::with_capacity(total_blocks);
for h in sweep_start..total_heights {
let ft = first_tx_index[h];
let next_ft = first_tx_index
.get(h + 1)
.copied()
.unwrap_or(TxIndex::from(total_txs));
let out_start = if ft.to_usize() + 1 < next_ft.to_usize() {
indexer
.vecs
.transactions
.first_txout_index
.collect_one(ft + 1)
.unwrap()
.to_usize()
} else {
out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize()
};
let out_end = out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
let values: Vec<Sats> = indexer
.vecs
.outputs
.value
.collect_range_at(out_start, out_end);
let output_types: Vec<OutputType> = indexer
.vecs
.outputs
.output_type
.collect_range_at(out_start, out_end);
let mut full_hist = Histogram::zeros();
let mut round_outputs = Vec::new();
for (sats, output_type) in values.into_iter().zip(output_types) {
if ref_config.excluded_output_types.contains(&output_type) {
continue;
}
if *sats < ref_config.min_sats {
continue;
}
if let Some(bin) = sats_to_bin(sats) {
full_hist.increment(bin);
if is_round(*sats) {
let d = leading_digit(*sats);
if (1..=9).contains(&d) {
round_outputs.push((bin as u16, d));
}
}
}
}
let (high_bin, low_bin) = if h < height_bands.len() {
height_bands[h]
} else {
(0.0, 0.0)
};
blocks.push(BlockData {
full_hist,
round_outputs,
high_bin,
low_bin,
});
if (h - sweep_start).is_multiple_of(50_000) {
eprint!(
"\r {}/{} ({:.0}%)",
h - sweep_start,
total_blocks,
(h - sweep_start) as f64 / total_blocks as f64 * 100.0
);
}
}
let mem_hists = blocks.len() * std::mem::size_of::<Histogram>();
let mem_rounds: usize = blocks.iter().map(|b| b.round_outputs.len() * 3).sum();
eprintln!(
"\r {} blocks precomputed ({:.1} GB hists + {:.0} MB rounds) in {:.1}s",
blocks.len(),
mem_hists as f64 / 1e9,
mem_rounds as f64 / 1e6,
t0.elapsed().as_secs_f64()
);
// Phase 2: sweep digit masks in parallel.
// Always filter digit 1 (powers of 10), sweep digits 2-9.
let base_mask: u16 = 1 << 0; // digit 1 always on
let num_masks: usize = 256; // 2^8 subsets of {2,...,9}
let num_threads = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(8);
eprintln!(
"Phase 2: sweeping {} masks across {} threads...",
num_masks, num_threads
);
let t1 = Instant::now();
let blocks = &blocks; // shared reference for threads
let all_results: Vec<(u16, Stats)> = std::thread::scope(|s| {
let masks_per_thread = num_masks.div_ceil(num_threads);
let handles: Vec<_> = (0..num_threads)
.map(|t| {
s.spawn(move || {
let mask_start = t * masks_per_thread;
let mask_end = ((t + 1) * masks_per_thread).min(num_masks);
let mut results = Vec::with_capacity(mask_end - mask_start);
for idx in mask_start..mask_end {
// Shift idx bits into positions 1-8 (digits 2-9) and add base_mask (digit 1).
let mask = base_mask | ((idx as u16) << 1);
let mut oracle = Oracle::new(
seed_bin(sweep_start),
Config {
exclude_common_round_values: false,
..Default::default()
},
);
let mut stats = Stats::new();
for bd in blocks.iter() {
let mut hist = bd.full_hist.clone();
for &(bin, digit) in &bd.round_outputs {
if mask & (1 << (digit - 1)) != 0 {
hist[bin as usize] -= 1;
}
}
let ref_bin = oracle.process_histogram(&hist);
if bd.high_bin > 0.0 && bd.low_bin > 0.0 {
let err = if ref_bin < bd.high_bin {
ref_bin - bd.high_bin
} else if ref_bin > bd.low_bin {
ref_bin - bd.low_bin
} else {
0.0
};
stats.update(err);
}
}
results.push((mask, stats));
}
results
})
})
.collect();
handles
.into_iter()
.flat_map(|h| h.join().unwrap())
.collect()
});
eprintln!(" Done in {:.1}s.", t1.elapsed().as_secs_f64());
// Sort by RMSE.
let mut results: Vec<&(u16, Stats)> = all_results.iter().collect();
results.sort_by(|a, b| a.1.rmse_pct().partial_cmp(&b.1.rmse_pct()).unwrap());
// Print top 20.
println!();
println!("Top 20 (by RMSE):");
println!(
"{:>4} {:>12} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}",
"#", "Digits", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias"
);
println!("{}", "-".repeat(70));
for (rank, (mask, s)) in results.iter().take(20).enumerate() {
println!(
"{:>4} {:>12} {:>8.3}% {:>8.1}% {:>6} {:>6} {:>6} {:>+8.2}",
rank + 1,
mask_label(*mask),
s.rmse_pct(),
s.max_pct(),
s.gt_5pct,
s.gt_10pct,
s.gt_20pct,
s.bias()
);
}
// Print bottom 5.
println!();
println!("Bottom 5 (worst):");
println!(
"{:>4} {:>12} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}",
"#", "Digits", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias"
);
println!("{}", "-".repeat(70));
for (mask, s) in results.iter().rev().take(5) {
println!(
"{:>4} {:>12} {:>8.3}% {:>8.1}% {:>6} {:>6} {:>6} {:>+8.2}",
"",
mask_label(*mask),
s.rmse_pct(),
s.max_pct(),
s.gt_5pct,
s.gt_10pct,
s.gt_20pct,
s.bias()
);
}
// Print current config {1,2,3,5} for reference.
let current_mask: u16 = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 4); // digits 1,2,3,5
let current_stats = all_results
.iter()
.find(|(m, _)| *m == current_mask)
.map(|(_, s)| s)
.unwrap();
let current_rank = results
.iter()
.position(|(m, _)| *m == current_mask)
.unwrap();
println!();
println!(
"Current {{1,2,3,5}} = rank {}/{}: RMSE {:.3}%, Max {:.1}%, >5%: {}, >10%: {}, >20%: {}",
current_rank + 1,
num_masks,
current_stats.rmse_pct(),
current_stats.max_pct(),
current_stats.gt_5pct,
current_stats.gt_10pct,
current_stats.gt_20pct,
);
println!("\nTotal time: {:.1}s", t0.elapsed().as_secs_f64());
}
@@ -1,452 +0,0 @@
//! Sweep round-value tolerance to find optimal rounding threshold.
//!
//! Tests different tolerance percentages (0%, 0.01%, 0.1%, 1%, etc.) for
//! detecting round BTC amounts, combined with several digit filter masks.
//!
//! Phase 1: single pass over indexer, store per-output relative errors.
//! Phase 2: sweep tolerance × mask combos across CPU cores.
//!
//! Run with: cargo run -p brk_oracle --example sweep_tolerance --release
use std::path::PathBuf;
use std::time::Instant;
use brk_indexer::Indexer;
use brk_oracle::{Config, Histogram, Oracle, PRICES, cents_to_bin, sats_to_bin};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
const BINS_5PCT: f64 = 4.24;
const BINS_10PCT: f64 = 8.28;
const BINS_20PCT: f64 = 15.84;
fn bins_to_pct(bins: f64) -> f64 {
(10.0_f64.powf(bins / 200.0) - 1.0) * 100.0
}
fn seed_bin(start_height: usize) -> f64 {
let price: f64 = PRICES
.lines()
.nth(start_height - 1)
.expect("prices.txt too short")
.parse()
.expect("Failed to parse seed price");
cents_to_bin(price * 100.0)
}
fn leading_digit(sats: u64) -> u8 {
let log = (sats as f64).log10();
let magnitude = 10.0_f64.powf(log.floor());
let d = (sats as f64 / magnitude).round() as u8;
if d >= 10 { 1 } else { d }
}
/// Returns the relative error of `sats` from its nearest round value (d × 10^n).
/// e.g. 10_050 → leading=1, round_val=10_000, rel_err = 50/10000 = 0.005
fn relative_roundness(sats: u64) -> f64 {
let log = (sats as f64).log10();
let magnitude = 10.0_f64.powf(log.floor());
let leading = (sats as f64 / magnitude).round();
let round_val = leading * magnitude;
(sats as f64 - round_val).abs() / round_val
}
struct Stats {
total_sq_err: f64,
total_bias: f64,
max_err: f64,
total_blocks: u64,
gt_5pct: u64,
gt_10pct: u64,
gt_20pct: u64,
}
impl Stats {
fn new() -> Self {
Self {
total_sq_err: 0.0,
total_bias: 0.0,
max_err: 0.0,
total_blocks: 0,
gt_5pct: 0,
gt_10pct: 0,
gt_20pct: 0,
}
}
fn update(&mut self, err: f64) {
self.total_sq_err += err * err;
self.total_bias += err;
self.total_blocks += 1;
let abs_err = err.abs();
if abs_err > self.max_err {
self.max_err = abs_err;
}
if abs_err > BINS_5PCT {
self.gt_5pct += 1;
}
if abs_err > BINS_10PCT {
self.gt_10pct += 1;
}
if abs_err > BINS_20PCT {
self.gt_20pct += 1;
}
}
fn rmse_pct(&self) -> f64 {
bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt())
}
fn max_pct(&self) -> f64 {
bins_to_pct(self.max_err)
}
fn bias(&self) -> f64 {
self.total_bias / self.total_blocks as f64
}
}
/// Per-output data: bin index, leading digit, relative error from round value.
struct RoundOutput {
bin: u16,
digit: u8,
rel_err: f32, // f32 is plenty of precision, saves memory
}
struct BlockData {
full_hist: Histogram,
round_outputs: Vec<RoundOutput>,
high_bin: f64,
low_bin: f64,
}
fn main() {
let t0 = Instant::now();
let data_dir = std::env::var("BRK_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| {
let home = std::env::var("HOME").unwrap();
PathBuf::from(home).join(".brk")
});
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
let total_heights = indexer.vecs.blocks.timestamp.len();
let manifest_dir = env!("CARGO_MANIFEST_DIR");
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
.expect("Failed to read height_price_ohlc.json"),
)
.expect("Failed to parse height OHLC");
let height_bands: Vec<(f64, f64)> = height_ohlc
.iter()
.map(|ohlc| {
let high = ohlc[1];
let low = ohlc[2];
if high > 0.0 && low > 0.0 {
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
} else {
(0.0, 0.0)
}
})
.collect();
let sweep_start: usize = 575_000;
// Phase 1: precompute per-block data.
// Store all potentially-round outputs with their relative error so we can
// filter at different tolerance thresholds in Phase 2.
eprintln!("Phase 1: precomputing block data...");
let total_txs = indexer.vecs.transactions.txid.len();
let total_outputs = indexer.vecs.outputs.value.len();
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
let ref_config = Config::default();
let total_blocks = total_heights - sweep_start;
let mut blocks: Vec<BlockData> = Vec::with_capacity(total_blocks);
// Use the widest tolerance we'll test (5%) to decide what to store.
// Outputs beyond 5% relative error will never be filtered at any tolerance.
let max_tolerance: f64 = 0.05;
for h in sweep_start..total_heights {
let ft = first_tx_index[h];
let next_ft = first_tx_index
.get(h + 1)
.copied()
.unwrap_or(TxIndex::from(total_txs));
let out_start = if ft.to_usize() + 1 < next_ft.to_usize() {
indexer
.vecs
.transactions
.first_txout_index
.collect_one(ft + 1)
.unwrap()
.to_usize()
} else {
out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize()
};
let out_end = out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
let values: Vec<Sats> = indexer
.vecs
.outputs
.value
.collect_range_at(out_start, out_end);
let output_types: Vec<OutputType> = indexer
.vecs
.outputs
.output_type
.collect_range_at(out_start, out_end);
let mut full_hist = Histogram::zeros();
let mut round_outputs = Vec::new();
for (sats, output_type) in values.into_iter().zip(output_types) {
if ref_config.excluded_output_types.contains(&output_type) {
continue;
}
if *sats < ref_config.min_sats {
continue;
}
if let Some(bin) = sats_to_bin(sats) {
full_hist.increment(bin);
let d = leading_digit(*sats);
if (1..=9).contains(&d) {
let rel_err = relative_roundness(*sats);
if rel_err <= max_tolerance {
round_outputs.push(RoundOutput {
bin: bin as u16,
digit: d,
rel_err: rel_err as f32,
});
}
}
}
}
let (high_bin, low_bin) = if h < height_bands.len() {
height_bands[h]
} else {
(0.0, 0.0)
};
blocks.push(BlockData {
full_hist,
round_outputs,
high_bin,
low_bin,
});
if (h - sweep_start).is_multiple_of(50_000) {
eprint!(
"\r {}/{} ({:.0}%)",
h - sweep_start,
total_blocks,
(h - sweep_start) as f64 / total_blocks as f64 * 100.0
);
}
}
let mem_hists = blocks.len() * std::mem::size_of::<Histogram>();
let mem_rounds: usize = blocks
.iter()
.map(|b| b.round_outputs.len() * std::mem::size_of::<RoundOutput>())
.sum();
eprintln!(
"\r {} blocks precomputed ({:.1} GB hists + {:.0} MB rounds) in {:.1}s",
blocks.len(),
mem_hists as f64 / 1e9,
mem_rounds as f64 / 1e6,
t0.elapsed().as_secs_f64()
);
// Phase 2: sweep tolerance × mask combos.
// Tolerances as fractions (not percentages).
let tolerances: &[(f64, &str)] = &[
(0.0, "0%"),
(0.0001, "0.01%"),
(0.0005, "0.05%"),
(0.001, "0.1%"),
(0.002, "0.2%"),
(0.005, "0.5%"),
(0.01, "1%"),
(0.02, "2%"),
(0.05, "5%"),
];
// 987654321
let masks: &[(u16, &str)] = &[
(0b0_0000_0000, "none"),
(0b0_0001_0111, "{1,2,3,5}"),
(0b0_0001_1111, "{1,2,3,4,5}"),
(0b0_0011_0111, "{1,2,3,5,6}"),
(0b0_0111_0111, "{1,2,3,5,6,7}"),
(0b1_1111_1111, "{1-9}"),
];
let num_configs = tolerances.len() * masks.len();
let num_threads = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(8);
eprintln!(
"Phase 2: sweeping {} configs ({} tolerances × {} masks) across {} threads...",
num_configs,
tolerances.len(),
masks.len(),
num_threads
);
let t1 = Instant::now();
let blocks = &blocks;
let tolerances_ref = tolerances;
let masks_ref = masks;
let all_results: Vec<(usize, usize, Stats)> = std::thread::scope(|s| {
let configs_per_thread = num_configs.div_ceil(num_threads);
let handles: Vec<_> = (0..num_threads)
.map(|t| {
s.spawn(move || {
let cfg_start = t * configs_per_thread;
let cfg_end = ((t + 1) * configs_per_thread).min(num_configs);
if cfg_start >= cfg_end {
return vec![];
}
let mut results = Vec::with_capacity(cfg_end - cfg_start);
for cfg_idx in cfg_start..cfg_end {
let ti = cfg_idx / masks_ref.len();
let mi = cfg_idx % masks_ref.len();
let (tolerance, _) = tolerances_ref[ti];
let (mask, _) = masks_ref[mi];
let mut oracle = Oracle::new(
seed_bin(sweep_start),
Config {
exclude_common_round_values: false,
..Default::default()
},
);
let mut stats = Stats::new();
for bd in blocks.iter() {
let mut hist = bd.full_hist.clone();
// Remove outputs matching this tolerance + mask.
let tol_f32 = tolerance as f32;
for ro in &bd.round_outputs {
if mask & (1 << (ro.digit - 1)) != 0 && ro.rel_err <= tol_f32 {
hist[ro.bin as usize] -= 1;
}
}
let ref_bin = oracle.process_histogram(&hist);
if bd.high_bin > 0.0 && bd.low_bin > 0.0 {
let err = if ref_bin < bd.high_bin {
ref_bin - bd.high_bin
} else if ref_bin > bd.low_bin {
ref_bin - bd.low_bin
} else {
0.0
};
stats.update(err);
}
}
results.push((ti, mi, stats));
}
results
})
})
.collect();
handles
.into_iter()
.flat_map(|h| h.join().unwrap())
.collect()
});
eprintln!(" Done in {:.1}s.", t1.elapsed().as_secs_f64());
// Print results grouped by tolerance.
println!();
println!(
"{:>8} {:>16} {:>8} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}",
"Tol", "Digits", "Blocks", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias"
);
println!("{}", "-".repeat(88));
for (ti, &(_, tol_label)) in tolerances.iter().enumerate() {
for (mi, &(_, mask_label)) in masks.iter().enumerate() {
let (_, _, stats) = all_results
.iter()
.find(|(t, m, _)| *t == ti && *m == mi)
.unwrap();
println!(
"{:>8} {:>16} {:>8} {:>8.3}% {:>8.1}% {:>6} {:>6} {:>6} {:>+8.2}",
tol_label,
mask_label,
stats.total_blocks,
stats.rmse_pct(),
stats.max_pct(),
stats.gt_5pct,
stats.gt_10pct,
stats.gt_20pct,
stats.bias()
);
}
println!();
}
// Find overall best config by RMSE.
let best = all_results
.iter()
.min_by(|a, b| a.2.rmse_pct().partial_cmp(&b.2.rmse_pct()).unwrap())
.unwrap();
let (bti, bmi, bs) = best;
println!(
"Best: tolerance={}, digits={} → RMSE {:.3}%, Max {:.1}%, >5%: {}, >10%: {}, >20%: {}",
tolerances[*bti].1,
masks[*bmi].1,
bs.rmse_pct(),
bs.max_pct(),
bs.gt_5pct,
bs.gt_10pct,
bs.gt_20pct,
);
// Show current config for reference.
let current = all_results
.iter()
.find(|(t, m, _)| tolerances[*t].0 == 0.001 && masks[*m].0 == 0b0_0011_0111)
.unwrap();
let (_, _, cs) = current;
println!(
"Current: tolerance=0.1%, digits={{1,2,3,5,6}} → RMSE {:.3}%, Max {:.1}%, >5%: {}, >10%: {}, >20%: {}",
cs.rmse_pct(),
cs.max_pct(),
cs.gt_5pct,
cs.gt_10pct,
cs.gt_20pct,
);
println!("\nTotal time: {:.1}s", t0.elapsed().as_secs_f64());
}
-286
View File
@@ -1,286 +0,0 @@
//! Validate oracle accuracy against exchange reference prices.
//!
//! Run with: cargo run -p brk_oracle --example validate --release
//!
//! Requires:
//! - ~/.brk indexed blockchain data (brk_indexer)
//! - examples/height_price_ohlc.json (per-height [open, high, low, close] in dollars)
use std::path::PathBuf;
use brk_indexer::Indexer;
use brk_oracle::{
Config, Histogram, Oracle, PRICES, START_HEIGHT, cents_to_bin, default_eligible_bin,
};
use brk_types::{OutputType, Sats, TxIndex, TxOutIndex};
use vecdb::{AnyVec, ReadableVec, VecIndex};
const BINS_5PCT: f64 = 4.24;
const BINS_10PCT: f64 = 8.28;
const BINS_20PCT: f64 = 15.84;
fn bins_to_pct(bins: f64) -> f64 {
(10.0_f64.powf(bins / 200.0) - 1.0) * 100.0
}
fn seed_bin(start_height: usize) -> f64 {
let price: f64 = PRICES
.lines()
.nth(start_height - 1)
.expect("prices.txt too short")
.parse()
.expect("Failed to parse seed price");
cents_to_bin(price * 100.0)
}
struct Stats {
total_sq_err: f64,
total_bias: f64,
max_err: f64,
total_blocks: u64,
gt_5pct: u64,
gt_10pct: u64,
gt_20pct: u64,
}
impl Stats {
fn new() -> Self {
Self {
total_sq_err: 0.0,
total_bias: 0.0,
max_err: 0.0,
total_blocks: 0,
gt_5pct: 0,
gt_10pct: 0,
gt_20pct: 0,
}
}
fn update(&mut self, err: f64) {
self.total_sq_err += err * err;
self.total_bias += err;
self.total_blocks += 1;
let abs_err = err.abs();
if abs_err > self.max_err {
self.max_err = abs_err;
}
if abs_err > BINS_5PCT {
self.gt_5pct += 1;
}
if abs_err > BINS_10PCT {
self.gt_10pct += 1;
}
if abs_err > BINS_20PCT {
self.gt_20pct += 1;
}
}
fn rmse_pct(&self) -> f64 {
bins_to_pct((self.total_sq_err / self.total_blocks as f64).sqrt())
}
fn max_pct(&self) -> f64 {
bins_to_pct(self.max_err)
}
fn bias(&self) -> f64 {
self.total_bias / self.total_blocks as f64
}
}
struct Run {
label: &'static str,
start_height: usize,
oracle: Option<Oracle>,
stats: Stats,
}
fn main() {
let data_dir = std::env::var("BRK_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| {
let home = std::env::var("HOME").unwrap();
PathBuf::from(home).join(".brk")
});
let indexer = Indexer::forced_import(&data_dir).expect("Failed to load indexer");
let total_heights = indexer.vecs.blocks.timestamp.len();
let manifest_dir = env!("CARGO_MANIFEST_DIR");
let height_ohlc: Vec<[f64; 4]> = serde_json::from_str(
&std::fs::read_to_string(format!("{manifest_dir}/examples/height_price_ohlc.json"))
.expect("Failed to read height_price_ohlc.json"),
)
.expect("Failed to parse height OHLC");
// Pre-compute per-height (high_bin, low_bin) tolerance band.
let height_bands: Vec<(f64, f64)> = height_ohlc
.iter()
.map(|ohlc| {
let high = ohlc[1];
let low = ohlc[2];
if high > 0.0 && low > 0.0 {
(cents_to_bin(high * 100.0), cents_to_bin(low * 100.0))
} else {
(0.0, 0.0)
}
})
.collect();
let mut runs = vec![
Run {
label: "w12 @ 575k",
start_height: 575_000,
oracle: None,
stats: Stats::new(),
},
Run {
label: "w12 @ 600k",
start_height: 600_000,
oracle: None,
stats: Stats::new(),
},
Run {
label: "w12 @ 630k",
start_height: 630_000,
oracle: None,
stats: Stats::new(),
},
];
// Build per-block filtered histograms from the indexer, feeding all oracles in one pass.
let total_txs = indexer.vecs.transactions.txid.len();
let total_outputs = indexer.vecs.outputs.value.len();
// Pre-collect height-indexed vecs (small). Transaction-indexed vecs are too large.
let first_tx_index: Vec<TxIndex> = indexer.vecs.transactions.first_tx_index.collect();
let out_first: Vec<TxOutIndex> = indexer.vecs.outputs.first_txout_index.collect();
for h in START_HEIGHT..total_heights {
let ft = first_tx_index[h];
let next_ft = first_tx_index
.get(h + 1)
.copied()
.unwrap_or(TxIndex::from(total_txs));
let out_start = if ft.to_usize() + 1 < next_ft.to_usize() {
indexer
.vecs
.transactions
.first_txout_index
.collect_one(ft + 1)
.unwrap()
.to_usize()
} else {
out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize()
};
let out_end = out_first
.get(h + 1)
.copied()
.unwrap_or(TxOutIndex::from(total_outputs))
.to_usize();
// Build filtered histogram once for all oracles.
let values: Vec<Sats> = indexer
.vecs
.outputs
.value
.collect_range_at(out_start, out_end);
let output_types: Vec<OutputType> = indexer
.vecs
.outputs
.output_type
.collect_range_at(out_start, out_end);
let mut hist = Histogram::zeros();
for (sats, output_type) in values.into_iter().zip(output_types) {
if let Some(bin) = default_eligible_bin(sats, output_type) {
hist.increment(bin as usize);
}
}
for run in &mut runs {
if h < run.start_height {
continue;
}
if run.oracle.is_none() {
let config = Config::default();
run.oracle = Some(Oracle::new(seed_bin(run.start_height), config));
}
let ref_bin = run.oracle.as_mut().unwrap().process_histogram(&hist);
if h < height_bands.len() {
let (high_bin, low_bin) = height_bands[h];
if high_bin > 0.0 && low_bin > 0.0 {
let err = if ref_bin < high_bin {
ref_bin - high_bin
} else if ref_bin > low_bin {
ref_bin - low_bin
} else {
0.0
};
run.stats.update(err);
}
}
}
}
// Print results.
println!();
println!(
"{:<14} {:>8} {:>10} {:>10} {:>6} {:>6} {:>6} {:>8}",
"Config", "Blocks", "RMSE%", "Max%", ">5%", ">10%", ">20%", "Bias"
);
println!("{}", "-".repeat(72));
for run in &runs {
let s = &run.stats;
println!(
"{:<14} {:>8} {:>7.2}% {:>7.1}% {:>6} {:>6} {:>6} {:>+8.2}",
run.label,
s.total_blocks,
s.rmse_pct(),
s.max_pct(),
s.gt_5pct,
s.gt_10pct,
s.gt_20pct,
s.bias()
);
}
println!();
// Verify exact counts against reference.
// Reference: trunc w12 @ 575k: 261 >5%, 40 >10%, 0 >20%
// trunc w12 @ 600k: 174 >5%, 31 >10%, 0 >20%
// trunc w12 @ 630k: 84 >5%, 9 >10%, 0 >20%
let expected: &[(&str, u64, u64, u64)] = &[
("w12 @ 575k", 237, 22, 0),
("w12 @ 600k", 152, 15, 0),
("w12 @ 630k", 84, 9, 0),
];
for (run, &(label, exp_5, exp_10, exp_20)) in runs.iter().zip(expected) {
let s = &run.stats;
assert_eq!(
s.gt_20pct, exp_20,
"{label}: expected {exp_20} blocks >20%, got {}",
s.gt_20pct
);
assert_eq!(
s.gt_10pct, exp_10,
"{label}: expected {exp_10} blocks >10%, got {}",
s.gt_10pct
);
assert_eq!(
s.gt_5pct, exp_5,
"{label}: expected {exp_5} blocks >5%, got {}",
s.gt_5pct
);
}
println!("All assertions passed!");
}
+88 -17
View File
@@ -1,13 +1,17 @@
use brk_types::OutputType;
use std::ops::Range;
/// Dust floor used by `Config::default()` and `default_eligible_bin`.
pub(crate) const DEFAULT_MIN_SATS: u64 = 1000;
/// First height the oracle computes on-chain, with the slow cold-start EMA
/// ([`slow`](Config::slow)). Below it, prices come from
/// [`pre_oracle_prices_from`](crate::pre_oracle_prices_from).
pub const START_HEIGHT_SLOW: usize = 340_000;
/// Output types skipped by `Config::default()` (protocol-dominated) and the
/// source of truth for `default_eligible_bin`'s precomputed exclusion mask.
pub(crate) const DEFAULT_EXCLUDED_OUTPUT_TYPES: &[OutputType] = &[OutputType::P2TR];
/// Height where the oracle switches slow -> fast EMA ([`default`](Config::default)).
/// The regimes are complementary: slow resists the round-USD half-price drift
/// that locks fast below here, while fast tracks the 2018-2019 crashes that lock
/// slow.
pub const START_HEIGHT_FAST: usize = 508_000;
#[derive(Clone)]
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Config {
/// EMA decay: 2/(N+1) where N is span in blocks. 2/7 = 6-block span.
pub alpha: f64,
@@ -16,12 +20,12 @@ pub struct Config {
/// Search window bins below/above previous estimate. Asymmetric for log-scale.
pub search_below: usize,
pub search_above: usize,
/// Minimum output value in sats (dust filter).
pub min_sats: u64,
/// Exclude round BTC amounts that create false stencil matches.
pub exclude_common_round_values: bool,
/// Output types to ignore (e.g. P2TR, P2WSH are noisy).
pub excluded_output_types: Vec<OutputType>,
/// Weight of the adaptive shape-anchoring restoring force added to the
/// stencil score. `0.0` disables it (mature regime, where the fast EMA
/// tracks real moves the shape term would resist). The slow cold-start uses
/// a positive weight to resist round-USD octave aliasing in the thin early
/// output mix.
pub shape_weight: f64,
}
impl Default for Config {
@@ -29,11 +33,78 @@ impl Default for Config {
Self {
alpha: 2.0 / 7.0,
window_size: 12,
search_below: 9,
search_below: 12,
search_above: 11,
min_sats: DEFAULT_MIN_SATS,
exclude_common_round_values: true,
excluded_output_types: DEFAULT_EXCLUDED_OUTPUT_TYPES.to_vec(),
shape_weight: 0.0,
}
}
}
impl Config {
/// Cold-start config below [`START_HEIGHT_FAST`]: a slow EMA
/// (span ~19) that resists the round-USD half-price drift the fast default
/// octave-locks onto in the thin pre-2018 output mix. Window grows to 40 to
/// hold the decay, and a shape-anchoring restoring force (`shape_weight`)
/// pulls the pick toward the octave whose arm-shape looks like real payments.
pub fn slow() -> Self {
Self {
alpha: 0.10,
window_size: 40,
shape_weight: 8.0,
..Self::default()
}
}
/// Config for `height`: [`slow`](Self::slow) below [`START_HEIGHT_FAST`], else
/// [`default`](Self::default).
pub fn for_height(height: usize) -> Self {
if height < START_HEIGHT_FAST {
Self::slow()
} else {
Self::default()
}
}
/// Split a block range into sub-ranges with a single EMA configuration.
pub fn segments_for_range(range: Range<usize>) -> impl Iterator<Item = Range<usize>> {
let split = START_HEIGHT_FAST.max(range.start).min(range.end);
[range.start..split, split..range.end]
.into_iter()
.filter(|range| !range.is_empty())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn segments_for_range_splits_at_fast_start() {
let segments: Vec<_> =
Config::segments_for_range((START_HEIGHT_FAST - 2)..(START_HEIGHT_FAST + 2)).collect();
assert_eq!(
segments,
vec![
(START_HEIGHT_FAST - 2)..START_HEIGHT_FAST,
START_HEIGHT_FAST..(START_HEIGHT_FAST + 2),
]
);
}
#[test]
fn segments_for_range_omits_empty_sides() {
let slow: Vec<_> =
Config::segments_for_range((START_HEIGHT_FAST - 2)..START_HEIGHT_FAST).collect();
assert_eq!(slow, vec![(START_HEIGHT_FAST - 2)..START_HEIGHT_FAST]);
let fast: Vec<_> =
Config::segments_for_range(START_HEIGHT_FAST..(START_HEIGHT_FAST + 2)).collect();
assert_eq!(fast, vec![START_HEIGHT_FAST..(START_HEIGHT_FAST + 2)]);
}
#[test]
fn for_height_selects_regime() {
assert_eq!(Config::for_height(START_HEIGHT_FAST - 1), Config::slow());
assert_eq!(Config::for_height(START_HEIGHT_FAST), Config::default());
}
}
+218
View File
@@ -0,0 +1,218 @@
use brk_types::{OutputType, Sats};
use crate::scale::{sats_to_bin, HistogramRaw};
/// Dust floor: outputs below this many sats are too small to be payments.
const MIN_SATS: u64 = 1000;
/// Output types skipped entirely (protocol-dominated, too noisy to carry the
/// round-dollar signal).
const EXCLUDED_OUTPUT_TYPES: &[OutputType] = &[OutputType::P2TR];
/// Bitmask form of [`EXCLUDED_OUTPUT_TYPES`], folded at compile time so
/// [`PaymentFilter::eligible_bin`] checks membership with a single AND.
const EXCLUDED_MASK: u16 = {
let mut mask = 0u16;
let mut i = 0;
while i < EXCLUDED_OUTPUT_TYPES.len() {
mask |= 1u16 << EXCLUDED_OUTPUT_TYPES[i] as u8;
i += 1;
}
mask
};
/// Round-dollar payment filter.
///
/// Input: transaction outputs. Output: eligible log-scale bins or a fresh block
/// histogram. The only state is the transaction-output fan-out cap selected by
/// block height, or [`MODERN`](Self::MODERN) for live modern transaction streams.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct PaymentFilter {
tx_output_fanout_cap: usize,
}
impl PaymentFilter {
/// Pre-modern transaction-output fan-out cap. Above this, the transaction is
/// a batch payout (exchange sweep, mixer fan-out), not a round-dollar
/// payment.
pub const PRE_MODERN_TX_OUTPUT_FANOUT_CAP: usize = 100;
/// Modern-chain transaction-output fan-out cap. Dense post-630k blocks can
/// carry more genuine payment outputs, but very large fan-outs can still
/// dominate one EMA slot and create a false round-dollar ladder.
pub const MODERN_TX_OUTPUT_FANOUT_CAP: usize = 250;
/// Height where [`Self::PRE_MODERN_TX_OUTPUT_FANOUT_CAP`] relaxes to
/// [`Self::MODERN_TX_OUTPUT_FANOUT_CAP`].
pub const MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT: usize = 630_000;
/// Filter for live or otherwise guaranteed-modern transaction streams.
pub const MODERN: Self = Self::with_fanout_cap(Self::MODERN_TX_OUTPUT_FANOUT_CAP);
const fn with_fanout_cap(tx_output_fanout_cap: usize) -> Self {
Self {
tx_output_fanout_cap,
}
}
/// Filter for transactions in `height`.
pub const fn for_height(height: usize) -> Self {
if height < Self::MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT {
Self::with_fanout_cap(Self::PRE_MODERN_TX_OUTPUT_FANOUT_CAP)
} else {
Self::MODERN
}
}
/// Bin index for `(sats, output_type)`, or `None` for an excluded type
/// (P2TR), dust, a round-BTC value, or an out-of-range bin. The per-output
/// half of the round-dollar payment filter.
#[inline(always)]
pub fn eligible_bin(sats: Sats, output_type: OutputType) -> Option<u16> {
if EXCLUDED_MASK & (1u16 << output_type as u8) != 0 {
return None;
}
if *sats < MIN_SATS || sats.is_common_round_value() {
return None;
}
sats_to_bin(sats).map(|b| b as u16)
}
/// Apply the transaction-level payment filter and call `emit(bin)` for each
/// eligible output, in order.
///
/// A whole transaction is dropped when it carries any OP_RETURN output (data
/// carriers, not payments) or when it has more outputs than this filter's
/// fan-out cap.
#[inline]
pub fn for_each_bin(
self,
outputs: impl ExactSizeIterator<Item = (Sats, OutputType)> + Clone,
mut emit: impl FnMut(u16),
) {
if outputs.len() > self.tx_output_fanout_cap {
return;
}
if outputs.clone().any(|(_, ty)| ty == OutputType::OpReturn) {
return;
}
for (sats, ty) in outputs {
if let Some(bin) = Self::eligible_bin(sats, ty) {
emit(bin);
}
}
}
/// Build a fresh eligible round-dollar payment histogram for one block's
/// non-coinbase transaction outputs.
#[inline]
pub fn histogram<Outputs>(self, txs: impl IntoIterator<Item = Outputs>) -> HistogramRaw
where
Outputs: ExactSizeIterator<Item = (Sats, OutputType)> + Clone,
{
let mut hist = HistogramRaw::zeros();
for outputs in txs {
self.for_each_bin(outputs, |bin| hist.increment(bin as usize));
}
hist
}
}
#[cfg(test)]
mod tests {
use super::*;
fn payment_outputs(len: usize) -> impl ExactSizeIterator<Item = (Sats, OutputType)> + Clone {
std::iter::repeat_n((Sats::new(12_345), OutputType::P2WPKH), len)
}
fn emitted_count(height: usize, len: usize) -> usize {
let mut count = 0;
PaymentFilter::for_height(height).for_each_bin(payment_outputs(len), |_| count += 1);
count
}
#[test]
fn early_fanout_cap_is_strict() {
assert_eq!(
emitted_count(
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT - 1,
PaymentFilter::PRE_MODERN_TX_OUTPUT_FANOUT_CAP,
),
PaymentFilter::PRE_MODERN_TX_OUTPUT_FANOUT_CAP
);
assert_eq!(
emitted_count(
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT - 1,
PaymentFilter::PRE_MODERN_TX_OUTPUT_FANOUT_CAP + 1,
),
0
);
}
#[test]
fn modern_fanout_cap_is_relaxed_but_not_lifted() {
assert_eq!(
emitted_count(
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT,
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP,
),
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP
);
assert_eq!(
emitted_count(
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP_START_HEIGHT,
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP + 1,
),
0
);
}
fn emitted_count_modern(len: usize) -> usize {
let mut count = 0;
PaymentFilter::MODERN.for_each_bin(payment_outputs(len), |_| count += 1);
count
}
#[test]
fn modern_helper_uses_modern_fanout_cap() {
assert_eq!(
emitted_count_modern(PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP),
PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP
);
assert_eq!(
emitted_count_modern(PaymentFilter::MODERN_TX_OUTPUT_FANOUT_CAP + 1),
0
);
}
#[test]
fn payment_histogram_drops_op_return_transaction() {
let sats = Sats::new(12_345);
let txs = vec![
vec![(sats, OutputType::P2WPKH), (sats, OutputType::P2PKH)],
vec![
(Sats::new(54_321), OutputType::OpReturn),
(sats, OutputType::P2WPKH),
],
];
let hist = PaymentFilter::MODERN.histogram(txs.into_iter().map(|tx| tx.into_iter()));
let bin = PaymentFilter::eligible_bin(sats, OutputType::P2WPKH).unwrap() as usize;
assert_eq!(hist[bin], 2);
}
#[test]
fn builds_fresh_payment_histogram() {
let sats = Sats::new(12_345);
let txs = vec![vec![
(sats, OutputType::P2WPKH),
(Sats::new(100_000_000), OutputType::P2WPKH),
]];
let hist = PaymentFilter::MODERN.histogram(txs.into_iter().map(|tx| tx.into_iter()));
let bin = PaymentFilter::eligible_bin(sats, OutputType::P2WPKH).unwrap() as usize;
assert_eq!(hist[bin], 1);
}
}

Some files were not shown because too many files have changed in this diff Show More