Compare commits

...

131 Commits

Author SHA1 Message Date
nym21 4daabcee2c release: v0.3.0-beta.10 2026-05-17 22:20:30 +02:00
nym21 a6021b26cc docs: update generated docs 2026-05-17 22:19:58 +02:00
nym21 1a706da13c deps: bumped 2026-05-17 22:14:05 +02:00
nym21 20c4a113c9 oracle: v2 2026-05-17 22:13:03 +02:00
nym21 e5819769e8 website: snap 2026-05-17 22:12:44 +02:00
nym21 421e5286ce global: snap 2026-05-15 23:53:55 +02:00
nym21 68db22b9e8 mempool: polish/cleanup 2026-05-14 23:29:10 +02:00
nym21 90aca2e048 mmpl: new, mempool + rpc: fixes 2026-05-14 13:59:15 +02:00
nym21 528c134f26 mempool: fixes 2026-05-13 18:36:02 +02:00
nym21 5cc3fbfa6e crates: snapshot 2026-05-12 22:33:09 +02:00
nym21 8fc2e71492 website: snap 2026-05-12 22:32:53 +02:00
nym21 445c60a6f1 mempool: fixes 2026-05-10 19:40:02 +02:00
nym21 dd6eca138b mempool: fixes 2026-05-10 16:23:06 +02:00
nym21 774580ee11 mempool: fixes 2026-05-10 14:04:08 +02:00
nym21 fe5f30bca6 mempool: snap 2026-05-10 00:24:02 +02:00
nym21 c52a076bfc mempool: fix 2026-05-09 13:36:06 +02:00
nym21 e62b0ac2a5 global: next block template (+ diff) 2026-05-09 12:56:11 +02:00
nym21 3f2b5d3084 mempool: cleanup 2026-05-08 12:26:01 +02:00
nym21 aab16f8832 clients: bump versions 2026-05-08 12:14:34 +02:00
nym21 a9c0a09191 release: v0.3.0-beta.9 2026-05-08 11:56:58 +02:00
nym21 948a7cdd88 docs: update generated docs 2026-05-08 11:56:30 +02:00
nym21 25b2268563 mempool: fixes 2026-05-08 11:51:44 +02:00
nym21 dd88996f7f release: v0.3.0-beta.8 2026-05-08 00:36:19 +02:00
nym21 1643cf86ed docs: update generated docs 2026-05-08 00:35:52 +02:00
nym21 6e8be1af22 bindgen: js fix name of txid[] field name 2026-05-08 00:32:46 +02:00
nym21 9d18e2db9b mempool: fixes 2026-05-08 00:16:37 +02:00
nym21 d2b8992932 mempool: cleanups 2026-05-07 18:56:44 +02:00
nym21 f4910efd7d mempool: use bitcoin projected block, rest is a very simple prediction 2026-05-07 18:30:26 +02:00
nym21 1b39d21bbe blk: fix 2026-05-07 14:12:50 +02:00
nym21 cc9ebfaf42 global + blk 2026-05-07 14:02:53 +02:00
nym21 9347b42c9a mempool: fixes 2026-05-07 11:21:37 +02:00
nym21 cb74087f27 mempool: fixes 2026-05-06 19:31:18 +02:00
nym21 086bfd9938 global: adding support for safe lengths 2026-05-06 15:33:07 +02:00
nym21 da7671744f deps: bumped 2026-05-05 23:22:10 +02:00
nym21 abcb238022 global: sigops 2026-05-04 19:06:41 +02:00
nym21 dc32bd480f global: fixes 2026-05-04 16:57:21 +02:00
nym21 4663d13194 global: fixes 2026-05-03 12:44:18 +02:00
nym21 9cb5f2c880 global: fixes 2026-05-03 00:57:22 +02:00
nym21 2b8a0a8cf7 global: fixes 2026-05-03 00:42:16 +02:00
nym21 6f879a5551 global: fixes 2026-05-01 19:14:15 +02:00
nym21 1068ad4e8f query: fixes 2026-04-30 19:19:09 +02:00
nym21 9b42b40a36 mempool: fixes 2026-04-30 12:38:34 +02:00
nym21 43f3be4924 global: fixes 2026-04-29 16:51:01 +02:00
nym21 a7e41df1c6 clients: add .len() 2026-04-29 12:06:22 +02:00
nym21 f1749472e7 mempool: general improvements 2026-04-28 18:46:37 +02:00
nym21 66494c081c global: fixes 2026-04-27 19:19:14 +02:00
nym21 6c8afc942c release: v0.3.0-beta.7 2026-04-27 12:57:19 +02:00
nym21 1dcbbd801b docs: update generated docs 2026-04-27 12:56:52 +02:00
nym21 76869ed2b6 global: fixes 2026-04-27 12:52:02 +02:00
nym21 b24bfdc15c global: fixes 2026-04-27 11:19:05 +02:00
nym21 e543e4a5db release: v0.3.0-beta.6 2026-04-27 10:41:26 +02:00
nym21 9b639ef7d1 docs: update generated docs 2026-04-27 10:40:58 +02:00
nym21 07bc2d42b8 server: reorg 2026-04-27 10:35:18 +02:00
nym21 7a0b4b5890 global: big snapshot 2026-04-26 23:12:17 +02:00
nym21 2210443e37 website: lightweight-charts bump 2026-04-25 23:52:22 +02:00
nym21 8bf6570843 clients: bump versions 2026-04-24 10:33:08 +02:00
nym21 26a3b0f5e8 release: v0.3.0-beta.5 2026-04-23 23:19:37 +02:00
nym21 741c957f31 docs: update generated docs 2026-04-23 23:19:11 +02:00
nym21 e4496742a4 global: reused + mempool + favicon 2026-04-23 23:13:39 +02:00
nym21 ce00de5da8 version: bump 2026-04-23 10:20:57 +02:00
nym21 f5c50e69fc release: v0.3.0-beta.4 2026-04-22 22:30:09 +02:00
nym21 9709c2040d docs: update generated docs 2026-04-22 22:29:43 +02:00
nym21 3faa989691 global: cost basis -> urpd 2026-04-22 22:23:52 +02:00
nym21 84e924b77e global: refactor 2026-04-22 22:23:39 +02:00
nym21 c5b16e7048 release: v0.3.0-beta.3 2026-04-22 15:56:15 +02:00
nym21 c1335cec31 docs: update generated docs 2026-04-22 15:55:53 +02:00
nym21 bdc3ba1df6 mempool: snap 2026-04-22 15:30:08 +02:00
nym21 6afce0bbdc mempool: fixes 2026-04-21 12:43:50 +02:00
nym21 327873d010 website: fixes 2026-04-20 18:11:30 +02:00
nym21 08175009d2 website: snap 2026-04-19 20:28:12 +02:00
nym21 a5d3be465e website: snap 2026-04-19 17:13:47 +02:00
nym21 fd2b93367d global: snap 2026-04-18 17:23:12 +02:00
nym21 2a93f51e81 global: snap 2026-04-17 21:23:11 +02:00
nym21 008143ff00 mempool: fix pending tx info 2026-04-16 23:35:55 +02:00
nym21 d340855c8b global: snap 2026-04-16 22:17:41 +02:00
nym21 78d6d9d6f1 changelog: updated 2026-04-16 10:15:23 +02:00
nym21 5cc85b0619 release: v0.3.0-beta.2 2026-04-15 19:49:13 +02:00
nym21 7433ce0d0e docs: update generated docs 2026-04-15 19:48:50 +02:00
nym21 75a97b4da9 client: js: add cache support to text fetches 2026-04-15 19:19:59 +02:00
nym21 c23e0f2a3c global: snap 2026-04-15 13:04:22 +02:00
nym21 08ba4ad996 global: snap 2026-04-15 12:51:30 +02:00
nym21 39da441d14 global: snapshot 2026-04-14 22:53:10 +02:00
nym21 904ec93668 deps: bumped 2026-04-14 01:42:24 +02:00
nym21 4cd8d9eb56 reader: snap 2026-04-14 01:37:04 +02:00
nym21 283baca848 global: big snapshot part 2 2026-04-13 22:47:08 +02:00
nym21 765261648d global: big snapshot 2026-04-13 22:46:56 +02:00
nym21 c3cef71aa3 global: snap 2026-04-12 18:00:02 +02:00
nym21 18d9c166d8 computer: snap 2026-04-11 22:11:48 +02:00
nym21 286256ebf0 global: veccached change 2026-04-10 11:30:29 +02:00
nym21 12aae503c9 global: snapshot pre cached change 2026-04-10 10:27:07 +02:00
nym21 95e5168244 deps: bumped 2026-04-09 15:08:06 +02:00
nym21 5fd9fff9cf global: speed improvement part4 2026-04-09 15:06:19 +02:00
nym21 db5b3887f9 global: speed improvement part3 2026-04-09 14:58:25 +02:00
nym21 5a3e1b4e6e global: speed improvement part2 2026-04-09 14:02:26 +02:00
nym21 21a0226a19 global: speed improvement 2026-04-09 11:52:01 +02:00
nym21 c5c49f62d1 clients: bump versions 2026-04-08 22:19:28 +02:00
nym21 dac66c988d release: v0.3.0-beta.1 2026-04-08 17:12:32 +02:00
nym21 303d168681 docs: update generated docs 2026-04-08 17:12:03 +02:00
nym21 1ddb3385e2 website: css 2026-04-08 17:06:54 +02:00
nym21 eb75274dbf website: fixes 2026-04-08 13:11:03 +02:00
nym21 3a7887348c global: snapshot 2026-04-08 12:09:35 +02:00
nym21 0a4cb0601f release: v0.3.0-beta.0 2026-04-08 01:46:43 +02:00
nym21 861e29277c docs: update generated docs 2026-04-08 01:46:13 +02:00
nym21 c76b149ef9 website: fixes 2026-04-08 01:43:58 +02:00
nym21 4c4c6fc840 global: snapshot 2026-04-08 01:38:03 +02:00
nym21 0c14dfe924 query: more optimizations 2026-04-07 17:57:57 +02:00
nym21 17e531b4ee query: more optimizations 2026-04-07 17:43:11 +02:00
nym21 f022f62cce global: snap 2026-04-07 13:49:02 +02:00
nym21 e91f1386b1 website: snap 2026-04-06 22:30:02 +02:00
nym21 02f543af38 release: v0.3.0-alpha.6 2026-04-05 22:48:37 +02:00
nym21 20c96fb551 docs: update generated docs 2026-04-05 22:48:10 +02:00
nym21 acd3d6f425 server: cache fixes 2026-04-05 22:43:30 +02:00
nym21 2b15a24b6d website: add pool logos 2026-04-05 19:46:41 +02:00
nym21 7fac0bc613 global: snap 2026-04-04 20:13:03 +02:00
nym21 62f51761ee global: snap 2026-04-04 18:19:11 +02:00
nym21 5340cc288e release: v0.3.0-alpha.5 2026-04-04 13:10:48 +02:00
nym21 befe3c8fb7 docs: update generated docs 2026-04-04 13:10:28 +02:00
nym21 41ec24c81e server: ms endpoint fixes 2026-04-04 13:05:39 +02:00
nym21 42b497ff65 server: ms endpoint fixes 2026-04-04 12:16:15 +02:00
nym21 01d908a560 release: v0.3.0-alpha.4 2026-04-04 11:59:17 +02:00
nym21 42debcce80 docs: update generated docs 2026-04-04 11:58:50 +02:00
nym21 8bc993eceb global: fixes 2026-04-04 11:53:27 +02:00
nym21 366ac33e23 release: v0.3.0-alpha.3 2026-04-04 01:05:04 +02:00
nym21 b5a7023bd3 docs: update generated docs 2026-04-04 01:04:38 +02:00
nym21 883b38c77c global: snap 2026-04-04 00:59:37 +02:00
nym21 59c767a9e2 release: v0.3.0-alpha.2 2026-04-03 17:56:37 +02:00
nym21 9b5bb848f7 docs: update generated docs 2026-04-03 17:56:12 +02:00
nym21 5bf06530ce store: replace fjal reset by dir nuking 2026-04-03 17:49:46 +02:00
nym21 768e6870cb global: snap 2026-04-03 15:51:27 +02:00
nym21 79829ddd53 changelog: updated 2026-04-03 01:19:47 +02:00
nym21 78082801c6 clients: bump versions 2026-04-03 00:52:09 +02:00
1544 changed files with 64371 additions and 34466 deletions
+1
View File
@@ -39,6 +39,7 @@ flamegraph.svg
# AI
.claude/settings*
!CLAUDE.md
# Expand
expand.rs
+1
View File
@@ -0,0 +1 @@
Codex will review your output once you are done.
Generated
+260 -331
View File
File diff suppressed because it is too large Load Diff
+37 -38
View File
@@ -4,7 +4,7 @@ members = ["crates/*"]
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
package.license = "MIT"
package.edition = "2024"
package.version = "0.3.0-alpha.1"
package.version = "0.3.0-beta.10"
package.homepage = "https://bitcoinresearchkit.org"
package.repository = "https://github.com/bitcoinresearchkit/brk"
package.readme = "README.md"
@@ -36,45 +36,44 @@ inherits = "release"
debug = true
[workspace.dependencies]
aide = { version = "0.16.0-alpha.3", features = ["axum-json", "axum-query"] }
axum = { version = "0.8.8", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] }
bitcoin = { version = "0.32.8", features = ["serde"] }
bitcoincore-rpc = "0.19.0"
brk_alloc = { version = "0.3.0-alpha.1", path = "crates/brk_alloc" }
brk_bencher = { version = "0.3.0-alpha.1", path = "crates/brk_bencher" }
brk_bindgen = { version = "0.3.0-alpha.1", path = "crates/brk_bindgen" }
brk_cli = { version = "0.3.0-alpha.1", path = "crates/brk_cli" }
brk_client = { version = "0.3.0-alpha.1", path = "crates/brk_client" }
brk_cohort = { version = "0.3.0-alpha.1", path = "crates/brk_cohort" }
brk_computer = { version = "0.3.0-alpha.1", path = "crates/brk_computer" }
brk_error = { version = "0.3.0-alpha.1", path = "crates/brk_error" }
brk_fetcher = { version = "0.3.0-alpha.1", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.3.0-alpha.1", path = "crates/brk_indexer" }
brk_iterator = { version = "0.3.0-alpha.1", path = "crates/brk_iterator" }
brk_logger = { version = "0.3.0-alpha.1", path = "crates/brk_logger" }
brk_mempool = { version = "0.3.0-alpha.1", path = "crates/brk_mempool" }
brk_oracle = { version = "0.3.0-alpha.1", path = "crates/brk_oracle" }
brk_query = { version = "0.3.0-alpha.1", path = "crates/brk_query", features = ["tokio"] }
brk_reader = { version = "0.3.0-alpha.1", path = "crates/brk_reader" }
brk_rpc = { version = "0.3.0-alpha.1", path = "crates/brk_rpc" }
brk_server = { version = "0.3.0-alpha.1", path = "crates/brk_server" }
brk_store = { version = "0.3.0-alpha.1", path = "crates/brk_store" }
brk_traversable = { version = "0.3.0-alpha.1", path = "crates/brk_traversable", features = ["pco", "derive"] }
brk_traversable_derive = { version = "0.3.0-alpha.1", path = "crates/brk_traversable_derive" }
brk_types = { version = "0.3.0-alpha.1", path = "crates/brk_types" }
brk_website = { version = "0.3.0-alpha.1", path = "crates/brk_website" }
aide = { version = "0.16.0-alpha.4", features = ["axum-json", "axum-query"] }
axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] }
bitcoin = { version = "0.32.9", features = ["serde"] }
brk_alloc = { version = "0.3.0-beta.10", path = "crates/brk_alloc" }
brk_bencher = { version = "0.3.0-beta.10", path = "crates/brk_bencher" }
brk_bindgen = { version = "0.3.0-beta.10", path = "crates/brk_bindgen" }
brk_cli = { version = "0.3.0-beta.10", path = "crates/brk_cli" }
brk_client = { version = "0.3.0-beta.10", path = "crates/brk_client" }
brk_cohort = { version = "0.3.0-beta.10", path = "crates/brk_cohort" }
brk_computer = { version = "0.3.0-beta.10", path = "crates/brk_computer" }
brk_error = { version = "0.3.0-beta.10", path = "crates/brk_error" }
brk_fetcher = { version = "0.3.0-beta.10", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.3.0-beta.10", path = "crates/brk_indexer" }
brk_iterator = { version = "0.3.0-beta.10", path = "crates/brk_iterator" }
brk_logger = { version = "0.3.0-beta.10", path = "crates/brk_logger" }
brk_mempool = { version = "0.3.0-beta.10", path = "crates/brk_mempool" }
brk_oracle = { version = "0.3.0-beta.10", path = "crates/brk_oracle" }
brk_query = { version = "0.3.0-beta.10", path = "crates/brk_query", features = ["tokio"] }
brk_reader = { version = "0.3.0-beta.10", path = "crates/brk_reader" }
brk_rpc = { version = "0.3.0-beta.10", path = "crates/brk_rpc" }
brk_server = { version = "0.3.0-beta.10", path = "crates/brk_server" }
brk_store = { version = "0.3.0-beta.10", path = "crates/brk_store" }
brk_traversable = { version = "0.3.0-beta.10", path = "crates/brk_traversable", features = ["pco", "derive"] }
brk_traversable_derive = { version = "0.3.0-beta.10", path = "crates/brk_traversable_derive" }
brk_types = { version = "0.3.0-beta.10", path = "crates/brk_types" }
brk_website = { version = "0.3.0-beta.10", path = "crates/brk_website" }
byteview = "0.10.1"
color-eyre = "0.6.5"
corepc-client = { package = "brk-corepc-client", version = "0.11.0", features = ["client-sync"] }
corepc-jsonrpc = { package = "brk-corepc-jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false }
corepc-jsonrpc = { package = "jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false }
corepc-types = { version = "0.13.0", features = ["std"], default-features = false }
derive_more = { version = "2.1.1", features = ["deref", "deref_mut"] }
fjall = "3.1.2"
indexmap = { version = "2.13.0", features = ["serde"] }
jiff = { version = "0.2.23", features = ["perf-inline", "tz-system"], default-features = false }
fjall = "=3.0.4"
indexmap = { version = "2.14.0", features = ["serde"] }
jiff = { version = "0.2.24", features = ["perf-inline", "tz-system"], default-features = false }
owo-colors = "4.3.0"
parking_lot = "0.12.5"
pco = "1.0.1"
rayon = "1.11.0"
pco = "1.0.2"
rayon = "1.12.0"
rustc-hash = "2.1.2"
schemars = { version = "1.2.1", features = ["indexmap2"] }
serde = "1.0.228"
@@ -82,12 +81,12 @@ serde_bytes = "0.11.19"
serde_derive = "1.0.228"
serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] }
smallvec = "1.15.1"
tokio = { version = "1.50.0", features = ["rt-multi-thread"] }
tower-http = { version = "0.6.8", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
tokio = { version = "1.52.3", features = ["rt-multi-thread"] }
tower-http = { version = "0.6.10", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
tower-layer = "0.3"
tracing = { version = "0.1", default-features = false, features = ["std"] }
ureq = { version = "3.3.0", features = ["json"] }
vecdb = { version = "0.9.3", features = ["derive", "serde_json", "pco", "schemars"] }
vecdb = { version = "0.10.3", features = ["derive", "serde_json", "pco", "schemars"] }
# vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] }
[workspace.metadata.release]
+2
View File
@@ -0,0 +1,2 @@
*.md
!README.md
+21
View File
@@ -0,0 +1,21 @@
[package]
name = "blk"
description = "A CLI to inspect Bitcoin Core blocks"
version.workspace = true
edition.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
[dependencies]
bitcoin = { workspace = true }
brk_error = { workspace = true }
brk_reader = { workspace = true }
brk_rpc = { workspace = true }
brk_types = { workspace = true }
owo-colors = { workspace = true, features = ["supports-colors"] }
serde_json = { workspace = true }
[[bin]]
name = "blk"
path = "src/main.rs"
+27
View File
@@ -0,0 +1,27 @@
# blk
A CLI to inspect Bitcoin Core blocks.
Reads `blk*.dat` files directly via [`brk_reader`](../brk_reader) and resolves
the chain tip / heights via the Bitcoin Core RPC. Output is shell-friendly:
bare values, NDJSON, pretty JSON, or TSV.
## Install
```sh
cargo install --path crates/blk
```
## Quick start
```sh
blk 800000 hash # bare hash
blk 800000 height hash time # one compact JSON line
blk 800000 tx.0.vout.0.value # coinbase output 0 sats
blk 0..2 hash tx.0.txid # 3 NDJSON lines
blk tip tx.0 # whole coinbase tx as JSON
```
## Reference
Run `blk --help` for the full field/selector/option reference.
+132
View File
@@ -0,0 +1,132 @@
use std::{collections::HashSet, path::PathBuf};
use brk_error::{Error, Result};
use brk_rpc::{Auth, Client};
use crate::path::Path;
pub struct Args {
pub selector: String,
pub paths: Vec<Path>,
pub pretty: bool,
pub compact: bool,
bitcoindir: Option<PathBuf>,
blocksdir: Option<PathBuf>,
rpcconnect: Option<String>,
rpcport: Option<u16>,
rpccookiefile: Option<PathBuf>,
rpcuser: Option<String>,
rpcpassword: Option<String>,
}
impl Args {
pub fn parse(raw: Vec<String>) -> Result<Self> {
let mut pretty = false;
let mut compact = false;
let mut bitcoindir = None;
let mut blocksdir = None;
let mut rpcconnect = None;
let mut rpcport = None;
let mut rpccookiefile = None;
let mut rpcuser = None;
let mut rpcpassword = None;
let mut positional: Vec<String> = Vec::new();
let mut iter = raw.into_iter();
while let Some(a) = iter.next() {
if a == "-p" || a == "--pretty" {
pretty = true;
continue;
}
if a == "-c" || a == "--compact" {
compact = true;
continue;
}
if let Some(rest) = a.strip_prefix("--") {
let (key, value) = match rest.split_once('=') {
Some((k, v)) => (k.to_string(), v.to_string()),
None => (
rest.to_string(),
iter.next()
.ok_or_else(|| Error::Parse(format!("--{rest} requires a value")))?,
),
};
match key.as_str() {
"bitcoindir" => bitcoindir = Some(PathBuf::from(value)),
"blocksdir" => blocksdir = Some(PathBuf::from(value)),
"rpcconnect" => rpcconnect = Some(value),
"rpcport" => {
rpcport = Some(value.parse().map_err(|_| {
Error::Parse(format!("--rpcport: '{value}' is not a valid port"))
})?);
}
"rpccookiefile" => rpccookiefile = Some(PathBuf::from(value)),
"rpcuser" => rpcuser = Some(value),
"rpcpassword" => rpcpassword = Some(value),
other => return Err(Error::Parse(format!("unknown flag --{other}"))),
}
continue;
}
if a.starts_with('-') {
return Err(Error::Parse(format!("unknown flag {a}")));
}
positional.push(a);
}
let mut iter = positional.into_iter();
let selector = iter
.next()
.ok_or_else(|| Error::Parse("missing selector".into()))?;
let paths: Vec<Path> = iter.map(|f| Path::parse(&f)).collect::<Result<_>>()?;
let mut seen = HashSet::with_capacity(paths.len());
for p in &paths {
if !seen.insert(p.raw.as_str()) {
return Err(Error::Parse(format!("duplicate field '{}'", p.raw)));
}
}
Ok(Self {
selector,
paths,
pretty,
compact,
bitcoindir,
blocksdir,
rpcconnect,
rpcport,
rpccookiefile,
rpcuser,
rpcpassword,
})
}
pub fn bitcoin_dir(&self) -> PathBuf {
self.bitcoindir
.clone()
.unwrap_or_else(Client::default_bitcoin_path)
}
pub fn blocks_dir(&self) -> PathBuf {
self.blocksdir
.clone()
.unwrap_or_else(|| self.bitcoin_dir().join("blocks"))
}
pub fn rpc(&self) -> Result<Client> {
let host = self.rpcconnect.as_deref().unwrap_or("localhost");
let port = self.rpcport.unwrap_or(8332);
let url = format!("http://{host}:{port}");
let cookie = self
.rpccookiefile
.clone()
.unwrap_or_else(|| self.bitcoin_dir().join(".cookie"));
let auth = if cookie.is_file() {
Auth::CookieFile(cookie)
} else if let (Some(u), Some(p)) = (self.rpcuser.as_deref(), self.rpcpassword.as_deref()) {
Auth::UserPass(u.to_string(), p.to_string())
} else {
return Err(Error::Parse(
"no RPC auth: cookie file missing and --rpcuser/--rpcpassword not set".into(),
));
};
Client::new(&url, auth)
}
}
+407
View File
@@ -0,0 +1,407 @@
use std::cell::OnceCell;
use bitcoin::{
Address, Block, Network, ScriptBuf, Transaction, TxIn, TxOut, consensus::encode::serialize_hex,
hex::DisplayHex,
};
use brk_error::{Error, Result};
use brk_types::ReadBlock;
use serde_json::{Map, Value, json};
use crate::path::{Path, Step};
// `hex` is intentionally absent: matches `bitcoin-cli getblock <hash> 2`
// and keeps NDJSON dumps tractable. Still reachable explicitly via `blk N hex`.
const BLOCK_FIELDS: &[&str] = &[
"height",
"hash",
"version",
"version_hex",
"merkle",
"time",
"nonce",
"bits",
"difficulty",
"prev",
"txs",
"n_inputs",
"n_outputs",
"witness_txs",
"size",
"strippedsize",
"weight",
"subsidy",
"coinbase",
"coinbase_hex",
"header_hex",
"tx",
];
const TX_FIELDS: &[&str] = &[
"txid",
"wtxid",
"version",
"locktime",
"size",
"base_size",
"vsize",
"weight",
"inputs",
"outputs",
"is_coinbase",
"has_witness",
"is_rbf",
"total_out",
"hex",
"vin",
"vout",
];
const VIN_FIELDS: &[&str] = &[
"prev_txid",
"prev_vout",
"sequence",
"script_sig",
"script_sig_asm",
"witness",
"has_witness",
"is_rbf",
"coinbase",
];
const VOUT_FIELDS: &[&str] = &[
"value",
"script_pubkey",
"script_pubkey_asm",
"type",
"address",
];
pub struct Ctx<'a> {
block: &'a ReadBlock,
network: Network,
size_weight: OnceCell<(usize, usize)>,
}
impl<'a> Ctx<'a> {
pub fn new(block: &'a ReadBlock, network: Network) -> Self {
Self {
block,
network,
size_weight: OnceCell::new(),
}
}
pub fn resolve(&self, path: &Path) -> Result<Value> {
let (step, rest) = pop(&path.steps)?;
self.block_field(&step.name, step.index, rest)
}
pub fn resolve_str(&self, path: &Path) -> Result<String> {
Ok(match self.resolve(path)? {
Value::String(s) => s,
other => other.to_string(),
})
}
pub fn full(&self) -> Value {
let mut obj = Map::with_capacity(BLOCK_FIELDS.len());
for &name in BLOCK_FIELDS {
obj.insert(
name.into(),
self.block_field(name, None, &[]).expect("known block field"),
);
}
Value::Object(obj)
}
fn size_and_weight(&self) -> (usize, usize) {
*self
.size_weight
.get_or_init(|| self.block.total_size_and_weight())
}
fn block_field(&self, name: &str, index: Option<usize>, rest: &[Step]) -> Result<Value> {
let b = self.block;
let raw: &Block = b;
let scalar = |v| scalar_leaf(v, name, index, rest);
match name {
"height" => scalar(json!(*b.height())),
"hash" => scalar(json!(b.hash().to_string())),
"time" => scalar(json!(b.header.time)),
"version" => scalar(json!(b.header.version.to_consensus())),
"version_hex" => scalar(json!(format!(
"{:08x}",
b.header.version.to_consensus() as u32
))),
"bits" => scalar(json!(format!("{:08x}", b.header.bits.to_consensus()))),
"nonce" => scalar(json!(b.header.nonce)),
"prev" => scalar(json!(b.header.prev_blockhash.to_string())),
"merkle" => scalar(json!(b.header.merkle_root.to_string())),
"difficulty" => scalar(json!(b.header.difficulty_float())),
"txs" => scalar(json!(b.txdata.len())),
"n_inputs" => scalar(json!(
b.txdata.iter().map(|tx| tx.input.len()).sum::<usize>()
)),
"n_outputs" => scalar(json!(
b.txdata.iter().map(|tx| tx.output.len()).sum::<usize>()
)),
"witness_txs" => scalar(json!(
b.txdata.iter().filter(|tx| tx_has_witness(tx)).count()
)),
"size" => scalar(json!(self.size_and_weight().0)),
"weight" => scalar(json!(self.size_and_weight().1)),
"strippedsize" => {
let (size, weight) = self.size_and_weight();
scalar(json!((weight - size) / 3))
}
"subsidy" => scalar(json!(subsidy_sats(*b.height()))),
"header_hex" => scalar(json!(serialize_hex(&b.header))),
"hex" => scalar(json!(serialize_hex(raw))),
"coinbase" => scalar(json!(b.coinbase_tag().as_str())),
"coinbase_hex" => {
debug_assert!(
!b.txdata.is_empty() && !b.txdata[0].input.is_empty(),
"consensus-valid block has a coinbase tx with at least one input"
);
scalar(json!(b.txdata[0].input[0].script_sig.to_hex_string()))
}
"tx" => pick(&b.txdata, name, index, |i, tx| {
self.resolve_tx(tx, i == 0, rest)
}),
other => Err(unknown("block", other)),
}
}
fn resolve_tx(&self, tx: &Transaction, is_coinbase: bool, steps: &[Step]) -> Result<Value> {
if steps.is_empty() {
let mut obj = Map::with_capacity(TX_FIELDS.len());
for &name in TX_FIELDS {
obj.insert(
name.into(),
self.tx_field(tx, is_coinbase, name, None, &[])
.expect("known tx field"),
);
}
return Ok(Value::Object(obj));
}
let (step, rest) = pop(steps)?;
self.tx_field(tx, is_coinbase, &step.name, step.index, rest)
}
fn tx_field(
&self,
tx: &Transaction,
is_coinbase: bool,
name: &str,
index: Option<usize>,
rest: &[Step],
) -> Result<Value> {
let scalar = |v| scalar_leaf(v, name, index, rest);
match name {
"txid" => scalar(json!(tx.compute_txid().to_string())),
"wtxid" => scalar(json!(tx.compute_wtxid().to_string())),
"version" => scalar(json!(tx.version.0)),
"locktime" => scalar(json!(tx.lock_time.to_consensus_u32())),
"size" => scalar(json!(tx.total_size())),
"base_size" => scalar(json!(tx.base_size())),
"vsize" => scalar(json!(tx.vsize())),
"weight" => scalar(json!(tx.weight().to_wu())),
"inputs" => scalar(json!(tx.input.len())),
"outputs" => scalar(json!(tx.output.len())),
"is_coinbase" => scalar(json!(is_coinbase)),
"has_witness" => scalar(json!(tx_has_witness(tx))),
"is_rbf" => scalar(json!(tx_is_rbf(tx))),
"total_out" => scalar(json!(tx_total_out(tx))),
"hex" => scalar(json!(serialize_hex(tx))),
"vin" => pick(&tx.input, name, index, |j, vin| {
resolve_vin(vin, is_coinbase && j == 0, rest)
}),
"vout" => pick(&tx.output, name, index, |_, vout| {
self.resolve_vout(vout, rest)
}),
other => Err(unknown("tx", other)),
}
}
fn resolve_vout(&self, vout: &TxOut, steps: &[Step]) -> Result<Value> {
if steps.is_empty() {
let mut obj = Map::with_capacity(VOUT_FIELDS.len());
for &name in VOUT_FIELDS {
obj.insert(
name.into(),
self.vout_field(vout, name, None, &[])
.expect("known vout field"),
);
}
return Ok(Value::Object(obj));
}
let (step, rest) = pop(steps)?;
self.vout_field(vout, &step.name, step.index, rest)
}
fn vout_field(
&self,
vout: &TxOut,
name: &str,
index: Option<usize>,
rest: &[Step],
) -> Result<Value> {
let scalar = |v| scalar_leaf(v, name, index, rest);
match name {
"value" => scalar(json!(vout.value.to_sat())),
"script_pubkey" => scalar(json!(vout.script_pubkey.to_hex_string())),
"script_pubkey_asm" => scalar(json!(vout.script_pubkey.to_asm_string())),
"type" => scalar(json!(script_type(&vout.script_pubkey))),
"address" => scalar(self.address_value(&vout.script_pubkey)),
other => Err(unknown("vout", other)),
}
}
fn address_value(&self, s: &ScriptBuf) -> Value {
Address::from_script(s, self.network)
.map(|a| Value::String(a.to_string()))
.unwrap_or(Value::Null)
}
}
fn resolve_vin(vin: &TxIn, is_coinbase: bool, steps: &[Step]) -> Result<Value> {
if steps.is_empty() {
let mut obj = Map::with_capacity(VIN_FIELDS.len());
for &name in VIN_FIELDS {
obj.insert(
name.into(),
vin_field(vin, is_coinbase, name, None, &[]).expect("known vin field"),
);
}
return Ok(Value::Object(obj));
}
let (step, rest) = pop(steps)?;
vin_field(vin, is_coinbase, &step.name, step.index, rest)
}
fn vin_field(
vin: &TxIn,
is_coinbase: bool,
name: &str,
index: Option<usize>,
rest: &[Step],
) -> Result<Value> {
let scalar = |v| scalar_leaf(v, name, index, rest);
match name {
"prev_txid" => scalar(json!(vin.previous_output.txid.to_string())),
"prev_vout" => scalar(json!(vin.previous_output.vout)),
"sequence" => scalar(json!(vin.sequence.0)),
"script_sig" => scalar(json!(vin.script_sig.to_hex_string())),
"script_sig_asm" => scalar(json!(vin.script_sig.to_asm_string())),
"witness" => {
if !rest.is_empty() {
return Err(Error::Parse(
"'witness' element has no fields to drill into".into(),
));
}
let items: Vec<String> = vin
.witness
.iter()
.map(|w| w.to_lower_hex_string())
.collect();
pick(&items, name, index, |_, hex| Ok(Value::String(hex.clone())))
}
"has_witness" => scalar(json!(!vin.witness.is_empty())),
"is_rbf" => scalar(json!(vin.sequence.is_rbf())),
"coinbase" => scalar(json!(is_coinbase)),
other => Err(unknown("vin", other)),
}
}
fn pick<T>(
items: &[T],
name: &str,
index: Option<usize>,
mut resolve: impl FnMut(usize, &T) -> Result<Value>,
) -> Result<Value> {
match index {
Some(i) => {
let item = items
.get(i)
.ok_or_else(|| out_of_range(name, i, items.len()))?;
resolve(i, item)
}
None => Ok(Value::Array(
items
.iter()
.enumerate()
.map(|(i, item)| resolve(i, item))
.collect::<Result<_>>()?,
)),
}
}
fn pop(steps: &[Step]) -> Result<(&Step, &[Step])> {
steps
.split_first()
.ok_or_else(|| Error::Parse("empty path segment".into()))
}
fn scalar_leaf(v: Value, name: &str, index: Option<usize>, rest: &[Step]) -> Result<Value> {
if index.is_some() {
return Err(Error::Parse(format!("'{name}' is not an array")));
}
if !rest.is_empty() {
return Err(Error::Parse(format!(
"'{name}' has no fields to drill into"
)));
}
Ok(v)
}
fn out_of_range(name: &str, i: usize, len: usize) -> Error {
Error::Parse(format!("{name}.{i} out of range (len {len})"))
}
fn unknown(level: &str, name: &str) -> Error {
Error::Parse(format!(
"unknown {level} field '{name}' (run `blk --help` for the list)"
))
}
fn tx_has_witness(tx: &Transaction) -> bool {
tx.input.iter().any(|i| !i.witness.is_empty())
}
fn tx_is_rbf(tx: &Transaction) -> bool {
tx.input.iter().any(|i| i.sequence.is_rbf())
}
fn tx_total_out(tx: &Transaction) -> u64 {
tx.output.iter().map(|o| o.value.to_sat()).sum()
}
fn subsidy_sats(height: u32) -> u64 {
let halvings = height / 210_000;
if halvings >= 64 {
0
} else {
(50 * 100_000_000u64) >> halvings
}
}
fn script_type(s: &ScriptBuf) -> &'static str {
if s.is_p2pkh() {
"p2pkh"
} else if s.is_p2sh() {
"p2sh"
} else if s.is_p2wpkh() {
"p2wpkh"
} else if s.is_p2wsh() {
"p2wsh"
} else if s.is_p2tr() {
"p2tr"
} else if s.is_op_return() {
"op_return"
} else if s.is_p2pk() {
"p2pk"
} else {
"unknown"
}
}
+61
View File
@@ -0,0 +1,61 @@
use brk_error::Result;
use serde_json::{Map, Value};
use crate::{fields::Ctx, mode::Mode, path::Path};
pub struct Formatter {
mode: Mode,
fields: Vec<Path>,
}
impl Formatter {
pub fn new(mode: Mode, fields: Vec<Path>) -> Self {
Self { mode, fields }
}
pub fn format(&self, ctx: &Ctx) -> Result<String> {
match self.mode {
Mode::Bare => self.bare(ctx, false),
Mode::Tsv => self.tsv(ctx),
Mode::Json => Ok(serde_json::to_string(&self.object(ctx)?)?),
Mode::Pretty if self.fields.len() == 1 => self.bare(ctx, true),
Mode::Pretty => Ok(serde_json::to_string_pretty(&self.object(ctx)?)?),
}
}
fn bare(&self, ctx: &Ctx, pretty: bool) -> Result<String> {
Ok(match ctx.resolve(&self.fields[0])? {
Value::String(s) => s,
other if pretty => serde_json::to_string_pretty(&other)?,
other => other.to_string(),
})
}
fn tsv(&self, ctx: &Ctx) -> Result<String> {
let mut row = String::new();
for (i, path) in self.fields.iter().enumerate() {
if i > 0 {
row.push('\t');
}
for c in ctx.resolve_str(path)?.chars() {
row.push(if matches!(c, '\t' | '\n' | '\r') {
' '
} else {
c
});
}
}
Ok(row)
}
fn object(&self, ctx: &Ctx) -> Result<Value> {
if self.fields.is_empty() {
return Ok(ctx.full());
}
let mut obj = Map::with_capacity(self.fields.len());
for path in &self.fields {
obj.insert(path.raw.clone(), ctx.resolve(path)?);
}
Ok(Value::Object(obj))
}
}
+58
View File
@@ -0,0 +1,58 @@
mod args;
mod fields;
mod formatter;
mod mode;
mod path;
mod selector;
mod usage;
use std::process::ExitCode;
use brk_error::Result;
use brk_reader::Reader;
use args::Args;
use fields::Ctx;
use formatter::Formatter;
use mode::Mode;
use selector::Selector;
fn main() -> ExitCode {
match run() {
Ok(()) => ExitCode::SUCCESS,
Err(e) => {
eprintln!("blk: {e}");
ExitCode::from(1)
}
}
}
fn run() -> Result<()> {
let raw: Vec<String> = std::env::args().skip(1).collect();
if raw.is_empty() || raw.iter().any(|a| matches!(a.as_str(), "-h" | "--help")) {
usage::print();
return Ok(());
}
let args = Args::parse(raw)?;
let client = args.rpc()?;
let (start, end) = Selector::parse(&args.selector, &client)?;
let network = client.get_network()?;
let mode = Mode::pick(args.pretty, args.compact, args.paths.len())?;
let reader = Reader::new(args.blocks_dir(), &client);
let formatter = Formatter::new(mode, args.paths);
let parser_threads = (std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(2)
/ 2)
.max(1);
for block in reader.range_with(start, end, parser_threads)?.iter() {
let block = block?;
let line = formatter.format(&Ctx::new(&block, network))?;
if !line.is_empty() {
println!("{line}");
}
}
Ok(())
}
+35
View File
@@ -0,0 +1,35 @@
use brk_error::{Error, Result};
#[derive(Clone, Copy)]
pub enum Mode {
Bare,
Tsv,
Json,
Pretty,
}
impl Mode {
pub fn pick(pretty: bool, compact: bool, n_fields: usize) -> Result<Self> {
if pretty && compact {
return Err(Error::Parse(
"--pretty and --compact are mutually exclusive".into(),
));
}
if compact && n_fields == 0 {
return Err(Error::Parse(
"--compact requires at least one field".into(),
));
}
Ok(if pretty {
Self::Pretty
} else if n_fields == 0 {
Self::Json
} else if n_fields == 1 {
Self::Bare
} else if compact {
Self::Tsv
} else {
Self::Json
})
}
}
+40
View File
@@ -0,0 +1,40 @@
use brk_error::{Error, Result};
pub struct Step {
pub name: String,
pub index: Option<usize>,
}
pub struct Path {
pub raw: String,
pub steps: Vec<Step>,
}
impl Path {
pub fn parse(s: &str) -> Result<Self> {
let parts: Vec<&str> = s.split('.').collect();
let mut steps = Vec::new();
let mut i = 0;
while i < parts.len() {
let name = parts[i];
if name.is_empty() {
return Err(Error::Parse(format!("bad path '{s}': empty segment")));
}
if name.parse::<usize>().is_ok() {
return Err(Error::Parse(format!(
"bad path '{s}': '{name}' must follow a field name"
)));
}
let index = parts.get(i + 1).and_then(|p| p.parse::<usize>().ok());
steps.push(Step {
name: name.to_string(),
index,
});
i += if index.is_some() { 2 } else { 1 };
}
Ok(Self {
raw: s.to_string(),
steps,
})
}
}
+44
View File
@@ -0,0 +1,44 @@
use brk_error::{Error, Result};
use brk_rpc::Client;
use brk_types::{CheckedSub, Height};
pub struct Selector;
impl Selector {
pub fn parse(s: &str, client: &Client) -> Result<(Height, Height)> {
let (a, b) = s.split_once("..").unwrap_or((s, s));
let needs_tip = |p: &str| p == "tip" || p.starts_with("tip-");
let tip = if needs_tip(a) || needs_tip(b) {
Some(client.get_last_height()?)
} else {
None
};
let start = Self::endpoint(a, tip)?;
let end = Self::endpoint(b, tip)?;
if end < start {
return Err(Error::Parse(format!(
"range end {end} before start {start}"
)));
}
Ok((start, end))
}
fn endpoint(s: &str, tip: Option<Height>) -> Result<Height> {
if s == "tip" {
return Ok(tip.expect("tip pre-resolved when input contains 'tip'"));
}
if let Some(rest) = s.strip_prefix("tip-") {
let n: u32 = rest
.parse()
.map_err(|_| Error::Parse(format!("bad tip offset: {s}")))?;
let tip = tip.expect("tip pre-resolved when input contains 'tip'");
return tip
.checked_sub(n)
.ok_or_else(|| Error::Parse(format!("tip-{n} underflows genesis")));
}
let n: u32 = s
.parse()
.map_err(|_| Error::Parse(format!("bad height: {s}")))?;
Ok(Height::new(n))
}
}
+190
View File
@@ -0,0 +1,190 @@
use owo_colors::{OwoColorize, Stream};
const SEL_W: usize = 5; // longest selector token: "tip-N"
const LABEL_W: usize = 28; // longest label across OUTPUT/OPTIONS/EXAMPLES (= example cmd "blk 800000 tx.0.vout.0.value")
const FLAG_W: usize = 15; // longest flag: "--rpccookiefile"
const PH_W: usize = LABEL_W - FLAG_W - 1; // placeholder column width so flag+ph total = LABEL_W
const GAP: usize = 4;
pub fn print() {
println!("{} - inspect a Bitcoin Core block", bold("blk"));
println!();
section("USAGE");
println!(
" blk {} [{} ...] [OPTIONS]",
dim("<selector>"),
dim("<field>")
);
println!(
" {}",
dim("no fields = full block as JSON (analog of `bitcoin-cli getblock <hash> 2`)")
);
println!();
section("SELECTOR");
sel("<n>", "single height (e.g. 800000)");
sel("tip", "current chain tip");
sel("tip-N", "tip minus N");
sel("a..b", "inclusive range, endpoints can be height/tip/tip-N");
println!();
section("FIELDS");
println!(
" {}",
dim("dotted paths drill into nested data, omit an index for arrays")
);
println!();
group("block");
fields(&[
"height, hash, time, version, version_hex, bits, nonce,",
"prev, merkle, difficulty, txs, n_inputs, n_outputs,",
"witness_txs, size, strippedsize, weight, subsidy,",
"coinbase, coinbase_hex, header_hex, hex",
]);
println!();
group_note("tx.i", "omit i for all txs");
fields(&[
"txid, wtxid, version, locktime, size, base_size, vsize,",
"weight, inputs, outputs, is_coinbase, has_witness, is_rbf,",
"total_out, hex",
]);
println!();
group_note("tx.i.vin.j", "omit j for all inputs");
fields(&[
"prev_txid, prev_vout, sequence, script_sig, script_sig_asm,",
"witness, has_witness, is_rbf, coinbase",
]);
println!();
group_note("tx.i.vout.j", "omit j for all outputs");
fields(&["value, script_pubkey, script_pubkey_asm, type, address"]);
println!();
println!(
" {}",
dim("Naked tx / tx.i / vin / vout returns the whole sub-object as JSON.")
);
println!();
section("OUTPUT");
out("no fields", "full block JSON object, one per line (NDJSON)");
out("1 field", "bare value, one per line");
out("2+ fields", "JSON object, one per line (NDJSON)");
out("-p, --pretty", "pretty JSON object instead");
out(
"-c, --compact",
"tab-separated values, no field names (TSV)",
);
println!();
section("OPTIONS");
opt(
"--bitcoindir",
"<PATH>",
"Bitcoin directory",
Some("[OS default]"),
);
opt(
"--blocksdir",
"<PATH>",
"Blocks directory",
Some("[<bitcoindir>/blocks]"),
);
opt("--rpcconnect", "<IP>", "RPC host", Some("[localhost]"));
opt("--rpcport", "<PORT>", "RPC port", Some("[8332]"));
opt(
"--rpccookiefile",
"<PATH>",
"RPC cookie file",
Some("[<bitcoindir>/.cookie]"),
);
opt("--rpcuser", "<USERNAME>", "RPC username", None);
opt("--rpcpassword", "<PASSWORD>", "RPC password", None);
println!();
section("EXAMPLES");
ex("blk 800000", "full block as JSON");
ex("blk 800000 hash", "bare hash");
ex("blk 800000 height hash time", "one compact JSON line");
ex("blk 800000 tx.0.txid", "coinbase txid");
ex("blk 800000 tx.txid", "all txids in block (array)");
ex("blk 800000 tx.0.vout.0.value", "coinbase output 0 sats");
ex("blk 800000 tx.0.vout.value", "all output sats for tx 0");
ex("blk 800000 tx.vout.value", "array of arrays (per tx)");
ex("blk 0..2 hash tx.0.txid", "3 NDJSON lines");
ex("blk tip tx.0", "whole coinbase tx as JSON");
}
fn section(name: &str) {
println!("{}", bold(&format!("{name}:")));
}
fn group(name: &str) {
println!(" {}", bold(&format!("{name}:")));
}
fn group_note(name: &str, note: &str) {
println!(
" {} {}",
bold(&format!("{name}:")),
dim(&format!("({note})"))
);
}
fn fields(lines: &[&str]) {
for line in lines {
println!(" {line}");
}
}
fn pad(s: &str, width: usize) -> String {
" ".repeat(width.saturating_sub(s.len()))
}
fn sel(token: &str, desc: &str) {
println!(
" {}{}{}{desc}",
dim(token),
pad(token, SEL_W),
" ".repeat(GAP),
);
}
fn out(label: &str, desc: &str) {
println!(
" {label}{}{}{desc}",
pad(label, LABEL_W),
" ".repeat(GAP)
);
}
fn opt(flag: &str, ph: &str, desc: &str, default: Option<&str>) {
let head = format!(
" {flag}{} {}{}{}",
pad(flag, FLAG_W),
dim(ph),
pad(ph, PH_W),
" ".repeat(GAP),
);
match default {
Some(d) => println!("{head}{desc} {}", dim(d)),
None => println!("{head}{desc}"),
}
}
fn ex(cmd: &str, note: &str) {
println!(
" {cmd}{}{}{}",
pad(cmd, LABEL_W),
" ".repeat(GAP),
dim(&format!("# {note}"))
);
}
fn bold(s: &str) -> String {
s.if_supports_color(Stream::Stdout, |t| t.bold()).to_string()
}
fn dim(s: &str) -> String {
s.if_supports_color(Stream::Stdout, |t| t.bright_black())
.to_string()
}
+1 -1
View File
@@ -64,7 +64,7 @@ brk_mempool = { workspace = true, optional = true }
brk_oracle = { workspace = true, optional = true }
brk_query = { workspace = true, optional = true }
brk_reader = { workspace = true, optional = true }
brk_rpc = { workspace = true, optional = true, features = ["corepc"] }
brk_rpc = { workspace = true, optional = true }
brk_server = { workspace = true, optional = true }
brk_store = { workspace = true, optional = true }
brk_traversable = { workspace = true, optional = true }
+2 -2
View File
@@ -8,5 +8,5 @@ homepage.workspace = true
repository.workspace = true
[dependencies]
libmimalloc-sys = { version = "0.1.44", features = ["extended"] }
mimalloc = { version = "0.1.48", features = ["v3"] }
libmimalloc-sys = { version = "0.1.47", features = ["extended"] }
mimalloc = { version = "0.1.50" }
+1 -1
View File
@@ -12,6 +12,6 @@ brk_cohort = { workspace = true }
brk_query = { workspace = true }
brk_types = { workspace = true }
indexmap = { workspace = true }
oas3 = "0.21"
oas3 = "0.22"
serde = { workspace = true }
serde_json = { workspace = true }
+5 -2
View File
@@ -3,7 +3,10 @@
//! This module detects repeating tree structures and analyzes them
//! using the bottom-up name deconstruction algorithm.
use std::collections::{BTreeMap, BTreeSet};
use std::{
cmp::Reverse,
collections::{BTreeMap, BTreeSet},
};
use brk_types::{TreeNode, extract_json_type};
@@ -111,7 +114,7 @@ pub fn detect_structural_patterns(
// Also collects node bases for each tree path
let node_bases = analyze_pattern_modes(tree, &mut patterns, &pattern_lookup);
patterns.sort_by(|a, b| b.fields.len().cmp(&a.fields.len()));
patterns.sort_by_key(|p| Reverse(p.fields.len()));
(patterns, concrete_to_pattern, type_mappings, node_bases)
}
+7 -5
View File
@@ -10,7 +10,7 @@ use brk_cohort::{
OVER_AMOUNT_NAMES, PROFIT_NAMES, PROFITABILITY_RANGE_NAMES, SPENDABLE_TYPE_NAMES, TERM_NAMES,
UNDER_AGE_NAMES, UNDER_AMOUNT_NAMES,
};
use brk_types::{Index, PoolSlug, pools};
use brk_types::{Index, pools};
use serde::Serialize;
use serde_json::Value;
@@ -20,7 +20,7 @@ use crate::{VERSION, to_camel_case};
pub struct ClientConstants {
pub version: String,
pub indexes: Vec<&'static str>,
pub pool_map: BTreeMap<PoolSlug, &'static str>,
pub pool_map: BTreeMap<String, &'static str>,
}
impl ClientConstants {
@@ -31,9 +31,11 @@ impl ClientConstants {
let pools = pools();
let mut sorted_pools: Vec<_> = pools.iter().collect();
sorted_pools.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase()));
let pool_map: BTreeMap<PoolSlug, &'static str> =
sorted_pools.iter().map(|p| (p.slug(), p.name)).collect();
sorted_pools.sort_by_key(|p| p.name.to_lowercase());
let pool_map: BTreeMap<String, &'static str> = sorted_pools
.iter()
.map(|p| (p.slug().to_string(), p.name))
.collect();
Self {
version: format!("v{}", VERSION),
@@ -4,7 +4,7 @@ use std::fmt::Write;
use crate::{
Endpoint, Parameter,
generators::{normalize_return_type, write_description},
generators::{javascript::types::jsdoc_normalize, normalize_return_type, write_description},
to_camel_case,
};
@@ -14,110 +14,250 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
if !endpoint.should_generate() {
continue;
}
match endpoint.method.as_str() {
"GET" => generate_get_method(output, endpoint),
"POST" => generate_post_method(output, endpoint),
_ => continue,
}
}
}
let method_name = endpoint_to_method_name(endpoint);
let base_return_type =
normalize_return_type(endpoint.response_type.as_deref().unwrap_or("*"));
let return_type = if endpoint.supports_csv {
format!("{} | string", base_return_type)
fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
let method_name = endpoint_to_method_name(endpoint);
let return_type = build_return_type(endpoint);
write_method_doc(output, endpoint);
for param in &endpoint.path_params {
let desc = format_param_desc(param.description.as_deref());
let ty = jsdoc_normalize(&param.param_type);
writeln!(output, " * @param {{{}}} {}{}", ty, param.name, desc).unwrap();
}
for param in &endpoint.query_params {
let optional = if param.required { "" } else { "=" };
let desc = format_param_desc(param.description.as_deref());
let ty = jsdoc_normalize(&param.param_type);
let ident = sanitize_ident(&param.name);
let name_decl = if param.required {
ident
} else {
base_return_type
format!("[{}]", ident)
};
writeln!(output, " /**").unwrap();
if let Some(summary) = &endpoint.summary {
writeln!(output, " * {}", summary).unwrap();
}
if let Some(desc) = &endpoint.description
&& endpoint.summary.as_ref() != Some(desc)
{
writeln!(output, " *").unwrap();
write_description(output, desc, " * ", " *");
}
// Add endpoint path
writeln!(output, " *").unwrap();
writeln!(
output,
" * Endpoint: `{} {}`",
endpoint.method.to_uppercase(),
endpoint.path
" * @param {{{}{}}} {}{}",
ty, optional, name_decl, desc
)
.unwrap();
}
writeln!(
output,
" * @param {{{{ signal?: AbortSignal, onValue?: (value: {}) => void }}}} [options]",
return_type
)
.unwrap();
writeln!(output, " * @returns {{Promise<{}>}}", return_type).unwrap();
writeln!(output, " */").unwrap();
if !endpoint.path_params.is_empty() || !endpoint.query_params.is_empty() {
writeln!(output, " *").unwrap();
}
let params = build_method_params(endpoint);
let params_with_opts = if params.is_empty() {
"{ signal, onValue } = {}".to_string()
} else {
format!("{}, {{ signal, onValue }} = {{}}", params)
};
writeln!(output, " async {}({}) {{", method_name, params_with_opts).unwrap();
for param in &endpoint.path_params {
let desc = format_param_desc(param.description.as_deref());
writeln!(
output,
" * @param {{{}}} {}{}",
param.param_type, param.name, desc
)
.unwrap();
let path = build_path_template(&endpoint.path, &endpoint.path_params);
let fetch_call: String = if endpoint.returns_binary() {
"this.getBytes(path, { signal, onValue })".to_string()
} else if endpoint.returns_json() {
"this.getJson(path, { signal, onValue })".to_string()
} else if endpoint.response_kind.text_is_numeric() {
"Number(await this.getText(path, { signal, onValue: onValue ? (v) => onValue(Number(v)) : undefined }))".to_string()
} else {
"this.getText(path, { signal, onValue })".to_string()
};
write_path_assignment(output, endpoint, &path);
if endpoint.supports_csv {
writeln!(
output,
" if (format === 'csv') return this.getText(path, {{ signal, onValue }});"
)
.unwrap();
}
writeln!(output, " return {};", fetch_call).unwrap();
writeln!(output, " }}\n").unwrap();
}
fn generate_post_method(output: &mut String, endpoint: &Endpoint) {
let method_name = endpoint_to_method_name(endpoint);
let return_type = build_return_type(endpoint);
write_method_doc(output, endpoint);
for param in &endpoint.path_params {
let desc = format_param_desc(param.description.as_deref());
let ty = jsdoc_normalize(&param.param_type);
writeln!(output, " * @param {{{}}} {}{}", ty, param.name, desc).unwrap();
}
for param in &endpoint.query_params {
let optional = if param.required { "" } else { "=" };
let desc = format_param_desc(param.description.as_deref());
let ty = jsdoc_normalize(&param.param_type);
writeln!(
output,
" * @param {{{}{}}} [{}]{}",
ty, optional, param.name, desc
)
.unwrap();
}
if let Some(body) = &endpoint.request_body {
let optional = if body.required { "" } else { "=" };
let ty = jsdoc_normalize(&body.body_type);
writeln!(
output,
" * @param {{{}{}}} body - Request body",
ty, optional
)
.unwrap();
}
writeln!(
output,
" * @param {{{{ signal?: AbortSignal }}}} [options]"
)
.unwrap();
writeln!(output, " * @returns {{Promise<{}>}}", return_type).unwrap();
writeln!(output, " */").unwrap();
let mut params = build_method_params(endpoint);
if endpoint.request_body.is_some() {
if !params.is_empty() {
params.push_str(", ");
}
params.push_str("body");
}
let params_with_opts = if params.is_empty() {
"{ signal } = {}".to_string()
} else {
format!("{}, {{ signal }} = {{}}", params)
};
writeln!(output, " async {}({}) {{", method_name, params_with_opts).unwrap();
let path = build_path_template(&endpoint.path, &endpoint.path_params);
let body_arg = if endpoint.request_body.is_some() {
"body"
} else {
"''"
};
let fetch_call: String = if endpoint.returns_binary() {
format!("this.postBytes(path, {}, {{ signal }})", body_arg)
} else if endpoint.returns_json() {
format!("this.postJson(path, {}, {{ signal }})", body_arg)
} else if endpoint.response_kind.text_is_numeric() {
format!(
"Number(await this.postText(path, {}, {{ signal }}))",
body_arg
)
} else {
format!("this.postText(path, {}, {{ signal }})", body_arg)
};
write_path_assignment(output, endpoint, &path);
writeln!(output, " return {};", fetch_call).unwrap();
writeln!(output, " }}\n").unwrap();
}
fn build_return_type(endpoint: &Endpoint) -> String {
let base = if endpoint.returns_binary() {
"Uint8Array".to_string()
} else {
jsdoc_normalize(&normalize_return_type(
endpoint.schema_name().unwrap_or("*"),
))
};
if endpoint.supports_csv {
format!("{} | string", base)
} else {
base
}
}
fn write_method_doc(output: &mut String, endpoint: &Endpoint) {
writeln!(output, " /**").unwrap();
if let Some(summary) = &endpoint.summary {
writeln!(output, " * {}", summary).unwrap();
}
if let Some(desc) = &endpoint.description
&& endpoint.summary.as_ref() != Some(desc)
{
writeln!(output, " *").unwrap();
write_description(output, desc, " * ", " *");
}
writeln!(output, " *").unwrap();
writeln!(
output,
" * Endpoint: `{} {}`",
endpoint.method.to_uppercase(),
endpoint.path
)
.unwrap();
let has_body_param = endpoint.method == "POST" && endpoint.request_body.is_some();
if !endpoint.path_params.is_empty() || !endpoint.query_params.is_empty() || has_body_param {
writeln!(output, " *").unwrap();
}
}
fn write_path_assignment(output: &mut String, endpoint: &Endpoint, path: &str) {
if endpoint.query_params.is_empty() {
writeln!(output, " const path = `{}`;", path).unwrap();
} else {
writeln!(output, " const params = new URLSearchParams();").unwrap();
for param in &endpoint.query_params {
let optional = if param.required { "" } else { "=" };
let desc = format_param_desc(param.description.as_deref());
writeln!(
output,
" * @param {{{}{}}} [{}]{}",
param.param_type, optional, param.name, desc
)
.unwrap();
}
writeln!(output, " * @returns {{Promise<{}>}}", return_type).unwrap();
writeln!(output, " */").unwrap();
let params = build_method_params(endpoint);
writeln!(output, " async {}({}) {{", method_name, params).unwrap();
let path = build_path_template(&endpoint.path, &endpoint.path_params);
if endpoint.query_params.is_empty() {
writeln!(output, " return this.getJson(`{}`);", path).unwrap();
} else {
writeln!(output, " const params = new URLSearchParams();").unwrap();
for param in &endpoint.query_params {
let ident = sanitize_ident(&param.name);
let ident = sanitize_ident(&param.name);
let is_array = param.param_type.ends_with("[]");
if is_array {
if param.required {
writeln!(
output,
" params.set('{}', String({}));",
param.name, ident
" for (const _v of {}) params.append('{}', String(_v));",
ident, param.name
)
.unwrap();
} else {
writeln!(
output,
" if ({} !== undefined) params.set('{}', String({}));",
ident, param.name, ident
" if ({}) for (const _v of {}) params.append('{}', String(_v));",
ident, ident, param.name
)
.unwrap();
}
}
writeln!(output, " const query = params.toString();").unwrap();
writeln!(
output,
" const path = `{}${{query ? '?' + query : ''}}`;",
path
)
.unwrap();
if endpoint.supports_csv {
writeln!(output, " if (format === 'csv') {{").unwrap();
writeln!(output, " return this.getText(path);").unwrap();
writeln!(output, " }}").unwrap();
writeln!(output, " return this.getJson(path);").unwrap();
} else if param.required {
writeln!(
output,
" params.set('{}', String({}));",
param.name, ident
)
.unwrap();
} else {
writeln!(output, " return this.getJson(path);").unwrap();
writeln!(
output,
" if ({} !== undefined) params.set('{}', String({}));",
ident, param.name, ident
)
.unwrap();
}
}
writeln!(output, " }}\n").unwrap();
writeln!(output, " const query = params.toString();").unwrap();
writeln!(
output,
" const path = `{}${{query ? '?' + query : ''}}`;",
path
)
.unwrap();
}
}
@@ -16,12 +16,16 @@ pub fn generate_base_client(output: &mut String) {
* @typedef {{Object}} BrkClientOptions
* @property {{string}} baseUrl - Base URL for the API
* @property {{number}} [timeout] - Request timeout in milliseconds
* @property {{string|boolean}} [cache] - Enable browser cache with default name (true), custom name (string), or disable (false). No effect in Node.js. Default: true
* @property {{string|boolean}} [browserCache] - Enable browser Cache API with default name (true), custom name (string), or disable (false). No effect in Node.js. Default: true
* @property {{number|boolean}} [memCache] - In-memory parsed-response cache size (LRU). true/undefined → 1000, false/0 → disabled. Lets 304 responses skip the JSON parse entirely. Default: 1000
*/
const _isBrowser = typeof window !== 'undefined' && 'caches' in window;
const _runIdle = (/** @type {{VoidFunction}} */ fn) => (globalThis.requestIdleCallback ?? setTimeout)(fn);
const _defaultCacheName = '__BRK_CLIENT__';
const _defaultBrowserCacheName = '__BRK_CLIENT__';
const _DEFAULT_MEM_CACHE_SIZE = 1000;
/** @template T @typedef {{{{ etag: string | null, value: T }}}} _MemEntry */
/** @param {{*}} v */
const _addCamelGetters = (v) => {{
if (Array.isArray(v)) {{ v.forEach(_addCamelGetters); return v; }}
@@ -38,12 +42,12 @@ const _addCamelGetters = (v) => {{
}};
/**
* @param {{string|boolean|undefined}} cache
* @param {{string|boolean|undefined}} option
* @returns {{Promise<Cache | null>}}
*/
const _openCache = (cache) => {{
if (!_isBrowser || cache === false) return Promise.resolve(null);
const name = typeof cache === 'string' ? cache : _defaultCacheName;
const _openBrowserCache = (option) => {{
if (!_isBrowser || option === false) return Promise.resolve(null);
const name = typeof option === 'string' ? option : _defaultBrowserCacheName;
return caches.open(name).catch(() => null);
}};
@@ -198,7 +202,6 @@ function _wrapSeriesData(raw) {{
* @property {{number}} version - Version of the series data
* @property {{Index}} index - The index type used for this query
* @property {{string}} type - Value type (e.g. "f32", "u64", "Sats")
* @property {{number}} total - Total number of data points
* @property {{number}} start - Start index (inclusive)
* @property {{number}} end - End index (exclusive)
* @property {{string}} stamp - ISO 8601 timestamp of when the response was generated
@@ -234,8 +237,10 @@ function _wrapSeriesData(raw) {{
* @property {{(n: number) => RangeBuilder<T>}} first - Get first n items
* @property {{(n: number) => RangeBuilder<T>}} last - Get last n items
* @property {{(n: number) => SkippedBuilder<T>}} skip - Skip first n items, chain with take()
* @property {{(onUpdate?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch all data
* @property {{(onValue?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch all data
* @property {{() => Promise<string>}} fetchCsv - Fetch all data as CSV
* @property {{() => Promise<number>}} len - Get total number of data points
* @property {{() => Promise<Version>}} version - Get the current version of the series
* @property {{Thenable<T>}} then - Thenable (await endpoint)
* @property {{string}} path - The endpoint path
*/
@@ -248,8 +253,10 @@ function _wrapSeriesData(raw) {{
* @property {{(n: number) => DateRangeBuilder<T>}} first - Get first n items
* @property {{(n: number) => DateRangeBuilder<T>}} last - Get last n items
* @property {{(n: number) => DateSkippedBuilder<T>}} skip - Skip first n items, chain with take()
* @property {{(onUpdate?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch all data
* @property {{(onValue?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch all data
* @property {{() => Promise<string>}} fetchCsv - Fetch all data as CSV
* @property {{() => Promise<number>}} len - Get total number of data points
* @property {{() => Promise<Version>}} version - Get the current version of the series
* @property {{DateThenable<T>}} then - Thenable (await endpoint)
* @property {{string}} path - The endpoint path
*/
@@ -257,39 +264,39 @@ function _wrapSeriesData(raw) {{
/** @typedef {{SeriesEndpoint<any>}} AnySeriesEndpoint */
/** @template T @typedef {{Object}} SingleItemBuilder
* @property {{(onUpdate?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch the item
* @property {{(onValue?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch the item
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{Thenable<T>}} then - Thenable
*/
/** @template T @typedef {{Object}} DateSingleItemBuilder
* @property {{(onUpdate?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch the item
* @property {{(onValue?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch the item
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{DateThenable<T>}} then - Thenable
*/
/** @template T @typedef {{Object}} SkippedBuilder
* @property {{(n: number) => RangeBuilder<T>}} take - Take n items after skipped position
* @property {{(onUpdate?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch from skipped position to end
* @property {{(onValue?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch from skipped position to end
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{Thenable<T>}} then - Thenable
*/
/** @template T @typedef {{Object}} DateSkippedBuilder
* @property {{(n: number) => DateRangeBuilder<T>}} take - Take n items after skipped position
* @property {{(onUpdate?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch from skipped position to end
* @property {{(onValue?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch from skipped position to end
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{DateThenable<T>}} then - Thenable
*/
/** @template T @typedef {{Object}} RangeBuilder
* @property {{(onUpdate?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch the range
* @property {{(onValue?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch the range
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{Thenable<T>}} then - Thenable
*/
/** @template T @typedef {{Object}} DateRangeBuilder
* @property {{(onUpdate?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch the range
* @property {{(onValue?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch the range
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
* @property {{DateThenable<T>}} then - Thenable
*/
@@ -308,7 +315,7 @@ function _wrapSeriesData(raw) {{
/**
* Create a series endpoint builder with typestate pattern.
* @template T
* @param {{BrkClientBase}} client
* @param {{BrkClient}} client
* @param {{string}} name - The series vec name
* @param {{Index}} index - The index name
* @returns {{DateSeriesEndpoint<T>}}
@@ -337,7 +344,7 @@ function _endpoint(client, name, index) {{
* @returns {{DateRangeBuilder<T>}}
*/
const rangeBuilder = (start, end) => ({{
fetch(onUpdate) {{ return client._fetchSeriesData(buildPath(start, end), onUpdate); }},
fetch(onValue) {{ return client._fetchSeriesData(buildPath(start, end), onValue); }},
fetchCsv() {{ return client.getText(buildPath(start, end, 'csv')); }},
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
}});
@@ -347,7 +354,7 @@ function _endpoint(client, name, index) {{
* @returns {{DateSingleItemBuilder<T>}}
*/
const singleItemBuilder = (idx) => ({{
fetch(onUpdate) {{ return client._fetchSeriesData(buildPath(idx, idx + 1), onUpdate); }},
fetch(onValue) {{ return client._fetchSeriesData(buildPath(idx, idx + 1), onValue); }},
fetchCsv() {{ return client.getText(buildPath(idx, idx + 1, 'csv')); }},
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
}});
@@ -358,7 +365,7 @@ function _endpoint(client, name, index) {{
*/
const skippedBuilder = (start) => ({{
take(n) {{ return rangeBuilder(start, start + n); }},
fetch(onUpdate) {{ return client._fetchSeriesData(buildPath(start, undefined), onUpdate); }},
fetch(onValue) {{ return client._fetchSeriesData(buildPath(start, undefined), onValue); }},
fetchCsv() {{ return client.getText(buildPath(start, undefined, 'csv')); }},
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
}});
@@ -374,8 +381,10 @@ function _endpoint(client, name, index) {{
first(n) {{ return rangeBuilder(undefined, n); }},
last(n) {{ return n === 0 ? rangeBuilder(undefined, 0) : rangeBuilder(-n, undefined); }},
skip(n) {{ return skippedBuilder(n); }},
fetch(onUpdate) {{ return client._fetchSeriesData(buildPath(), onUpdate); }},
fetch(onValue) {{ return client._fetchSeriesData(buildPath(), onValue); }},
fetchCsv() {{ return client.getText(buildPath(undefined, undefined, 'csv')); }},
len() {{ return client.getSeriesLen(name, index); }},
version() {{ return client.getSeriesVersion(name, index); }},
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
get path() {{ return p; }},
}};
@@ -396,99 +405,247 @@ class BrkClientBase {{
this.baseUrl = rawUrl.endsWith('/') ? rawUrl.slice(0, -1) : rawUrl;
this.timeout = isString ? 5000 : (options.timeout ?? 5000);
/** @type {{Promise<Cache | null>}} */
this._cachePromise = _openCache(isString ? undefined : options.cache);
this._browserCachePromise = _openBrowserCache(isString ? undefined : options.browserCache);
/** @type {{Cache | null}} */
this._cache = null;
this._cachePromise.then(c => this._cache = c);
this._browserCache = null;
this._browserCachePromise.then(c => this._browserCache = c);
const memOpt = isString ? undefined : options.memCache;
this._memCacheMax = memOpt === false || memOpt === 0
? 0
: (typeof memOpt === 'number' ? memOpt : _DEFAULT_MEM_CACHE_SIZE);
/** @type {{Map<string, _MemEntry<unknown>>}} */
this._memCache = new Map();
}}
/**
* @template T
* @param {{string}} key
* @returns {{_MemEntry<T> | undefined}}
*/
_memGet(key) {{
if (!this._memCacheMax) return undefined;
const hit = this._memCache.get(key);
if (!hit) return undefined;
this._memCache.delete(key);
this._memCache.set(key, hit);
return /** @type {{_MemEntry<T>}} */ (hit);
}}
/**
* @param {{string}} key
* @param {{string | null}} etag
* @param {{unknown}} value
*/
_memSet(key, etag, value) {{
if (!this._memCacheMax) return;
if (this._memCache.has(key)) this._memCache.delete(key);
else if (this._memCache.size >= this._memCacheMax) {{
const oldest = this._memCache.keys().next().value;
if (oldest !== undefined) this._memCache.delete(oldest);
}}
this._memCache.set(key, {{ etag, value }});
}}
/**
* @param {{string}} path
* @param {{{{ signal?: AbortSignal }}}} [options]
* @returns {{Promise<Response>}}
*/
async get(path) {{
async get(path, {{ signal }} = {{}}) {{
const url = `${{this.baseUrl}}${{path}}`;
const res = await fetch(url, {{ signal: AbortSignal.timeout(this.timeout) }});
const signals = [AbortSignal.timeout(this.timeout)];
if (signal) signals.push(signal);
const res = await fetch(url, {{ signal: AbortSignal.any(signals) }});
if (!res.ok) throw new BrkError(`HTTP ${{res.status}}: ${{url}}`, res.status);
return res;
}}
/**
* Make a GET request - races cache vs network, first to resolve calls onUpdate
* Make a GET request with layered caching.
*
* Contract:
* - The returned Promise resolves with the **freshest** value (post-revalidation).
* - `onValue` fires once with the freshest value, or twice if a stale snapshot
* could be shown first (stale-while-revalidate). On a 304 there is no second fire.
*
* Layers:
* - L1 (memCache): in-memory parsed values keyed by URL+ETag. Lets 304s skip the parse entirely.
* - L2 (browserCache): Cache API, survives reload and feeds onValue fast on cold start.
*
* @template T
* @param {{string}} path
* @param {{(value: T) => void}} [onUpdate] - Called when data is available (may be called twice: cache then network)
* @param {{(res: Response) => Promise<T>}} parse - Response body reader
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal }}}} [options]
* @returns {{Promise<T>}}
*/
async getJson(path, onUpdate) {{
async _getCached(path, parse, {{ onValue, signal }} = {{}}) {{
const url = `${{this.baseUrl}}${{path}}`;
const cache = this._cache ?? await this._cachePromise;
/** @type {{_MemEntry<T> | undefined}} */
const memHit = this._memGet(url);
const browserCache = this._browserCache ?? await this._browserCachePromise;
let resolved = false;
/** @type {{Response | null}} */
let cachedRes = null;
// Race cache vs network - first to resolve calls onUpdate
const cachePromise = cache?.match(url).then(async (res) => {{
cachedRes = res ?? null;
if (!res) return null;
const json = _addCamelGetters(await res.json());
if (!resolved && onUpdate) {{
resolved = true;
onUpdate(json);
}}
return json;
}});
const networkPromise = this.get(path).then(async (res) => {{
const cloned = res.clone();
const json = _addCamelGetters(await res.json());
// Skip update if ETag matches and cache already delivered
if (cachedRes?.headers.get('ETag') === res.headers.get('ETag')) {{
if (!resolved && onUpdate) {{
resolved = true;
onUpdate(json);
// L1 fast path: deliver from memCache, revalidate via network.
// ETag match → zero parse, zero clone, zero cache write, no second onValue fire.
if (memHit) {{
if (onValue) onValue(memHit.value);
try {{
const res = await this.get(path, {{ signal }});
const netEtag = res.headers.get('ETag');
if (netEtag && netEtag === memHit.etag) return memHit.value;
const cloned = browserCache ? res.clone() : null;
const value = await parse(res);
this._memSet(url, netEtag, value);
if (onValue) onValue(value);
if (cloned && browserCache) {{
const cache = browserCache;
_runIdle(() => cache.put(url, cloned));
}}
return json;
return value;
}} catch {{
return memHit.value;
}}
resolved = true;
if (onUpdate) {{
onUpdate(json);
}}
if (cache) _runIdle(() => cache.put(url, cloned));
return json;
}});
}}
// L1 miss: race browserCache (stale snapshot) vs network (fresh).
let networkSettled = false;
const stalePromise = onValue && browserCache
? browserCache.match(url).then(async (res) => {{
if (!res || networkSettled) return null;
const value = await parse(res);
if (networkSettled) return value;
this._memSet(url, res.headers.get('ETag'), value);
onValue(value);
return value;
}}).catch(() => null)
: null;
try {{
return await networkPromise;
const res = await this.get(path, {{ signal }});
networkSettled = true;
const netEtag = res.headers.get('ETag');
// Stale won and populated memCache with matching ETag → reuse, skip parse + second onValue.
const populated = /** @type {{_MemEntry<T> | undefined}} */ (this._memGet(url));
if (populated && netEtag && netEtag === populated.etag) return populated.value;
const cloned = browserCache ? res.clone() : null;
const value = await parse(res);
this._memSet(url, netEtag, value);
if (onValue) onValue(value);
if (cloned && browserCache) {{
const cache = browserCache;
_runIdle(() => cache.put(url, cloned));
}}
return value;
}} catch (e) {{
// Network failed - wait for cache
const cachedJson = await cachePromise?.catch(() => null);
if (cachedJson) return cachedJson;
const stale = await stalePromise;
if (stale != null) return stale;
throw e;
}}
}}
/**
* Make a GET request and return raw text (for CSV responses)
* Make a GET request expecting a JSON response. Cached and supports `onValue`.
* @template T
* @param {{string}} path
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal }}}} [options]
* @returns {{Promise<T>}}
*/
getJson(path, options) {{
return this._getCached(path, async (res) => _addCamelGetters(await res.json()), options);
}}
/**
* Make a GET request expecting a text response (text/plain, text/csv, ...).
* Cached and supports `onValue`, same as `getJson`.
* @param {{string}} path
* @param {{{{ onValue?: (value: string) => void, signal?: AbortSignal }}}} [options]
* @returns {{Promise<string>}}
*/
async getText(path) {{
const res = await this.get(path);
getText(path, options) {{
return this._getCached(path, (res) => res.text(), options);
}}
/**
* Make a GET request expecting binary data (application/octet-stream).
* Cached and supports `onValue`, same as `getJson`.
* @param {{string}} path
* @param {{{{ onValue?: (value: Uint8Array) => void, signal?: AbortSignal }}}} [options]
* @returns {{Promise<Uint8Array>}}
*/
getBytes(path, options) {{
return this._getCached(path, async (res) => new Uint8Array(await res.arrayBuffer()), options);
}}
/**
* Make a POST request with a string body.
*
* POST responses are uncached and never invoke `onValue` — every call hits
* the network with the same body and returns the upstream response.
*
* @param {{string}} path
* @param {{string}} body
* @param {{{{ signal?: AbortSignal }}}} [options]
* @returns {{Promise<Response>}}
*/
async post(path, body, {{ signal }} = {{}}) {{
const url = `${{this.baseUrl}}${{path}}`;
const signals = [AbortSignal.timeout(this.timeout)];
if (signal) signals.push(signal);
const res = await fetch(url, {{
method: 'POST',
body,
signal: AbortSignal.any(signals),
}});
if (!res.ok) throw new BrkError(`HTTP ${{res.status}}: ${{url}}`, res.status);
return res;
}}
/**
* Make a POST request expecting a JSON response.
* @template T
* @param {{string}} path
* @param {{string}} body
* @param {{{{ signal?: AbortSignal }}}} [options]
* @returns {{Promise<T>}}
*/
async postJson(path, body, options) {{
const res = await this.post(path, body, options);
return _addCamelGetters(await res.json());
}}
/**
* Make a POST request expecting a text response.
* @param {{string}} path
* @param {{string}} body
* @param {{{{ signal?: AbortSignal }}}} [options]
* @returns {{Promise<string>}}
*/
async postText(path, body, options) {{
const res = await this.post(path, body, options);
return res.text();
}}
/**
* Make a POST request expecting binary data (application/octet-stream).
* @param {{string}} path
* @param {{string}} body
* @param {{{{ signal?: AbortSignal }}}} [options]
* @returns {{Promise<Uint8Array>}}
*/
async postBytes(path, body, options) {{
const res = await this.post(path, body, options);
return new Uint8Array(await res.arrayBuffer());
}}
/**
* Fetch series data and wrap with helper methods (internal)
* @template T
* @param {{string}} path
* @param {{(value: DateSeriesData<T>) => void}} [onUpdate]
* @param {{(value: DateSeriesData<T>) => void}} [onValue]
* @returns {{Promise<DateSeriesData<T>>}}
*/
async _fetchSeriesData(path, onUpdate) {{
const wrappedOnUpdate = onUpdate ? (/** @type {{SeriesData<T>}} */ raw) => onUpdate(_wrapSeriesData(raw)) : undefined;
const raw = await this.getJson(path, wrappedOnUpdate);
async _fetchSeriesData(path, onValue) {{
const wrappedOnValue = onValue ? (/** @type {{SeriesData<T>}} */ raw) => onValue(_wrapSeriesData(raw)) : undefined;
const raw = await this.getJson(path, {{ onValue: wrappedOnValue }});
return _wrapSeriesData(raw);
}}
}}
@@ -611,7 +768,7 @@ pub fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern
r#"/**
* Generic series pattern factory.
* @template T
* @param {{BrkClientBase}} client
* @param {{BrkClient}} client
* @param {{string}} name - The series vec name
* @param {{readonly Index[]}} indexes - The supported indexes
*/
@@ -664,7 +821,7 @@ function _mp(client, name, indexes) {{
// Generate thin wrapper that calls the generic factory
writeln!(
output,
"/** @template T @param {{BrkClientBase}} client @param {{string}} name @returns {{{}<T>}} */",
"/** @template T @param {{BrkClient}} client @param {{string}} name @returns {{{}<T>}} */",
pattern.name
)
.unwrap();
@@ -726,7 +883,7 @@ pub fn generate_structural_patterns(
if pattern.is_generic {
writeln!(output, " * @template T").unwrap();
}
writeln!(output, " * @param {{BrkClientBase}} client").unwrap();
writeln!(output, " * @param {{BrkClient}} client").unwrap();
writeln!(output, " * @param {{string}} acc - Accumulated series name").unwrap();
if pattern.is_templated() {
writeln!(output, " * @param {{string}} disc - Discriminator suffix").unwrap();
@@ -108,15 +108,14 @@ pub fn generate_main_client(
writeln!(output, " constructor(options) {{").unwrap();
writeln!(output, " super(options);").unwrap();
writeln!(output, " /** @type {{SeriesTree}} */").unwrap();
writeln!(output, " this.series = this._buildTree('');").unwrap();
writeln!(output, " this.series = this._buildTree();").unwrap();
writeln!(output, " }}\n").unwrap();
writeln!(output, " /**").unwrap();
writeln!(output, " * @private").unwrap();
writeln!(output, " * @param {{string}} basePath").unwrap();
writeln!(output, " * @returns {{SeriesTree}}").unwrap();
writeln!(output, " */").unwrap();
writeln!(output, " _buildTree(basePath) {{").unwrap();
writeln!(output, " _buildTree() {{").unwrap();
writeln!(output, " return {{").unwrap();
let mut generated = BTreeSet::new();
generate_tree_initializer(
@@ -52,7 +52,7 @@ pub fn generate_type_definitions(output: &mut String, schemas: &TypeSchemas) {
.map(|arr| arr.iter().any(|v| v.as_str() == Some(prop_name)))
.unwrap_or(false);
let optional = if required { "" } else { "=" };
let safe_name = to_camel_case(prop_name);
let safe_name = to_camel_case(&prop_name.replace(['[', ']'], ""));
let prop_desc = prop_schema
.get("description")
.and_then(|d| d.as_str())
@@ -111,6 +111,25 @@ fn json_type_to_js(ty: &str, schema: &Value, current_type: Option<&str>) -> Stri
}
}
/// JSDoc has no `integer` keyword, only `number`. Map `integer` (and `integer[]`,
/// `Foo<integer>`, etc.) to `number` before emitting type strings to JS.
pub fn jsdoc_normalize(ty: &str) -> String {
let mut out = ty.to_string();
let mut prev = String::new();
while prev != out {
prev = out.clone();
out = out.replace("integer[]", "number[]");
out = out.replace("<integer>", "<number>");
out = out.replace("(integer)", "(number)");
out = out.replace("integer | ", "number | ");
out = out.replace(" | integer", " | number");
}
if out == "integer" {
return "number".to_string();
}
out
}
/// Convert a JSON schema to a JavaScript type string.
pub fn schema_to_js_type(schema: &Value, current_type: Option<&str>) -> String {
if let Some(all_of) = schema.get("allOf").and_then(|v| v.as_array()) {
+70 -15
View File
@@ -96,13 +96,16 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
}
let method_name = endpoint_to_method_name(endpoint);
let base_return_type = normalize_return_type(
&endpoint
.response_type
.as_deref()
.map(js_type_to_python)
.unwrap_or_else(|| "str".to_string()),
);
let base_return_type = if endpoint.returns_binary() {
"bytes".to_string()
} else {
normalize_return_type(
&endpoint
.schema_name()
.map(js_type_to_python)
.unwrap_or_else(|| "str".to_string()),
)
};
let return_type = if endpoint.supports_csv {
format!("Union[{}, str]", base_return_type)
@@ -159,24 +162,58 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
// Build path
let path = build_path_template(&endpoint.path, &endpoint.path_params);
let fetch_method = if endpoint.returns_json() {
"get_json"
let is_post = endpoint.method == "POST";
let fetch_method = match (is_post, &endpoint.response_kind) {
(false, _) if endpoint.returns_binary() => "get",
(false, _) if endpoint.returns_json() => "get_json",
(false, _) => "get_text",
(true, _) if endpoint.returns_binary() => "post",
(true, _) if endpoint.returns_json() => "post_json",
(true, _) => "post_text",
};
let body_arg = if is_post && endpoint.request_body.is_some() {
", body"
} else {
"get_text"
""
};
let (wrap_prefix, wrap_suffix) = if endpoint.response_kind.text_is_numeric() {
("int(", ")")
} else {
("", "")
};
if endpoint.query_params.is_empty() {
if endpoint.path_params.is_empty() {
writeln!(output, " return self.{}('{}')", fetch_method, path).unwrap();
writeln!(
output,
" return {}self.{}('{}'{}){}",
wrap_prefix, fetch_method, path, body_arg, wrap_suffix
)
.unwrap();
} else {
writeln!(output, " return self.{}(f'{}')", fetch_method, path).unwrap();
writeln!(
output,
" return {}self.{}(f'{}'{}){}",
wrap_prefix, fetch_method, path, body_arg, wrap_suffix
)
.unwrap();
}
} else {
writeln!(output, " params = []").unwrap();
for param in &endpoint.query_params {
// Use safe name for Python variable, original name for API query parameter
let safe_name = escape_python_keyword(&param.name);
if param.required {
let is_array = param.param_type.ends_with("[]");
if is_array {
writeln!(
output,
" for _v in {}: params.append(f'{}={{_v}}')",
safe_name, param.name
)
.unwrap();
} else if param.required {
writeln!(
output,
" params.append(f'{}={{{}}}')",
@@ -203,9 +240,19 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
if endpoint.supports_csv {
writeln!(output, " if format == 'csv':").unwrap();
writeln!(output, " return self.get_text(path)").unwrap();
writeln!(output, " return self.{}(path)", fetch_method).unwrap();
writeln!(
output,
" return {}self.{}(path{}){}",
wrap_prefix, fetch_method, body_arg, wrap_suffix
)
.unwrap();
} else {
writeln!(output, " return self.{}(path)", fetch_method).unwrap();
writeln!(
output,
" return {}self.{}(path{}){}",
wrap_prefix, fetch_method, body_arg, wrap_suffix
)
.unwrap();
}
}
@@ -240,6 +287,14 @@ fn build_method_params(endpoint: &Endpoint) -> String {
params.push(format!(", {}: Optional[{}] = None", safe_name, py_type));
}
}
if let Some(body) = &endpoint.request_body {
let py_type = js_type_to_python(&body.body_type);
if body.required {
params.push(format!(", body: {}", py_type));
} else {
params.push(format!(", body: Optional[{}] = None", py_type));
}
}
params.join("")
}
@@ -99,6 +99,28 @@ class BrkClientBase:
"""Make a GET request and return text."""
return self.get(path).decode()
def post(self, path: str, body: str) -> bytes:
"""Make a POST request with a string body and return raw bytes."""
try:
conn = self._connect()
conn.request("POST", path, body=body)
res = conn.getresponse()
data = res.read()
if res.status >= 400:
raise BrkError(f"HTTP error: {{res.status}}", res.status)
return data
except (ConnectionError, OSError, TimeoutError) as e:
self._conn = None
raise BrkError(str(e))
def post_json(self, path: str, body: str) -> Any:
"""Make a POST request and return JSON."""
return json.loads(self.post(path, body))
def post_text(self, path: str, body: str) -> str:
"""Make a POST request and return text."""
return self.post(path, body).decode()
def close(self) -> None:
"""Close the HTTP client."""
if self._conn:
@@ -221,7 +243,6 @@ class SeriesData(Generic[T]):
version: int
index: Index
type: str
total: int
start: int
end: int
stamp: str
@@ -326,13 +347,13 @@ AnyDateSeriesData = DateSeriesData[Any]
class _EndpointConfig:
"""Shared endpoint configuration."""
client: BrkClientBase
client: BrkClient
name: str
index: Index
start: Optional[int]
end: Optional[int]
def __init__(self, client: BrkClientBase, name: str, index: Index,
def __init__(self, client: BrkClient, name: str, index: Index,
start: Optional[int] = None, end: Optional[int] = None):
self.client = client
self.name = name
@@ -367,6 +388,12 @@ class _EndpointConfig:
def get_csv(self) -> str:
return self.client.get_text(self._build_path(format='csv'))
def get_len(self) -> int:
return self.client.get_series_len(self.name, self.index)
def get_version(self) -> Version:
return self.client.get_series_version(self.name, self.index)
class RangeBuilder(Generic[T]):
"""Builder with range specified."""
@@ -450,7 +477,7 @@ class SeriesEndpoint(Generic[T]):
data = endpoint.skip(100).take(10).fetch()
"""
def __init__(self, client: BrkClientBase, name: str, index: Index):
def __init__(self, client: BrkClient, name: str, index: Index):
self._config = _EndpointConfig(client, name, index)
@overload
@@ -484,6 +511,14 @@ class SeriesEndpoint(Generic[T]):
"""Fetch all data as CSV."""
return self._config.get_csv()
def len(self) -> int:
"""Total number of data points for this series."""
return self._config.get_len()
def version(self) -> Version:
"""Current version of the series."""
return self._config.get_version()
def path(self) -> str:
"""Get the base endpoint path."""
return self._config.path()
@@ -501,7 +536,7 @@ class DateSeriesEndpoint(Generic[T]):
data = endpoint[:10].fetch()
"""
def __init__(self, client: BrkClientBase, name: str, index: Index):
def __init__(self, client: BrkClient, name: str, index: Index):
self._config = _EndpointConfig(client, name, index)
@overload
@@ -547,6 +582,14 @@ class DateSeriesEndpoint(Generic[T]):
"""Fetch all data as CSV."""
return self._config.get_csv()
def len(self) -> int:
"""Total number of data points for this series."""
return self._config.get_len()
def version(self) -> Version:
"""Current version of the series."""
return self._config.get_version()
def path(self) -> str:
"""Get the base endpoint path."""
return self._config.path()
@@ -605,10 +648,10 @@ pub fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern
// Generate helper functions
writeln!(
output,
r#"def _ep(c: BrkClientBase, n: str, i: Index) -> SeriesEndpoint[Any]:
r#"def _ep(c: BrkClient, n: str, i: Index) -> SeriesEndpoint[Any]:
return SeriesEndpoint(c, n, i)
def _dep(c: BrkClientBase, n: str, i: Index) -> DateSeriesEndpoint[Any]:
def _dep(c: BrkClient, n: str, i: Index) -> DateSeriesEndpoint[Any]:
return DateSeriesEndpoint(c, n, i)
"#
)
@@ -624,7 +667,7 @@ def _dep(c: BrkClientBase, n: str, i: Index) -> DateSeriesEndpoint[Any]:
writeln!(output, "class {}(Generic[T]):", by_class_name).unwrap();
writeln!(
output,
" def __init__(self, c: BrkClientBase, n: str): self._c, self._n = c, n"
" def __init__(self, c: BrkClient, n: str): self._c, self._n = c, n"
)
.unwrap();
for index in &pattern.indexes {
@@ -649,7 +692,7 @@ def _dep(c: BrkClientBase, n: str, i: Index) -> DateSeriesEndpoint[Any]:
writeln!(output, " by: {}[T]", by_class_name).unwrap();
writeln!(
output,
" def __init__(self, c: BrkClientBase, n: str): self._n, self.by = n, {}(c, n)",
" def __init__(self, c: BrkClient, n: str): self._n, self.by = n, {}(c, n)",
by_class_name
)
.unwrap();
@@ -706,13 +749,13 @@ pub fn generate_structural_patterns(
if pattern.is_templated() {
writeln!(
output,
" def __init__(self, client: BrkClientBase, acc: str, disc: str):"
" def __init__(self, client: BrkClient, acc: str, disc: str):"
)
.unwrap();
} else {
writeln!(
output,
" def __init__(self, client: BrkClientBase, acc: str):"
" def __init__(self, client: BrkClient, acc: str):"
)
.unwrap();
}
@@ -64,7 +64,7 @@ fn generate_tree_class(
writeln!(output, " ").unwrap();
writeln!(
output,
" def __init__(self, client: BrkClientBase, base_path: str = ''):"
" def __init__(self, client: BrkClient, base_path: str = ''):"
)
.unwrap();
@@ -110,8 +110,9 @@ fn topological_sort_schemas(schemas: &TypeSchemas) -> Vec<String> {
for (name, schema) in schemas {
let mut type_deps = BTreeSet::new();
collect_schema_refs(schema, &mut type_deps);
// Only keep deps that are in our schemas
type_deps.retain(|d| schemas.contains_key(d));
// Only keep deps that are in our schemas, and drop self-references
// (handled at emit time by quoting via current_type)
type_deps.retain(|d| schemas.contains_key(d) && d != name);
deps.insert(name.clone(), type_deps);
}
+190 -114
View File
@@ -87,124 +87,200 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
if !endpoint.should_generate() {
continue;
}
let method_name = endpoint_to_method_name(endpoint);
let base_return_type = endpoint
.response_type
.as_deref()
.map(js_type_to_rust)
.unwrap_or_else(|| "String".to_string());
let return_type = if endpoint.supports_csv {
format!("FormatResponse<{}>", base_return_type)
} else {
base_return_type.clone()
};
writeln!(
output,
" /// {}",
endpoint.summary.as_deref().unwrap_or(&method_name)
)
.unwrap();
if let Some(desc) = &endpoint.description
&& endpoint.summary.as_ref() != Some(desc)
{
writeln!(output, " ///").unwrap();
write_description(output, desc, " /// ", " ///");
match endpoint.method.as_str() {
"GET" => generate_get_method(output, endpoint),
"POST" => generate_post_method(output, endpoint),
_ => continue,
}
// Add endpoint path
writeln!(output, " ///").unwrap();
writeln!(
output,
" /// Endpoint: `{} {}`",
endpoint.method.to_uppercase(),
endpoint.path
)
.unwrap();
let params = build_method_params(endpoint);
writeln!(
output,
" pub fn {}(&self{}) -> Result<{}> {{",
method_name, params, return_type
)
.unwrap();
let (path, index_arg) = build_path_template(endpoint);
let fetch_method = if endpoint.returns_json() {
"get_json"
} else {
"get_text"
};
if endpoint.query_params.is_empty() {
writeln!(
output,
" self.base.{}(&format!(\"{}\"{}))",
fetch_method, path, index_arg
)
.unwrap();
} else {
writeln!(output, " let mut query = Vec::new();").unwrap();
for param in &endpoint.query_params {
let ident = sanitize_ident(&param.name);
let is_array = param.param_type.ends_with("[]");
if is_array {
writeln!(
output,
" for v in {} {{ query.push(format!(\"{}={{}}\", v)); }}",
ident, param.name
)
.unwrap();
} else if param.required {
writeln!(
output,
" query.push(format!(\"{}={{}}\", {}));",
param.name, ident
)
.unwrap();
} else {
writeln!(
output,
" if let Some(v) = {} {{ query.push(format!(\"{}={{}}\", v)); }}",
ident, param.name
)
.unwrap();
}
}
writeln!(output, " let query_str = if query.is_empty() {{ String::new() }} else {{ format!(\"?{{}}\", query.join(\"&\")) }};").unwrap();
writeln!(
output,
" let path = format!(\"{}{{}}\"{}, query_str);",
path, index_arg
)
.unwrap();
if endpoint.supports_csv {
writeln!(output, " if format == Some(Format::CSV) {{").unwrap();
writeln!(
output,
" self.base.get_text(&path).map(FormatResponse::Csv)"
)
.unwrap();
writeln!(output, " }} else {{").unwrap();
writeln!(
output,
" self.base.{}(&path).map(FormatResponse::Json)",
fetch_method
)
.unwrap();
writeln!(output, " }}").unwrap();
} else {
writeln!(output, " self.base.{}(&path)", fetch_method).unwrap();
}
}
writeln!(output, " }}\n").unwrap();
}
}
fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
let method_name = endpoint_to_method_name(endpoint);
let return_type = build_return_type(endpoint);
write_method_doc(output, endpoint);
let params = build_method_params(endpoint);
writeln!(
output,
" pub fn {}(&self{}) -> Result<{}> {{",
method_name, params, return_type
)
.unwrap();
let (path, index_arg) = build_path_template(endpoint);
let fetch_method = if endpoint.returns_binary() {
"get_bytes"
} else if endpoint.returns_json() {
"get_json"
} else {
"get_text"
};
if endpoint.query_params.is_empty() {
writeln!(
output,
" self.base.{}(&format!(\"{}\"{}))",
fetch_method, path, index_arg
)
.unwrap();
} else {
write_query_assembly(output, endpoint, &path, index_arg);
if endpoint.supports_csv {
writeln!(output, " if format == Some(Format::CSV) {{").unwrap();
writeln!(
output,
" self.base.get_text(&path).map(FormatResponse::Csv)"
)
.unwrap();
writeln!(output, " }} else {{").unwrap();
writeln!(
output,
" self.base.{}(&path).map(FormatResponse::Json)",
fetch_method
)
.unwrap();
writeln!(output, " }}").unwrap();
} else {
writeln!(output, " self.base.{}(&path)", fetch_method).unwrap();
}
}
writeln!(output, " }}\n").unwrap();
}
fn generate_post_method(output: &mut String, endpoint: &Endpoint) {
let method_name = endpoint_to_method_name(endpoint);
let return_type = build_return_type(endpoint);
write_method_doc(output, endpoint);
let mut params = build_method_params(endpoint);
if endpoint.request_body.is_some() {
params.push_str(", body: &str");
}
writeln!(
output,
" pub fn {}(&self{}) -> Result<{}> {{",
method_name, params, return_type
)
.unwrap();
let (path, index_arg) = build_path_template(endpoint);
let body_arg = if endpoint.request_body.is_some() {
"body"
} else {
"\"\""
};
let fetch_method = if endpoint.returns_binary() {
"post_bytes"
} else if endpoint.returns_json() {
"post_json"
} else {
"post_text"
};
if endpoint.query_params.is_empty() {
writeln!(
output,
" self.base.{}(&format!(\"{}\"{}), {})",
fetch_method, path, index_arg, body_arg
)
.unwrap();
} else {
write_query_assembly(output, endpoint, &path, index_arg);
writeln!(
output,
" self.base.{}(&path, {})",
fetch_method, body_arg
)
.unwrap();
}
writeln!(output, " }}\n").unwrap();
}
fn build_return_type(endpoint: &Endpoint) -> String {
let base = if endpoint.returns_binary() {
"Vec<u8>".to_string()
} else if endpoint.returns_text() {
"String".to_string()
} else {
endpoint
.schema_name()
.map(js_type_to_rust)
.unwrap_or_else(|| "String".to_string())
};
if endpoint.supports_csv {
format!("FormatResponse<{}>", base)
} else {
base
}
}
fn write_method_doc(output: &mut String, endpoint: &Endpoint) {
let method_name = endpoint_to_method_name(endpoint);
writeln!(
output,
" /// {}",
endpoint.summary.as_deref().unwrap_or(&method_name)
)
.unwrap();
if let Some(desc) = &endpoint.description
&& endpoint.summary.as_ref() != Some(desc)
{
writeln!(output, " ///").unwrap();
write_description(output, desc, " /// ", " ///");
}
writeln!(output, " ///").unwrap();
writeln!(
output,
" /// Endpoint: `{} {}`",
endpoint.method.to_uppercase(),
endpoint.path
)
.unwrap();
}
fn write_query_assembly(output: &mut String, endpoint: &Endpoint, path: &str, index_arg: &str) {
writeln!(output, " let mut query = Vec::new();").unwrap();
for param in &endpoint.query_params {
let ident = sanitize_ident(&param.name);
let is_array = param.param_type.ends_with("[]");
if is_array {
writeln!(
output,
" for v in {} {{ query.push(format!(\"{}={{}}\", v)); }}",
ident, param.name
)
.unwrap();
} else if param.required {
writeln!(
output,
" query.push(format!(\"{}={{}}\", {}));",
param.name, ident
)
.unwrap();
} else {
writeln!(
output,
" if let Some(v) = {} {{ query.push(format!(\"{}={{}}\", v)); }}",
ident, param.name
)
.unwrap();
}
}
writeln!(output, " let query_str = if query.is_empty() {{ String::new() }} else {{ format!(\"?{{}}\", query.join(\"&\")) }};").unwrap();
writeln!(
output,
" let path = format!(\"{}{{}}\"{}, query_str);",
path, index_arg
)
.unwrap();
}
fn endpoint_to_method_name(endpoint: &Endpoint) -> String {
to_snake_case(&endpoint.operation_name())
}
@@ -103,6 +103,38 @@ impl BrkClientBase {{
.and_then(|mut r| r.body_mut().read_to_string())
.map_err(|e| BrkError {{ message: e.to_string() }})
}}
/// Make a GET request and return raw bytes response.
pub fn get_bytes(&self, path: &str) -> Result<Vec<u8>> {{
self.agent.get(&self.url(path))
.call()
.and_then(|mut r| r.body_mut().read_to_vec())
.map_err(|e| BrkError {{ message: e.to_string() }})
}}
/// Make a POST request and deserialize JSON response.
pub fn post_json<T: DeserializeOwned>(&self, path: &str, body: &str) -> Result<T> {{
self.agent.post(&self.url(path))
.send(body)
.and_then(|mut r| r.body_mut().read_json())
.map_err(|e| BrkError {{ message: e.to_string() }})
}}
/// Make a POST request and return raw text response.
pub fn post_text(&self, path: &str, body: &str) -> Result<String> {{
self.agent.post(&self.url(path))
.send(body)
.and_then(|mut r| r.body_mut().read_to_string())
.map_err(|e| BrkError {{ message: e.to_string() }})
}}
/// Make a POST request and return raw bytes response.
pub fn post_bytes(&self, path: &str, body: &str) -> Result<Vec<u8>> {{
self.agent.post(&self.url(path))
.send(body)
.and_then(|mut r| r.body_mut().read_to_vec())
.map_err(|e| BrkError {{ message: e.to_string() }})
}}
}}
/// Build series name with suffix.
@@ -187,6 +219,14 @@ impl EndpointConfig {{
fn get_text(&self, format: Option<&str>) -> Result<String> {{
self.client.get_text(&self.build_path(format))
}}
fn get_len(&self) -> Result<i64> {{
self.client.get_json(&format!("/api/series/{{}}/{{}}/len", self.name, self.index.name()))
}}
fn get_version(&self) -> Result<Version> {{
self.client.get_json(&format!("/api/series/{{}}/{{}}/version", self.name, self.index.name()))
}}
}}
/// Builder for series endpoint queries.
@@ -280,6 +320,17 @@ impl<T: DeserializeOwned, D: DeserializeOwned> SeriesEndpoint<T, D> {{
self.config.get_text(Some("csv"))
}}
/// Total number of data points for this series.
#[allow(clippy::len_without_is_empty)]
pub fn len(&self) -> Result<i64> {{
self.config.get_len()
}}
/// Current version of the series.
pub fn version(&self) -> Result<Version> {{
self.config.get_version()
}}
/// Get the base endpoint path.
pub fn path(&self) -> String {{
self.config.path()
@@ -25,6 +25,7 @@ pub fn generate_rust_client(
writeln!(output, "// Auto-generated BRK Rust client").unwrap();
writeln!(output, "// Do not edit manually\n").unwrap();
writeln!(output, "#![allow(non_camel_case_types)]").unwrap();
writeln!(output, "#![allow(non_snake_case)]").unwrap();
writeln!(output, "#![allow(dead_code)]").unwrap();
writeln!(output, "#![allow(unused_variables)]").unwrap();
writeln!(output, "#![allow(clippy::useless_format)]").unwrap();
@@ -0,0 +1,92 @@
use crate::openapi::{Parameter, ResponseKind};
/// Request body shape for POST/PUT/PATCH endpoints.
#[derive(Debug, Clone)]
pub struct RequestBody {
/// Body content type as a name (e.g. "string" for text/plain, "Foo" for an `application/json` $ref).
pub body_type: String,
/// Whether the body is required.
pub required: bool,
}
/// Endpoint information extracted from OpenAPI spec.
#[derive(Debug, Clone)]
pub struct Endpoint {
/// HTTP method (GET, POST, etc.)
pub method: String,
/// Path template (e.g., "/blocks/{hash}")
pub path: String,
/// Operation ID (e.g., "getBlockByHash")
pub operation_id: Option<String>,
/// Short summary
pub summary: Option<String>,
/// Detailed description
pub description: Option<String>,
/// Path parameters
pub path_params: Vec<Parameter>,
/// Query parameters
pub query_params: Vec<Parameter>,
/// Request body, if any (POST/PUT/PATCH).
pub request_body: Option<RequestBody>,
/// Body kind for the 200 response.
pub response_kind: ResponseKind,
/// Whether this endpoint is deprecated
pub deprecated: bool,
/// Whether this endpoint supports CSV format (text/csv content type)
pub supports_csv: bool,
}
impl Endpoint {
/// Returns true if this endpoint should be included in client generation.
/// Non-deprecated GET and POST endpoints are included.
pub fn should_generate(&self) -> bool {
!self.deprecated && (self.method == "GET" || self.method == "POST")
}
/// Returns true if this endpoint returns JSON.
pub fn returns_json(&self) -> bool {
matches!(self.response_kind, ResponseKind::Json(_))
}
/// Returns true if this endpoint returns binary data (application/octet-stream).
pub fn returns_binary(&self) -> bool {
matches!(self.response_kind, ResponseKind::Binary)
}
/// Returns true if this endpoint returns plain text (typed or opaque).
pub fn returns_text(&self) -> bool {
matches!(self.response_kind, ResponseKind::Text(_))
}
/// Schema name attached to the response, if any.
pub fn schema_name(&self) -> Option<&str> {
self.response_kind.schema_name()
}
/// Returns the operation ID or generates one from the path.
/// The returned string uses the raw case from the spec (typically camelCase).
pub fn operation_name(&self) -> String {
if let Some(op_id) = &self.operation_id {
return op_id.clone();
}
let mut parts: Vec<String> = Vec::new();
let mut prev_segment = "";
for segment in self.path.split('/').filter(|s| !s.is_empty()) {
if segment == "api" {
continue;
}
if let Some(param) = segment.strip_prefix('{').and_then(|s| s.strip_suffix('}')) {
let prev_normalized = prev_segment.replace('-', "_");
if !prev_normalized.ends_with(param) {
parts.push(format!("by_{}", param));
}
} else {
let normalized = segment.replace('-', "_");
parts.push(normalized);
prev_segment = segment;
}
}
format!("get_{}", parts.join("_"))
}
}
@@ -1,3 +1,13 @@
mod endpoint;
mod parameter;
mod response_kind;
mod text_schema;
pub use endpoint::{Endpoint, RequestBody};
pub use parameter::Parameter;
pub use response_kind::ResponseKind;
pub use text_schema::TextSchema;
use std::{collections::BTreeMap, io};
use crate::ref_to_type_name;
@@ -11,83 +21,6 @@ use serde_json::Value;
/// Type schema extracted from OpenAPI components
pub type TypeSchemas = BTreeMap<String, Value>;
/// Endpoint information extracted from OpenAPI spec
#[derive(Debug, Clone)]
pub struct Endpoint {
/// HTTP method (GET, POST, etc.)
pub method: String,
/// Path template (e.g., "/blocks/{hash}")
pub path: String,
/// Operation ID (e.g., "getBlockByHash")
pub operation_id: Option<String>,
/// Short summary
pub summary: Option<String>,
/// Detailed description
pub description: Option<String>,
/// Path parameters
pub path_params: Vec<Parameter>,
/// Query parameters
pub query_params: Vec<Parameter>,
/// Response type (simplified)
pub response_type: Option<String>,
/// Whether this endpoint is deprecated
pub deprecated: bool,
/// Whether this endpoint supports CSV format (text/csv content type)
pub supports_csv: bool,
}
impl Endpoint {
/// Returns true if this endpoint should be included in client generation.
/// Only non-deprecated GET endpoints are included.
pub fn should_generate(&self) -> bool {
self.method == "GET" && !self.deprecated
}
/// Returns true if this endpoint returns JSON (has a response_type extracted from application/json).
pub fn returns_json(&self) -> bool {
self.response_type.is_some()
}
/// Returns the operation ID or generates one from the path.
/// The returned string uses the raw case from the spec (typically camelCase).
pub fn operation_name(&self) -> String {
if let Some(op_id) = &self.operation_id {
return op_id.clone();
}
// Generate from path: /api/block/{hash} -> "get_block"
// Skip "api" prefix, convert hyphens to underscores, avoid redundant param names
let mut parts: Vec<String> = Vec::new();
let mut prev_segment = "";
for segment in self.path.split('/').filter(|s| !s.is_empty()) {
if segment == "api" {
continue;
}
if let Some(param) = segment.strip_prefix('{').and_then(|s| s.strip_suffix('}')) {
// Only add "by_{param}" if the previous segment doesn't already contain the param name
let prev_normalized = prev_segment.replace('-', "_");
if !prev_normalized.ends_with(param) {
parts.push(format!("by_{}", param));
}
} else {
let normalized = segment.replace('-', "_");
parts.push(normalized);
prev_segment = segment;
}
}
format!("get_{}", parts.join("_"))
}
}
/// Parameter information
#[derive(Debug, Clone)]
pub struct Parameter {
pub name: String,
pub required: bool,
pub param_type: String,
pub description: Option<String>,
}
/// Parse OpenAPI spec from JSON string
///
/// Pre-processes the JSON to handle oas3 limitations:
@@ -164,7 +97,7 @@ pub fn extract_endpoints(spec: &Spec) -> Vec<Endpoint> {
for (path, path_item) in paths {
for (method, operation) in get_operations(path_item) {
if let Some(endpoint) = extract_endpoint(path, method, operation) {
if let Some(endpoint) = extract_endpoint(path, method, operation, spec) {
endpoints.push(endpoint);
}
}
@@ -186,11 +119,17 @@ fn get_operations(path_item: &PathItem) -> Vec<(&'static str, &Operation)> {
.collect()
}
fn extract_endpoint(path: &str, method: &str, operation: &Operation) -> Option<Endpoint> {
fn extract_endpoint(
path: &str,
method: &str,
operation: &Operation,
spec: &Spec,
) -> Option<Endpoint> {
let path_params = extract_path_parameters(path, operation);
let query_params = extract_parameters(operation, ParameterIn::Query);
let response_type = extract_response_type(operation);
let response_kind = extract_response_kind(operation, spec);
let request_body = extract_request_body(operation);
let supports_csv = check_csv_support(operation);
Some(Endpoint {
@@ -201,12 +140,38 @@ fn extract_endpoint(path: &str, method: &str, operation: &Operation) -> Option<E
description: operation.description.clone(),
path_params,
query_params,
response_type,
request_body,
response_kind,
deprecated: operation.deprecated.unwrap_or(false),
supports_csv,
})
}
/// Extract the request body shape, if any.
/// Prefers `text/plain` (string) over `application/json` (typed).
fn extract_request_body(operation: &Operation) -> Option<RequestBody> {
let req = operation.request_body.as_ref()?;
let req = match req {
ObjectOrReference::Object(rb) => rb,
ObjectOrReference::Ref { .. } => return None,
};
let body_type = if req.content.contains_key("text/plain; charset=utf-8")
|| req.content.contains_key("text/plain")
{
"string".to_string()
} else if let Some(content) = req.content.get("application/json") {
schema_name_from_content(content).unwrap_or_else(|| "Object".to_string())
} else {
"string".to_string()
};
Some(RequestBody {
body_type,
required: req.required.unwrap_or(false),
})
}
/// Check if the endpoint supports CSV format (has text/csv in 200 response content types).
fn check_csv_support(operation: &Operation) -> bool {
let Some(responses) = operation.responses.as_ref() else {
@@ -253,12 +218,7 @@ fn extract_parameters(operation: &Operation, location: ParameterIn) -> Vec<Param
let param_type = param
.schema
.as_ref()
.and_then(|s| match s {
ObjectOrReference::Ref { ref_path, .. } => {
ref_to_type_name(ref_path).map(|s| s.to_string())
}
ObjectOrReference::Object(obj_schema) => schema_to_type_name(obj_schema),
})
.and_then(schema_type_from_schema)
.unwrap_or_else(|| "string".to_string());
Some(Parameter {
name: param.name.clone(),
@@ -272,28 +232,59 @@ fn extract_parameters(operation: &Operation, location: ParameterIn) -> Vec<Param
.collect()
}
fn extract_response_type(operation: &Operation) -> Option<String> {
let responses = operation.responses.as_ref()?;
fn extract_response_kind(operation: &Operation, spec: &Spec) -> ResponseKind {
let response = operation
.responses
.as_ref()
.and_then(|r| r.get("200"))
.and_then(|r| match r {
ObjectOrReference::Object(o) => Some(o),
ObjectOrReference::Ref { .. } => None,
});
let Some(response) = response else {
return ResponseKind::Text(None);
};
// Look for 200 OK response
let response = responses.get("200")?;
match response {
ObjectOrReference::Object(response) => {
// Look for JSON content
let content = response.content.get("application/json")?;
match &content.schema {
Some(ObjectOrReference::Ref { ref_path, .. }) => {
// Extract type name from reference like "#/components/schemas/Block"
Some(ref_to_type_name(ref_path)?.to_string())
}
Some(ObjectOrReference::Object(schema)) => schema_to_type_name(schema),
None => None,
}
}
ObjectOrReference::Ref { .. } => None,
if response.content.contains_key("application/octet-stream") {
return ResponseKind::Binary;
}
if let Some(content) = response.content.get("application/json") {
return ResponseKind::Json(
schema_name_from_content(content).unwrap_or_else(|| "*".to_string()),
);
}
if let Some(content) = response.content.get("text/plain; charset=utf-8") {
let schema = schema_name_from_content(content).map(|name| {
let is_numeric = is_numeric_schema(spec, &name);
TextSchema { name, is_numeric }
});
return ResponseKind::Text(schema);
}
ResponseKind::Text(None)
}
fn schema_name_from_content(content: &oas3::spec::MediaType) -> Option<String> {
schema_type_from_schema(content.schema.as_ref()?)
}
/// Resolves `name` against `components.schemas` and reports whether the
/// underlying primitive is `integer` or `number`.
fn is_numeric_schema(spec: &Spec, name: &str) -> bool {
let Some(components) = spec.components.as_ref() else {
return false;
};
let Some(Schema::Object(obj_or_ref)) = components.schemas.get(name) else {
return false;
};
let ObjectOrReference::Object(schema) = obj_or_ref.as_ref() else {
return false;
};
matches!(
schema.schema_type.as_ref(),
Some(SchemaTypeSet::Single(
SchemaType::Integer | SchemaType::Number
))
)
}
fn schema_type_from_schema(schema: &Schema) -> Option<String> {
@@ -337,19 +328,21 @@ fn schema_to_type_name(schema: &ObjectSchema) -> Option<String> {
let types: Vec<String> = variants
.iter()
.filter_map(|v| match v {
ObjectOrReference::Ref { ref_path, .. } => {
ref_to_type_name(ref_path).map(|s| s.to_string())
}
ObjectOrReference::Object(obj) => {
// Skip null variants
if matches!(
obj.schema_type.as_ref(),
Some(SchemaTypeSet::Single(SchemaType::Null))
) {
return None;
Schema::Boolean(_) => None,
Schema::Object(obj_or_ref) => match obj_or_ref.as_ref() {
ObjectOrReference::Ref { ref_path, .. } => {
ref_to_type_name(ref_path).map(|s| s.to_string())
}
schema_to_type_name(obj)
}
ObjectOrReference::Object(obj) => {
if matches!(
obj.schema_type.as_ref(),
Some(SchemaTypeSet::Single(SchemaType::Null))
) {
return None;
}
schema_to_type_name(obj)
}
},
})
.collect();
@@ -364,7 +357,7 @@ fn single_type_to_name(t: &SchemaType, schema: &ObjectSchema) -> Option<String>
match t {
SchemaType::String => Some("string".to_string()),
SchemaType::Number => Some("number".to_string()),
SchemaType::Integer => Some("number".to_string()),
SchemaType::Integer => Some("integer".to_string()),
SchemaType::Boolean => Some("boolean".to_string()),
SchemaType::Array => {
let inner = match &schema.items {
@@ -0,0 +1,8 @@
/// Parameter information.
#[derive(Debug, Clone)]
pub struct Parameter {
pub name: String,
pub required: bool,
pub param_type: String,
pub description: Option<String>,
}
@@ -0,0 +1,29 @@
use crate::openapi::TextSchema;
/// 200-response body shape.
#[derive(Debug, Clone)]
pub enum ResponseKind {
/// JSON body, schema named (e.g. "Block").
Json(String),
/// `text/plain` body. `Some(schema)` carries a typed shape (e.g. "Height", "Hex");
/// `None` is the escape hatch for opaque text.
Text(Option<TextSchema>),
/// `application/octet-stream`.
Binary,
}
impl ResponseKind {
/// Schema name, if the body is named (Json or typed Text).
pub fn schema_name(&self) -> Option<&str> {
match self {
Self::Json(s) => Some(s.as_str()),
Self::Text(Some(t)) => Some(t.name.as_str()),
_ => None,
}
}
/// True when a typed text body needs numeric parsing (`int(...)` etc.).
pub fn text_is_numeric(&self) -> bool {
matches!(self, Self::Text(Some(t)) if t.is_numeric)
}
}
@@ -0,0 +1,8 @@
/// Schema metadata for a typed `text/plain` response.
#[derive(Debug, Clone)]
pub struct TextSchema {
/// Schema name, e.g. "Height", "Hex".
pub name: String,
/// True when the underlying primitive is `integer`/`number` (body needs numeric parsing).
pub is_numeric: bool,
}
+1 -1
View File
@@ -17,7 +17,7 @@ brk_logger = { workspace = true }
brk_mempool = { workspace = true }
brk_query = { workspace = true }
brk_reader = { workspace = true }
brk_rpc = { workspace = true, features = ["corepc"] }
brk_rpc = { workspace = true }
brk_server = { workspace = true }
brk_types = { workspace = true }
lexopt = "0.3"
+3 -2
View File
@@ -19,10 +19,11 @@ BRK uses [sparse files](https://en.wikipedia.org/wiki/Sparse_file). Tools like `
## Install
```bash
rustup update
RUSTFLAGS="-C target-cpu=native" cargo install --locked brk_cli
rustup update && RUSTFLAGS="-C target-cpu=native" cargo install --locked brk_cli --version $(cargo search brk_cli | head -1 | awk -F'"' '{print $2}')
```
Updates Rust, then builds `brk_cli` with optimizations tuned to your CPU. The `--version $(...)` subshell queries crates.io for the absolute latest published version, including pre-releases (rc/beta/alpha); without it, `cargo install` only picks the latest stable.
Portable build (without native CPU optimizations):
```bash
+82 -28
View File
@@ -1,47 +1,57 @@
use std::{
fs,
fs, io,
path::{Path, PathBuf},
};
use brk_error::{Error, Result};
use brk_rpc::{Auth, Client};
use brk_server::Website;
use brk_server::{CdnCacheMode, DEFAULT_MAX_UTXOS, DEFAULT_MAX_WEIGHT, Website};
use brk_types::Port;
use owo_colors::OwoColorize;
use serde::{Deserialize, Deserializer, Serialize};
use serde::{Deserialize, Serialize};
use crate::{default_brk_path, dot_brk_path, fix_user_path};
#[derive(Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct Config {
#[serde(default, deserialize_with = "default_on_error")]
#[serde(default)]
brkdir: Option<String>,
#[serde(default, deserialize_with = "default_on_error")]
#[serde(default)]
brkport: Option<Port>,
#[serde(default, deserialize_with = "default_on_error")]
#[serde(default)]
website: Option<Website>,
#[serde(default, deserialize_with = "default_on_error")]
#[serde(default)]
cdn: Option<bool>,
#[serde(default)]
maxweight: Option<usize>,
#[serde(default)]
maxutxos: Option<usize>,
#[serde(default)]
bitcoindir: Option<String>,
#[serde(default, deserialize_with = "default_on_error")]
#[serde(default)]
blocksdir: Option<String>,
#[serde(default, deserialize_with = "default_on_error")]
#[serde(default)]
rpcconnect: Option<String>,
#[serde(default, deserialize_with = "default_on_error")]
#[serde(default)]
rpcport: Option<u16>,
#[serde(default, deserialize_with = "default_on_error")]
#[serde(default)]
rpccookiefile: Option<String>,
#[serde(default, deserialize_with = "default_on_error")]
#[serde(default)]
rpcuser: Option<String>,
#[serde(default, deserialize_with = "default_on_error")]
#[serde(default)]
rpcpassword: Option<String>,
}
@@ -66,6 +76,15 @@ impl Config {
if let Some(v) = config_args.website {
config.website = Some(v);
}
if let Some(v) = config_args.cdn {
config.cdn = Some(v);
}
if let Some(v) = config_args.maxweight {
config.maxweight = Some(v);
}
if let Some(v) = config_args.maxutxos {
config.maxutxos = Some(v);
}
if let Some(v) = config_args.bitcoindir {
config.bitcoindir = Some(v);
}
@@ -112,6 +131,13 @@ impl Config {
Long("brkdir") => config.brkdir = Some(parser.value().unwrap().parse().unwrap()),
Long("brkport") => config.brkport = Some(parser.value().unwrap().parse().unwrap()),
Long("website") => config.website = Some(parser.value().unwrap().parse().unwrap()),
Long("cdn") => config.cdn = Some(parser.value().unwrap().parse().unwrap()),
Long("maxweight") => {
config.maxweight = Some(parser.value().unwrap().parse().unwrap())
}
Long("maxutxos") => {
config.maxutxos = Some(parser.value().unwrap().parse().unwrap())
}
Long("bitcoindir") => {
config.bitcoindir = Some(parser.value().unwrap().parse().unwrap())
}
@@ -171,6 +197,21 @@ impl Config {
"<BOOL|PATH>".bright_black(),
"[true]".bright_black()
);
println!(
" --cdn {} Aggressive CDN cache, requires purge on deploy {}",
"<BOOL>".bright_black(),
"[false]".bright_black()
);
println!(
" --maxweight {} Server cap on series response weight in bytes; rejects /api/{{series,metric}}/... over the limit {}",
"<BYTES>".bright_black(),
format!("[{}]", DEFAULT_MAX_WEIGHT).bright_black()
);
println!(
" --maxutxos {} Server cap on UTXOs per address; /api/address/{{addr}}/utxo errors past the limit {}",
"<COUNT>".bright_black(),
format!("[{}]", DEFAULT_MAX_UTXOS).bright_black()
);
println!();
println!(
" --bitcoindir {} Bitcoin directory {}",
@@ -263,10 +304,18 @@ Finally, you can run the program with '-h' for help."
}
fn read(path: &Path) -> Self {
fs::read_to_string(path).map_or_else(
|_| Config::default(),
|contents| toml::from_str(&contents).unwrap_or_default(),
)
let contents = match fs::read_to_string(path) {
Ok(contents) => contents,
Err(e) if e.kind() == io::ErrorKind::NotFound => return Config::default(),
Err(e) => {
eprintln!("Cannot read {}: {e}", path.display());
std::process::exit(1);
}
};
toml::from_str(&contents).unwrap_or_else(|e| {
eprintln!("Invalid {}:\n{e}", path.display());
std::process::exit(1);
})
}
pub fn rpc(&self) -> Result<Client> {
@@ -333,18 +382,23 @@ Finally, you can run the program with '-h' for help."
self.website.clone().unwrap_or_default()
}
pub fn cdn_cache_mode(&self) -> CdnCacheMode {
if self.cdn.unwrap_or(false) {
CdnCacheMode::Aggressive
} else {
CdnCacheMode::Live
}
}
pub fn max_weight(&self) -> usize {
self.maxweight.unwrap_or(DEFAULT_MAX_WEIGHT)
}
pub fn max_utxos(&self) -> usize {
self.maxutxos.unwrap_or(DEFAULT_MAX_UTXOS)
}
pub fn brkport(&self) -> Option<Port> {
self.brkport
}
}
fn default_on_error<'de, D, T>(deserializer: D) -> Result<T, D::Error>
where
D: Deserializer<'de>,
T: Deserialize<'de> + Default,
{
match T::deserialize(deserializer) {
Ok(v) => Ok(v),
Err(_) => Ok(T::default()),
}
}
+21 -14
View File
@@ -13,7 +13,7 @@ use brk_indexer::Indexer;
use brk_mempool::Mempool;
use brk_query::AsyncQuery;
use brk_reader::Reader;
use brk_server::Server;
use brk_server::{Server, ServerConfig};
use tracing::info;
use vecdb::Exit;
@@ -42,7 +42,7 @@ pub fn main() -> anyhow::Result<()> {
{
// Pre-run indexer if too far behind, then drop and reimport to reduce memory
let chain_height = client.get_last_height()?;
let indexed_height = indexer.vecs.starting_height();
let indexed_height = indexer.vecs.next_height();
let blocks_behind = chain_height.saturating_sub(*indexed_height);
if blocks_behind > 10_000 {
info!("---");
@@ -60,21 +60,26 @@ pub fn main() -> anyhow::Result<()> {
let mempool = Mempool::new(&client);
let query = AsyncQuery::build(&reader, &indexer, &computer, Some(mempool.clone()));
let mempool_clone = mempool.clone();
let resolver = query.sync(|q| q.indexer_prevout_resolver());
thread::spawn(move || {
mempool_clone.start();
mempool_clone.start_with(resolver);
});
let query = AsyncQuery::build(&reader, &indexer, &computer, Some(mempool));
let data_path = config.brkdir();
let website = config.website();
let server_config = ServerConfig {
data_path: config.brkdir(),
website: config.website(),
cdn_cache_mode: config.cdn_cache_mode(),
max_weight: config.max_weight(),
max_utxos: config.max_utxos(),
};
let port = config.brkport();
let future = async move {
let server = Server::new(&query, data_path, website);
let server = Server::new(&query, server_config);
tokio::spawn(async move {
server.serve(port).await.unwrap();
@@ -98,15 +103,17 @@ pub fn main() -> anyhow::Result<()> {
let total_start = Instant::now();
let starting_indexes = if cfg!(debug_assertions) {
indexer.checked_index(&reader, &client, &exit)?
if cfg!(debug_assertions) {
indexer.checked_index(&reader, &client, &exit)?;
} else {
indexer.index(&reader, &client, &exit)?
};
indexer.index(&reader, &client, &exit)?;
}
Mimalloc::collect();
computer.compute(&indexer, starting_indexes, &exit)?;
computer.compute(&indexer, &exit)?;
indexer.advance_safe_lengths()?;
info!("Total time: {:?}", total_start.elapsed());
info!("Waiting for new blocks...");
+1 -1
View File
@@ -6,7 +6,7 @@ pub fn dot_brk_path() -> PathBuf {
}
pub fn dot_brk_log_path() -> PathBuf {
dot_brk_path().join("log")
dot_brk_path().join("logs")
}
pub fn default_brk_path() -> PathBuf {
+2930 -5796
View File
File diff suppressed because it is too large Load Diff
+59 -2
View File
@@ -1,16 +1,34 @@
use std::ops::{Add, AddAssign};
use brk_traversable::Traversable;
use brk_types::OutputType;
use rayon::prelude::*;
use super::{SpendableType, UnspendableType};
use super::{Filter, SpendableType, UnspendableType};
#[derive(Default, Clone, Debug)]
pub const OP_RETURN: &str = "op_return";
#[derive(Default, Clone, Debug, Traversable)]
pub struct ByType<T> {
#[traversable(flatten)]
pub spendable: SpendableType<T>,
#[traversable(flatten)]
pub unspendable: UnspendableType<T>,
}
impl<T> ByType<T> {
pub fn try_new<F, E>(mut create: F) -> Result<Self, E>
where
F: FnMut(Filter, &'static str) -> Result<T, E>,
{
Ok(Self {
spendable: SpendableType::try_new(&mut create)?,
unspendable: UnspendableType {
op_return: create(Filter::Type(OutputType::OpReturn), OP_RETURN)?,
},
})
}
pub fn get(&self, output_type: OutputType) -> &T {
match output_type {
OutputType::P2PK65 => &self.spendable.p2pk65,
@@ -44,6 +62,45 @@ impl<T> ByType<T> {
OutputType::OpReturn => &mut self.unspendable.op_return,
}
}
pub fn iter(&self) -> impl Iterator<Item = &T> {
self.spendable
.iter()
.chain(std::iter::once(&self.unspendable.op_return))
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
self.spendable
.iter_mut()
.chain(std::iter::once(&mut self.unspendable.op_return))
}
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
let Self {
spendable,
unspendable,
} = self;
spendable
.par_iter_mut()
.chain([&mut unspendable.op_return].into_par_iter())
}
pub fn iter_typed(&self) -> impl Iterator<Item = (OutputType, &T)> {
self.spendable.iter_typed().chain(std::iter::once((
OutputType::OpReturn,
&self.unspendable.op_return,
)))
}
pub fn iter_typed_mut(&mut self) -> impl Iterator<Item = (OutputType, &mut T)> {
self.spendable.iter_typed_mut().chain(std::iter::once((
OutputType::OpReturn,
&mut self.unspendable.op_return,
)))
}
}
impl<T> Add for ByType<T>
+17
View File
@@ -116,6 +116,23 @@ impl<T> SpendableType<T> {
})
}
pub fn get(&self, output_type: OutputType) -> &T {
match output_type {
OutputType::P2PK65 => &self.p2pk65,
OutputType::P2PK33 => &self.p2pk33,
OutputType::P2PKH => &self.p2pkh,
OutputType::P2MS => &self.p2ms,
OutputType::P2SH => &self.p2sh,
OutputType::P2WPKH => &self.p2wpkh,
OutputType::P2WSH => &self.p2wsh,
OutputType::P2TR => &self.p2tr,
OutputType::P2A => &self.p2a,
OutputType::Unknown => &self.unknown,
OutputType::Empty => &self.empty,
_ => unreachable!(),
}
}
pub fn get_mut(&mut self, output_type: OutputType) -> &mut T {
match output_type {
OutputType::P2PK65 => &mut self.p2pk65,
+1 -1
View File
@@ -15,7 +15,7 @@ brk_cohort = { workspace = true }
brk_indexer = { workspace = true }
brk_oracle = { workspace = true }
brk_logger = { workspace = true }
brk_rpc = { workspace = true, features = ["corepc"] }
brk_rpc = { workspace = true }
brk_traversable = { workspace = true }
brk_types = { workspace = true }
derive_more = { workspace = true }
+1 -1
View File
@@ -22,7 +22,7 @@ Compute 1000+ on-chain metrics from indexed blockchain data: supply breakdowns,
let mut computer = Computer::forced_import(&outputs_path, &indexer)?;
// Compute all metrics for new blocks
computer.compute(&indexer, starting_indexes, &reader, &exit)?;
computer.compute(&indexer, &exit)?;
// Access computed data via traversable vecs
let supply = computer.distribution.utxo_cohorts.all.metrics.supply.total.sats.height;
+3 -3
View File
@@ -37,7 +37,7 @@ pub fn main() -> color_eyre::Result<()> {
// Pre-run indexer if too far behind, then drop and reimport to reduce memory
let chain_height = client.get_last_height()?;
let indexed_height = indexer.vecs.starting_height();
let indexed_height = indexer.vecs.next_height();
if u32::from(chain_height).saturating_sub(u32::from(indexed_height)) > 1000 {
indexer.checked_index(&reader, &client, &exit)?;
drop(indexer);
@@ -49,11 +49,11 @@ pub fn main() -> color_eyre::Result<()> {
loop {
let i = Instant::now();
let starting_indexes = indexer.checked_index(&reader, &client, &exit)?;
indexer.checked_index(&reader, &client, &exit)?;
Mimalloc::collect();
computer.compute(&indexer, starting_indexes, &exit)?;
computer.compute(&indexer, &exit)?;
dbg!(i.elapsed());
sleep(Duration::from_secs(10));
}
@@ -44,13 +44,13 @@ pub fn main() -> Result<()> {
});
let i = Instant::now();
let starting_indexes = indexer.index(&reader, &client, &exit)?;
indexer.index(&reader, &client, &exit)?;
info!("Done in {:?}", i.elapsed());
Mimalloc::collect();
let i = Instant::now();
computer.compute(&indexer, starting_indexes, &exit)?;
computer.compute(&indexer, &exit)?;
info!("Done in {:?}", i.elapsed());
// We want to benchmark the drop too
+3 -3
View File
@@ -48,7 +48,7 @@ pub fn main() -> color_eyre::Result<()> {
// Pre-run indexer if too far behind, then drop and reimport to reduce memory
let chain_height = client.get_last_height()?;
let indexed_height = indexer.vecs.starting_height();
let indexed_height = indexer.vecs.next_height();
if chain_height.saturating_sub(*indexed_height) > 1000 {
indexer.index(&reader, &client, &exit)?;
drop(indexer);
@@ -60,13 +60,13 @@ pub fn main() -> color_eyre::Result<()> {
loop {
let i = Instant::now();
let starting_indexes = indexer.index(&reader, &client, &exit)?;
indexer.index(&reader, &client, &exit)?;
info!("Done in {:?}", i.elapsed());
Mimalloc::collect();
let i = Instant::now();
computer.compute(&indexer, starting_indexes, &exit)?;
computer.compute(&indexer, &exit)?;
info!("Done in {:?}", i.elapsed());
sleep(Duration::from_secs(60));
+7 -9
View File
@@ -2,7 +2,6 @@ use std::thread;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::Indexes;
use vecdb::Exit;
use crate::indexes;
@@ -14,13 +13,12 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.db.sync_bg_tasks()?;
// lookback depends on indexes.timestamp.monotonic
self.lookback.compute(indexes, starting_indexes, exit)?;
self.lookback.compute(indexer, indexes, exit)?;
// Parallel: remaining sub-modules are independent of each other.
// size depends on lookback (already computed above).
@@ -35,12 +33,12 @@ impl Vecs {
..
} = self;
thread::scope(|s| -> Result<()> {
let r1 = s.spawn(|| count.compute(indexer, starting_indexes, exit));
let r2 = s.spawn(|| interval.compute(indexer, starting_indexes, exit));
let r3 = s.spawn(|| weight.compute(indexer, starting_indexes, exit));
let r4 = s.spawn(|| difficulty.compute(indexer, indexes, starting_indexes, exit));
let r5 = s.spawn(|| halving.compute(indexes, starting_indexes, exit));
size.compute(indexer, &*lookback, starting_indexes, exit)?;
let r1 = s.spawn(|| count.compute(indexer, exit));
let r2 = s.spawn(|| interval.compute(indexer, exit));
let r3 = s.spawn(|| weight.compute(indexer, exit));
let r4 = s.spawn(|| difficulty.compute(indexer, indexes, exit));
let r5 = s.spawn(|| halving.compute(indexer, indexes, exit));
size.compute(indexer, &*lookback, exit)?;
r1.join().unwrap()?;
r2.join().unwrap()?;
r3.join().unwrap()?;
@@ -1,25 +1,20 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{Indexes, StoredU32};
use brk_types::StoredU32;
use vecdb::Exit;
use super::Vecs;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
// Block count raw + cumulative
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
self.total.block.compute_range(
starting_indexes.height,
starting_height,
&indexer.vecs.blocks.weight,
|h| (h, StoredU32::from(1_u32)),
exit,
)?;
self.total.compute_rest(starting_indexes.height, exit)?;
self.total.compute_rest(starting_height, exit)?;
Ok(())
}
@@ -7,7 +7,7 @@ use crate::{
indexes,
internal::{
BlockCountTarget1m, BlockCountTarget1w, BlockCountTarget1y, BlockCountTarget24h,
CachedWindowStarts, ConstantVecs, PerBlockCumulativeRolling, Windows,
ConstantVecs, PerBlockCumulativeRolling, WindowStartVec, Windows,
},
};
@@ -16,7 +16,7 @@ impl Vecs {
db: &Database,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
Ok(Self {
target: Windows {
@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{Indexes, StoredU32};
use brk_types::StoredU32;
use vecdb::Exit;
use super::Vecs;
@@ -11,26 +11,26 @@ impl Vecs {
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
self.adjustment.bps.height.compute_ratio_change(
starting_indexes.height,
starting_height,
&indexer.vecs.blocks.difficulty,
2016,
exit,
)?;
self.epoch.height.compute_transform(
starting_indexes.height,
starting_height,
&indexes.height.epoch,
|(h, epoch, ..)| (h, epoch),
exit,
)?;
self.blocks_to_retarget.height.compute_transform(
starting_indexes.height,
&indexes.height.identity,
starting_height,
&indexes.height.epoch,
|(h, ..)| (h, StoredU32::from(h.left_before_next_diff_adj())),
exit,
)?;
@@ -20,10 +20,10 @@ impl Vecs {
) -> Result<Self> {
let v2 = Version::TWO;
let hashrate = LazyPerBlock::from_height_source::<DifficultyToHashF64>(
let hashrate = LazyPerBlock::from_height_source::<DifficultyToHashF64, _>(
"difficulty_hashrate",
version,
indexer.vecs.blocks.difficulty.read_only_boxed_clone(),
indexer.vecs.blocks.difficulty.read_only_clone(),
indexes,
);
@@ -40,7 +40,7 @@ impl Vecs {
Ok(Self {
value: Resolutions::forced_import(
"difficulty",
indexer.vecs.blocks.difficulty.read_only_boxed_clone(),
indexer.vecs.blocks.difficulty.read_only_clone(),
version,
indexes,
),
@@ -1,5 +1,6 @@
use brk_error::Result;
use brk_types::{Indexes, StoredU32};
use brk_indexer::Indexer;
use brk_types::StoredU32;
use vecdb::Exit;
use super::Vecs;
@@ -8,20 +9,21 @@ use crate::indexes;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
self.epoch.height.compute_transform(
starting_indexes.height,
starting_height,
&indexes.height.halving,
|(h, epoch, ..)| (h, epoch),
exit,
)?;
self.blocks_to_halving.height.compute_transform(
starting_indexes.height,
&indexes.height.identity,
starting_height,
&indexes.height.halving,
|(h, ..)| (h, StoredU32::from(h.left_before_next_halving())),
exit,
)?;
+5 -5
View File
@@ -24,11 +24,11 @@ impl Vecs {
let version = parent_version;
let lookback = LookbackVecs::forced_import(&db, version)?;
let cached_starts = &lookback.cached_window_starts;
let count = CountVecs::forced_import(&db, version, indexes, cached_starts)?;
let interval = IntervalVecs::forced_import(&db, version, indexes, cached_starts)?;
let size = SizeVecs::forced_import(&db, version, indexes, cached_starts)?;
let weight = WeightVecs::forced_import(&db, version, indexes, cached_starts, &size)?;
let cached_starts = lookback.cached_window_starts();
let count = CountVecs::forced_import(&db, version, indexes, &cached_starts)?;
let interval = IntervalVecs::forced_import(&db, version, indexes, &cached_starts)?;
let size = SizeVecs::forced_import(&db, version, indexes, &cached_starts)?;
let weight = WeightVecs::forced_import(&db, version, indexes, &cached_starts, &size)?;
let difficulty = DifficultyVecs::forced_import(&db, version, indexer, indexes)?;
let halving = HalvingVecs::forced_import(&db, version, indexes)?;
@@ -1,21 +1,17 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{CheckedSub, Indexes, Timestamp};
use brk_types::{CheckedSub, Timestamp};
use vecdb::{Exit, ReadableVec};
use super::Vecs;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let mut prev_timestamp = None;
self.0.compute(starting_indexes.height, exit, |vec| {
self.0.compute(starting_height, exit, |vec| {
vec.compute_transform(
starting_indexes.height,
starting_height,
&indexer.vecs.blocks.timestamp,
|(h, timestamp, ..)| {
let interval = if let Some(prev_h) = h.decremented() {
@@ -5,7 +5,7 @@ use vecdb::Database;
use super::Vecs;
use crate::{
indexes,
internal::{CachedWindowStarts, PerBlockRollingAverage},
internal::{PerBlockRollingAverage, WindowStartVec, Windows},
};
impl Vecs {
@@ -13,7 +13,7 @@ impl Vecs {
db: &Database,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
let interval = PerBlockRollingAverage::forced_import(
db,
@@ -8,5 +8,5 @@ use crate::internal::PerBlockRollingAverage;
#[derive(Deref, DerefMut, Traversable)]
pub struct Vecs<M: StorageMode = Rw>(
#[traversable(flatten)] pub PerBlockRollingAverage<Timestamp, M>,
#[traversable(flatten)] pub PerBlockRollingAverage<Timestamp, Timestamp, M>,
);
+82 -89
View File
@@ -1,6 +1,7 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_traversable::Traversable;
use brk_types::{Height, Indexes, Timestamp, Version};
use brk_types::{Height, Timestamp, Version};
use vecdb::{
AnyVec, CachedVec, Cursor, Database, EagerVec, Exit, ImportableVec, PcoVec, ReadableVec, Rw,
StorageMode, VecIndex,
@@ -8,17 +9,15 @@ use vecdb::{
use crate::{
indexes,
internal::{CachedWindowStarts, WindowStarts, Windows},
internal::{WindowStartVec, WindowStarts, Windows},
};
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
#[traversable(skip)]
pub cached_window_starts: CachedWindowStarts,
pub _1h: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _24h: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 1d
pub _24h: CachedVec<M::Stored<EagerVec<PcoVec<Height, Height>>>>, // 1d
pub _3d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _1w: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 7d
pub _1w: CachedVec<M::Stored<EagerVec<PcoVec<Height, Height>>>>, // 7d
pub _8d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _9d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _12d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
@@ -26,7 +25,7 @@ pub struct Vecs<M: StorageMode = Rw> {
pub _2w: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 14d
pub _21d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _26d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _1m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 30d
pub _1m: CachedVec<M::Stored<EagerVec<PcoVec<Height, Height>>>>, // 30d
pub _34d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _55d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _2m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 60d
@@ -43,7 +42,7 @@ pub struct Vecs<M: StorageMode = Rw> {
pub _9m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 270d
pub _350d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _12m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 360d
pub _1y: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 365d
pub _1y: CachedVec<M::Stored<EagerVec<PcoVec<Height, Height>>>>, // 365d
pub _14m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 420d
pub _2y: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 730d
pub _26m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 780d
@@ -106,19 +105,11 @@ impl Vecs {
let _14y = ImportableVec::forced_import(db, "height_14y_ago", version)?;
let _26y = ImportableVec::forced_import(db, "height_26y_ago", version)?;
let cached_window_starts = CachedWindowStarts(Windows {
_24h: CachedVec::new(&_24h),
_1w: CachedVec::new(&_1w),
_1m: CachedVec::new(&_1m),
_1y: CachedVec::new(&_1y),
});
Ok(Self {
cached_window_starts,
_1h,
_24h,
_24h: CachedVec::wrap(_24h),
_3d,
_1w,
_1w: CachedVec::wrap(_1w),
_8d,
_9d,
_12d,
@@ -126,7 +117,7 @@ impl Vecs {
_2w,
_21d,
_26d,
_1m,
_1m: CachedVec::wrap(_1m),
_34d,
_55d,
_2m,
@@ -143,7 +134,7 @@ impl Vecs {
_9m,
_350d,
_12m,
_1y,
_1y: CachedVec::wrap(_1y),
_14m,
_2y,
_26m,
@@ -161,8 +152,8 @@ impl Vecs {
})
}
pub fn window_starts(&self) -> WindowStarts<'_> {
WindowStarts {
pub fn cached_window_starts(&self) -> Windows<&WindowStartVec> {
Windows {
_24h: &self._24h,
_1w: &self._1w,
_1m: &self._1m,
@@ -170,11 +161,20 @@ impl Vecs {
}
}
pub fn window_starts(&self) -> WindowStarts<'_> {
WindowStarts {
_24h: &self._24h.inner,
_1w: &self._1w.inner,
_1m: &self._1m.inner,
_1y: &self._1y.inner,
}
}
pub fn start_vec(&self, days: usize) -> &EagerVec<PcoVec<Height, Height>> {
match days {
1 => &self._24h,
1 => &self._24h.inner,
3 => &self._3d,
7 => &self._1w,
7 => &self._1w.inner,
8 => &self._8d,
9 => &self._9d,
12 => &self._12d,
@@ -182,7 +182,7 @@ impl Vecs {
14 => &self._2w,
21 => &self._21d,
26 => &self._26d,
30 => &self._1m,
30 => &self._1m.inner,
34 => &self._34d,
55 => &self._55d,
60 => &self._2m,
@@ -199,7 +199,7 @@ impl Vecs {
270 => &self._9m,
350 => &self._350d,
360 => &self._12m,
365 => &self._1y,
365 => &self._1y.inner,
420 => &self._14m,
730 => &self._2y,
780 => &self._26m,
@@ -220,53 +220,54 @@ impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
self.compute_rolling_start_hours(indexes, starting_indexes, exit, 1, |s| &mut s._1h)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 1, |s| &mut s._24h)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 3, |s| &mut s._3d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 7, |s| &mut s._1w)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 8, |s| &mut s._8d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 9, |s| &mut s._9d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 12, |s| &mut s._12d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 13, |s| &mut s._13d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 14, |s| &mut s._2w)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 21, |s| &mut s._21d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 26, |s| &mut s._26d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 30, |s| &mut s._1m)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 34, |s| &mut s._34d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 55, |s| &mut s._55d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 60, |s| &mut s._2m)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 63, |s| &mut s._9w)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 84, |s| &mut s._12w)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 89, |s| &mut s._89d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 90, |s| &mut s._3m)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 98, |s| &mut s._14w)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 111, |s| &mut s._111d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 144, |s| &mut s._144d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 180, |s| &mut s._6m)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 182, |s| &mut s._26w)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 200, |s| &mut s._200d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 270, |s| &mut s._9m)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 350, |s| &mut s._350d)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 360, |s| &mut s._12m)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 365, |s| &mut s._1y)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 420, |s| &mut s._14m)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 730, |s| &mut s._2y)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 780, |s| &mut s._26m)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 1095, |s| &mut s._3y)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 1400, |s| &mut s._200w)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 1460, |s| &mut s._4y)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 1825, |s| &mut s._5y)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 2190, |s| &mut s._6y)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 2920, |s| &mut s._8y)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 3285, |s| &mut s._9y)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 3650, |s| &mut s._10y)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 4380, |s| &mut s._12y)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 5110, |s| &mut s._14y)?;
self.compute_rolling_start(indexes, starting_indexes, exit, 9490, |s| &mut s._26y)?;
let starting_height = indexer.safe_lengths().height;
self.compute_rolling_start_hours(indexes, starting_height, exit, 1, |s| &mut s._1h)?;
self.compute_rolling_start(indexes, starting_height, exit, 1, |s| &mut s._24h.inner)?;
self.compute_rolling_start(indexes, starting_height, exit, 3, |s| &mut s._3d)?;
self.compute_rolling_start(indexes, starting_height, exit, 7, |s| &mut s._1w.inner)?;
self.compute_rolling_start(indexes, starting_height, exit, 8, |s| &mut s._8d)?;
self.compute_rolling_start(indexes, starting_height, exit, 9, |s| &mut s._9d)?;
self.compute_rolling_start(indexes, starting_height, exit, 12, |s| &mut s._12d)?;
self.compute_rolling_start(indexes, starting_height, exit, 13, |s| &mut s._13d)?;
self.compute_rolling_start(indexes, starting_height, exit, 14, |s| &mut s._2w)?;
self.compute_rolling_start(indexes, starting_height, exit, 21, |s| &mut s._21d)?;
self.compute_rolling_start(indexes, starting_height, exit, 26, |s| &mut s._26d)?;
self.compute_rolling_start(indexes, starting_height, exit, 30, |s| &mut s._1m.inner)?;
self.compute_rolling_start(indexes, starting_height, exit, 34, |s| &mut s._34d)?;
self.compute_rolling_start(indexes, starting_height, exit, 55, |s| &mut s._55d)?;
self.compute_rolling_start(indexes, starting_height, exit, 60, |s| &mut s._2m)?;
self.compute_rolling_start(indexes, starting_height, exit, 63, |s| &mut s._9w)?;
self.compute_rolling_start(indexes, starting_height, exit, 84, |s| &mut s._12w)?;
self.compute_rolling_start(indexes, starting_height, exit, 89, |s| &mut s._89d)?;
self.compute_rolling_start(indexes, starting_height, exit, 90, |s| &mut s._3m)?;
self.compute_rolling_start(indexes, starting_height, exit, 98, |s| &mut s._14w)?;
self.compute_rolling_start(indexes, starting_height, exit, 111, |s| &mut s._111d)?;
self.compute_rolling_start(indexes, starting_height, exit, 144, |s| &mut s._144d)?;
self.compute_rolling_start(indexes, starting_height, exit, 180, |s| &mut s._6m)?;
self.compute_rolling_start(indexes, starting_height, exit, 182, |s| &mut s._26w)?;
self.compute_rolling_start(indexes, starting_height, exit, 200, |s| &mut s._200d)?;
self.compute_rolling_start(indexes, starting_height, exit, 270, |s| &mut s._9m)?;
self.compute_rolling_start(indexes, starting_height, exit, 350, |s| &mut s._350d)?;
self.compute_rolling_start(indexes, starting_height, exit, 360, |s| &mut s._12m)?;
self.compute_rolling_start(indexes, starting_height, exit, 365, |s| &mut s._1y.inner)?;
self.compute_rolling_start(indexes, starting_height, exit, 420, |s| &mut s._14m)?;
self.compute_rolling_start(indexes, starting_height, exit, 730, |s| &mut s._2y)?;
self.compute_rolling_start(indexes, starting_height, exit, 780, |s| &mut s._26m)?;
self.compute_rolling_start(indexes, starting_height, exit, 1095, |s| &mut s._3y)?;
self.compute_rolling_start(indexes, starting_height, exit, 1400, |s| &mut s._200w)?;
self.compute_rolling_start(indexes, starting_height, exit, 1460, |s| &mut s._4y)?;
self.compute_rolling_start(indexes, starting_height, exit, 1825, |s| &mut s._5y)?;
self.compute_rolling_start(indexes, starting_height, exit, 2190, |s| &mut s._6y)?;
self.compute_rolling_start(indexes, starting_height, exit, 2920, |s| &mut s._8y)?;
self.compute_rolling_start(indexes, starting_height, exit, 3285, |s| &mut s._9y)?;
self.compute_rolling_start(indexes, starting_height, exit, 3650, |s| &mut s._10y)?;
self.compute_rolling_start(indexes, starting_height, exit, 4380, |s| &mut s._12y)?;
self.compute_rolling_start(indexes, starting_height, exit, 5110, |s| &mut s._14y)?;
self.compute_rolling_start(indexes, starting_height, exit, 9490, |s| &mut s._26y)?;
Ok(())
}
@@ -274,7 +275,7 @@ impl Vecs {
fn compute_rolling_start<F>(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
starting_height: Height,
exit: &Exit,
days: usize,
get_field: F,
@@ -282,19 +283,15 @@ impl Vecs {
where
F: FnOnce(&mut Self) -> &mut EagerVec<PcoVec<Height, Height>>,
{
self.compute_rolling_start_inner(
indexes,
starting_indexes,
exit,
get_field,
|t, prev_ts| t.difference_in_days_between(prev_ts) >= days,
)
self.compute_rolling_start_inner(indexes, starting_height, exit, get_field, |t, prev_ts| {
t.difference_in_days_between(prev_ts) >= days
})
}
fn compute_rolling_start_hours<F>(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
starting_height: Height,
exit: &Exit,
hours: usize,
get_field: F,
@@ -302,19 +299,15 @@ impl Vecs {
where
F: FnOnce(&mut Self) -> &mut EagerVec<PcoVec<Height, Height>>,
{
self.compute_rolling_start_inner(
indexes,
starting_indexes,
exit,
get_field,
|t, prev_ts| t.difference_in_hours_between(prev_ts) >= hours,
)
self.compute_rolling_start_inner(indexes, starting_height, exit, get_field, |t, prev_ts| {
t.difference_in_hours_between(prev_ts) >= hours
})
}
fn compute_rolling_start_inner<F, D>(
&mut self,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
starting_height: Height,
exit: &Exit,
get_field: F,
expired: D,
@@ -324,7 +317,7 @@ impl Vecs {
D: Fn(Timestamp, Timestamp) -> bool,
{
let field = get_field(self);
let resume_from = field.len().min(starting_indexes.height.to_usize());
let resume_from = field.len().min(starting_height.to_usize());
let mut prev = if resume_from > 0 {
field.collect_one_at(resume_from - 1).unwrap()
} else {
@@ -334,7 +327,7 @@ impl Vecs {
cursor.advance(prev.to_usize());
let mut prev_ts = cursor.next().unwrap();
Ok(field.compute_transform(
starting_indexes.height,
starting_height,
&indexes.timestamp.monotonic,
|(h, t, ..)| {
while expired(t, prev_ts) {
@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{Indexes, StoredU64};
use brk_types::StoredU64;
use vecdb::Exit;
use super::Vecs;
@@ -11,25 +11,23 @@ impl Vecs {
&mut self,
indexer: &Indexer,
lookback: &blocks::LookbackVecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let window_starts = lookback.window_starts();
// vbytes = floor(weight / 4), stored at height level
self.vbytes
.compute(starting_indexes.height, &window_starts, exit, |height| {
.compute(starting_height, &window_starts, exit, |height| {
Ok(height.compute_transform(
starting_indexes.height,
starting_height,
&indexer.vecs.blocks.weight,
|(h, weight, ..)| (h, StoredU64::from(weight.to_vbytes_floor())),
exit,
)?)
})?;
// size from indexer total_size
self.size.compute(
starting_indexes.height,
starting_height,
&window_starts,
&indexer.vecs.blocks.total,
exit,
@@ -5,7 +5,7 @@ use vecdb::Database;
use super::Vecs;
use crate::{
indexes,
internal::{CachedWindowStarts, PerBlockFull, PerBlockRolling},
internal::{PerBlockFull, PerBlockRolling, WindowStartVec, Windows},
};
impl Vecs {
@@ -13,7 +13,7 @@ impl Vecs {
db: &Database,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
Ok(Self {
vbytes: PerBlockFull::forced_import(
@@ -1,19 +1,15 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{BasisPoints16, Indexes};
use brk_types::BasisPoints16;
use vecdb::Exit;
use super::Vecs;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
self.fullness.bps.compute_transform(
starting_indexes.height,
starting_height,
&indexer.vecs.blocks.weight,
|(h, weight, ..)| (h, BasisPoints16::from(weight.fullness())),
exit,
@@ -6,7 +6,7 @@ use super::Vecs;
use crate::{
blocks::SizeVecs,
indexes,
internal::{CachedWindowStarts, LazyPerBlockRolling, PercentVec, VBytesToWeight},
internal::{LazyPerBlockRolling, PercentVec, VBytesToWeight, WindowStartVec, Windows},
};
impl Vecs {
@@ -14,7 +14,7 @@ impl Vecs {
db: &Database,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts,
cached_starts: &Windows<&WindowStartVec>,
size: &SizeVecs,
) -> Result<Self> {
let weight = LazyPerBlockRolling::from_per_block_full::<VBytesToWeight>(
@@ -1,5 +1,6 @@
use brk_error::Result;
use brk_types::{Bitcoin, Indexes, StoredF64};
use brk_indexer::Indexer;
use brk_types::{Bitcoin, StoredF64};
use vecdb::Exit;
use super::Vecs;
@@ -8,17 +9,18 @@ use crate::distribution;
impl Vecs {
pub(crate) fn compute(
&mut self,
starting_indexes: &Indexes,
indexer: &Indexer,
distribution: &distribution::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let all_metrics = &distribution.utxo_cohorts.all.metrics;
let circulating_supply = &all_metrics.supply.total.sats.height;
self.coinblocks_created
.compute(starting_indexes.height, exit, |vec| {
.compute(starting_height, exit, |vec| {
vec.compute_transform(
starting_indexes.height,
starting_height,
circulating_supply,
|(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))),
exit,
@@ -27,9 +29,9 @@ impl Vecs {
})?;
self.coinblocks_stored
.compute(starting_indexes.height, exit, |vec| {
.compute(starting_height, exit, |vec| {
vec.compute_subtract(
starting_indexes.height,
starting_height,
&self.coinblocks_created.block,
&distribution.coinblocks_destroyed.block,
exit,
@@ -38,14 +40,14 @@ impl Vecs {
})?;
self.liveliness.height.compute_divide(
starting_indexes.height,
starting_height,
&distribution.coinblocks_destroyed.cumulative.height,
&self.coinblocks_created.cumulative.height,
exit,
)?;
self.ratio.height.compute_divide(
starting_indexes.height,
starting_height,
&self.liveliness.height,
&self.vaultedness.height,
exit,
@@ -6,7 +6,7 @@ use super::Vecs;
use crate::{
indexes,
internal::{
CachedWindowStarts, LazyPerBlock, OneMinusF64, PerBlock, PerBlockCumulativeRolling,
LazyPerBlock, OneMinusF64, PerBlock, PerBlockCumulativeRolling, WindowStartVec, Windows,
},
};
@@ -15,7 +15,7 @@ impl Vecs {
db: &Database,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
let liveliness = PerBlock::forced_import(db, "liveliness", version, indexes)?;
@@ -1,5 +1,6 @@
use brk_error::Result;
use brk_types::{BasisPointsSigned32, Indexes};
use brk_indexer::Indexer;
use brk_types::BasisPointsSigned32;
use vecdb::Exit;
use super::super::activity;
@@ -9,13 +10,15 @@ use crate::supply;
impl Vecs {
pub(crate) fn compute(
&mut self,
starting_indexes: &Indexes,
indexer: &Indexer,
supply: &supply::Vecs,
activity: &activity::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
self.inflation_rate.bps.height.compute_transform2(
starting_indexes.height,
starting_height,
&activity.ratio.height,
&supply.inflation_rate.bps.height,
|(h, a2vr, inflation, ..)| {
@@ -28,14 +31,14 @@ impl Vecs {
)?;
self.tx_velocity_native.height.compute_multiply(
starting_indexes.height,
starting_height,
&activity.ratio.height,
&supply.velocity.native.height,
exit,
)?;
self.tx_velocity_fiat.height.compute_multiply(
starting_indexes.height,
starting_height,
&activity.ratio.height,
&supply.velocity.fiat.height,
exit,
@@ -1,5 +1,6 @@
use brk_error::Result;
use brk_types::{Dollars, Indexes};
use brk_indexer::Indexer;
use brk_types::Dollars;
use vecdb::Exit;
use super::super::{activity, value};
@@ -10,40 +11,41 @@ impl Vecs {
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute(
&mut self,
starting_indexes: &Indexes,
indexer: &Indexer,
mining: &mining::Vecs,
distribution: &distribution::Vecs,
activity: &activity::Vecs,
value: &value::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_lengths = indexer.safe_lengths();
let all_metrics = &distribution.utxo_cohorts.all.metrics;
let realized_cap_cents = &all_metrics.realized.cap.cents.height;
let circulating_supply = &all_metrics.supply.total.btc.height;
self.thermo.cents.height.compute_transform(
starting_indexes.height,
starting_lengths.height,
&mining.rewards.subsidy.cumulative.cents.height,
|(i, v, ..)| (i, v),
exit,
)?;
self.investor.cents.height.compute_subtract(
starting_indexes.height,
starting_lengths.height,
realized_cap_cents,
&self.thermo.cents.height,
exit,
)?;
self.vaulted.cents.height.compute_multiply(
starting_indexes.height,
starting_lengths.height,
realized_cap_cents,
&activity.vaultedness.height,
exit,
)?;
self.active.cents.height.compute_multiply(
starting_indexes.height,
starting_lengths.height,
realized_cap_cents,
&activity.liveliness.height,
exit,
@@ -51,7 +53,7 @@ impl Vecs {
// cointime_cap = (cointime_value_destroyed_cumulative * circulating_supply) / coinblocks_stored_cumulative
self.cointime.cents.height.compute_transform3(
starting_indexes.height,
starting_lengths.height,
&value.destroyed.cumulative.height,
circulating_supply,
&activity.coinblocks_stored.cumulative.height,
@@ -67,7 +69,7 @@ impl Vecs {
// AVIV = active_cap / investor_cap
self.aviv.compute_ratio(
starting_indexes,
&starting_lengths,
&self.active.cents.height,
&self.investor.cents.height,
exit,
+10 -16
View File
@@ -1,5 +1,5 @@
use brk_error::Result;
use brk_types::Indexes;
use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
@@ -9,7 +9,7 @@ impl Vecs {
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute(
&mut self,
starting_indexes: &Indexes,
indexer: &Indexer,
prices: &prices::Vecs,
blocks: &blocks::Vecs,
mining: &mining::Vecs,
@@ -20,29 +20,23 @@ impl Vecs {
self.db.sync_bg_tasks()?;
// Activity computes first (liveliness, vaultedness, etc.)
self.activity
.compute(starting_indexes, distribution, exit)?;
self.activity.compute(indexer, distribution, exit)?;
// Phase 2: supply, adjusted, value are independent (all depend only on activity)
let (r1, r2) = rayon::join(
|| {
self.supply
.compute(starting_indexes, prices, distribution, &self.activity, exit)
.compute(indexer, prices, distribution, &self.activity, exit)
},
|| {
rayon::join(
|| {
self.adjusted
.compute(starting_indexes, supply_vecs, &self.activity, exit)
.compute(indexer, supply_vecs, &self.activity, exit)
},
|| {
self.value.compute(
starting_indexes,
prices,
distribution,
&self.activity,
exit,
)
self.value
.compute(indexer, prices, distribution, &self.activity, exit)
},
)
},
@@ -53,7 +47,7 @@ impl Vecs {
// Cap depends on activity + value
self.cap.compute(
starting_indexes,
indexer,
mining,
distribution,
&self.activity,
@@ -65,7 +59,7 @@ impl Vecs {
let (r3, r4) = rayon::join(
|| {
self.prices.compute(
starting_indexes,
indexer,
prices,
distribution,
&self.activity,
@@ -76,7 +70,7 @@ impl Vecs {
},
|| {
self.reserve_risk
.compute(starting_indexes, blocks, prices, &self.value, exit)
.compute(indexer, blocks, prices, &self.value, exit)
},
);
r3?;
+2 -2
View File
@@ -13,14 +13,14 @@ use super::{
ValueVecs, Vecs,
};
use crate::internal::CachedWindowStarts;
use crate::internal::{WindowStartVec, Windows};
impl Vecs {
pub(crate) fn forced_import(
parent_path: &Path,
parent_version: Version,
indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
let db = open_db(parent_path, DB_NAME, 250_000)?;
let version = parent_version;
@@ -1,5 +1,6 @@
use brk_error::Result;
use brk_types::{Cents, Indexes};
use brk_indexer::Indexer;
use brk_types::Cents;
use vecdb::Exit;
use super::super::{activity, cap, supply};
@@ -10,7 +11,7 @@ impl Vecs {
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute(
&mut self,
starting_indexes: &Indexes,
indexer: &Indexer,
prices: &prices::Vecs,
distribution: &distribution::Vecs,
activity: &activity::Vecs,
@@ -18,14 +19,15 @@ impl Vecs {
cap: &cap::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_lengths = indexer.safe_lengths();
let all_metrics = &distribution.utxo_cohorts.all.metrics;
let circulating_supply = &all_metrics.supply.total.btc.height;
let realized_price = &all_metrics.realized.price.cents.height;
self.vaulted
.compute_all(prices, starting_indexes, exit, |v| {
.compute_all(prices, &starting_lengths, exit, |v| {
Ok(v.compute_transform2(
starting_indexes.height,
starting_lengths.height,
realized_price,
&activity.vaultedness.height,
|(i, price, vaultedness, ..)| {
@@ -36,9 +38,9 @@ impl Vecs {
})?;
self.active
.compute_all(prices, starting_indexes, exit, |v| {
.compute_all(prices, &starting_lengths, exit, |v| {
Ok(v.compute_transform2(
starting_indexes.height,
starting_lengths.height,
realized_price,
&activity.liveliness.height,
|(i, price, liveliness, ..)| {
@@ -49,9 +51,9 @@ impl Vecs {
})?;
self.true_market_mean
.compute_all(prices, starting_indexes, exit, |v| {
.compute_all(prices, &starting_lengths, exit, |v| {
Ok(v.compute_transform2(
starting_indexes.height,
starting_lengths.height,
&cap.investor.cents.height,
&supply.active.btc.height,
|(i, cap_cents, supply_btc, ..)| {
@@ -62,9 +64,9 @@ impl Vecs {
})?;
self.cointime
.compute_all(prices, starting_indexes, exit, |v| {
.compute_all(prices, &starting_lengths, exit, |v| {
Ok(v.compute_transform2(
starting_indexes.height,
starting_lengths.height,
&cap.cointime.cents.height,
circulating_supply,
|(i, cap_cents, supply_btc, ..)| {
@@ -1,5 +1,6 @@
use brk_error::Result;
use brk_types::{Indexes, StoredF64};
use brk_indexer::Indexer;
use brk_types::StoredF64;
use vecdb::Exit;
use super::{super::value, Vecs};
@@ -8,21 +9,23 @@ use crate::{blocks, internal::algo::ComputeRollingMedianFromStarts, prices};
impl Vecs {
pub(crate) fn compute(
&mut self,
starting_indexes: &Indexes,
indexer: &Indexer,
blocks: &blocks::Vecs,
prices: &prices::Vecs,
value: &value::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
self.vocdd_median_1y.compute_rolling_median_from_starts(
starting_indexes.height,
starting_height,
&blocks.lookback._1y,
&value.vocdd.block,
exit,
)?;
self.hodl_bank.compute_cumulative_transformed_binary(
starting_indexes.height,
starting_height,
&prices.spot.usd.height,
&self.vocdd_median_1y,
|price, median| StoredF64::from(f64::from(price) - f64::from(median)),
@@ -30,7 +33,7 @@ impl Vecs {
)?;
self.value.height.compute_divide(
starting_indexes.height,
starting_height,
&prices.spot.usd.height,
&self.hodl_bank,
exit,
@@ -1,5 +1,5 @@
use brk_error::Result;
use brk_types::Indexes;
use brk_indexer::Indexer;
use vecdb::Exit;
use super::super::activity;
@@ -9,12 +9,13 @@ use crate::{distribution, prices};
impl Vecs {
pub(crate) fn compute(
&mut self,
starting_indexes: &Indexes,
indexer: &Indexer,
prices: &prices::Vecs,
distribution: &distribution::Vecs,
activity: &activity::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let circulating_supply = &distribution
.utxo_cohorts
.all
@@ -25,22 +26,21 @@ impl Vecs {
.height;
self.vaulted.sats.height.compute_multiply(
starting_indexes.height,
starting_height,
circulating_supply,
&activity.vaultedness.height,
exit,
)?;
self.active.sats.height.compute_multiply(
starting_indexes.height,
starting_height,
circulating_supply,
&activity.liveliness.height,
exit,
)?;
self.vaulted
.compute(prices, starting_indexes.height, exit)?;
self.active.compute(prices, starting_indexes.height, exit)?;
self.vaulted.compute(prices, starting_height, exit)?;
self.active.compute(prices, starting_height, exit)?;
Ok(())
}
@@ -3,7 +3,7 @@ use brk_types::Version;
use vecdb::Database;
use super::Vecs;
use crate::{indexes, internal::AmountPerBlock};
use crate::{indexes, internal::ValuePerBlock};
impl Vecs {
pub(crate) fn forced_import(
@@ -12,8 +12,8 @@ impl Vecs {
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self {
vaulted: AmountPerBlock::forced_import(db, "vaulted_supply", version, indexes)?,
active: AmountPerBlock::forced_import(db, "active_supply", version, indexes)?,
vaulted: ValuePerBlock::forced_import(db, "vaulted_supply", version, indexes)?,
active: ValuePerBlock::forced_import(db, "active_supply", version, indexes)?,
})
}
}
@@ -1,10 +1,10 @@
use brk_traversable::Traversable;
use vecdb::{Rw, StorageMode};
use crate::internal::AmountPerBlock;
use crate::internal::ValuePerBlock;
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub vaulted: AmountPerBlock<M>,
pub active: AmountPerBlock<M>,
pub vaulted: ValuePerBlock<M>,
pub active: ValuePerBlock<M>,
}
@@ -1,5 +1,6 @@
use brk_error::Result;
use brk_types::{Bitcoin, Dollars, Indexes, StoredF64};
use brk_indexer::Indexer;
use brk_types::{Bitcoin, Dollars, StoredF64};
use vecdb::Exit;
use super::super::activity;
@@ -9,31 +10,31 @@ use crate::{distribution, prices};
impl Vecs {
pub(crate) fn compute(
&mut self,
starting_indexes: &Indexes,
indexer: &Indexer,
prices: &prices::Vecs,
distribution: &distribution::Vecs,
activity: &activity::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let all_metrics = &distribution.utxo_cohorts.all.metrics;
let coinblocks_destroyed = &distribution.coinblocks_destroyed;
let coindays_destroyed = &all_metrics.activity.coindays_destroyed;
let circulating_supply = &all_metrics.supply.total.btc.height;
self.destroyed
.compute(starting_indexes.height, exit, |vec| {
vec.compute_multiply(
starting_indexes.height,
&prices.spot.usd.height,
&coinblocks_destroyed.block,
exit,
)?;
Ok(())
})?;
self.created.compute(starting_indexes.height, exit, |vec| {
self.destroyed.compute(starting_height, exit, |vec| {
vec.compute_multiply(
starting_indexes.height,
starting_height,
&prices.spot.usd.height,
&coinblocks_destroyed.block,
exit,
)?;
Ok(())
})?;
self.created.compute(starting_height, exit, |vec| {
vec.compute_multiply(
starting_height,
&prices.spot.usd.height,
&activity.coinblocks_created.block,
exit,
@@ -41,9 +42,9 @@ impl Vecs {
Ok(())
})?;
self.stored.compute(starting_indexes.height, exit, |vec| {
self.stored.compute(starting_height, exit, |vec| {
vec.compute_multiply(
starting_indexes.height,
starting_height,
&prices.spot.usd.height,
&activity.coinblocks_stored.block,
exit,
@@ -54,9 +55,9 @@ impl Vecs {
// VOCDD: Value of Coin Days Destroyed = price × (CDD / circulating_supply)
// Supply-adjusted to account for growing supply over time
// This is a key input for Reserve Risk / HODL Bank calculation
self.vocdd.compute(starting_indexes.height, exit, |vec| {
self.vocdd.compute(starting_height, exit, |vec| {
vec.compute_transform3(
starting_indexes.height,
starting_height,
&prices.spot.usd.height,
&coindays_destroyed.block,
circulating_supply,
@@ -5,7 +5,7 @@ use vecdb::Database;
use super::Vecs;
use crate::{
indexes,
internal::{CachedWindowStarts, PerBlockCumulativeRolling},
internal::{PerBlockCumulativeRolling, WindowStartVec, Windows},
};
impl Vecs {
@@ -13,7 +13,7 @@ impl Vecs {
db: &Database,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
Ok(Self {
destroyed: PerBlockCumulativeRolling::forced_import(
@@ -7,19 +7,20 @@
//! | `receiving` | Unique addresses that received this block |
//! | `sending` | Unique addresses that sent this block |
//! | `reactivated` | Addresses that were empty and now have funds |
//! | `both` | Addresses that both sent AND received same block |
//! | `bidirectional` | Addresses that both sent AND received in same block |
//! | `active` | Distinct addresses involved (sent received) |
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, StoredU32, Version};
use brk_types::{Height, StoredU32, StoredU64, Version};
use derive_more::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, AnyVec, Database, Exit, Rw, StorageMode, WritableVec};
use crate::{
indexes,
internal::{CachedWindowStarts, PerBlockRollingAverage},
internal::{PerBlockRollingAverage, WindowStartVec, Windows},
};
/// Per-block activity counts - reset each block.
@@ -28,7 +29,7 @@ pub struct BlockActivityCounts {
pub reactivated: u32,
pub sending: u32,
pub receiving: u32,
pub both: u32,
pub bidirectional: u32,
}
impl BlockActivityCounts {
@@ -56,7 +57,7 @@ impl AddrTypeToActivityCounts {
total.reactivated += counts.reactivated;
total.sending += counts.sending;
total.receiving += counts.receiving;
total.both += counts.both;
total.bidirectional += counts.bidirectional;
}
total
}
@@ -65,45 +66,61 @@ impl AddrTypeToActivityCounts {
/// Activity count vectors for a single category (e.g., one address type or "all").
#[derive(Traversable)]
pub struct ActivityCountVecs<M: StorageMode = Rw> {
pub reactivated: PerBlockRollingAverage<StoredU32, M>,
pub sending: PerBlockRollingAverage<StoredU32, M>,
pub receiving: PerBlockRollingAverage<StoredU32, M>,
pub both: PerBlockRollingAverage<StoredU32, M>,
pub reactivated: PerBlockRollingAverage<StoredU32, StoredU64, M>,
pub sending: PerBlockRollingAverage<StoredU32, StoredU64, M>,
pub receiving: PerBlockRollingAverage<StoredU32, StoredU64, M>,
pub bidirectional: PerBlockRollingAverage<StoredU32, StoredU64, M>,
/// Distinct addresses involved in this block (sent received),
/// computed at push time as `sending + receiving - bidirectional`
/// via inclusion-exclusion. For per-type instances this is
/// per-type. For the `all` aggregate it's the cross-type total.
pub active: PerBlockRollingAverage<StoredU32, StoredU64, M>,
}
impl ActivityCountVecs {
/// `prefix` is prepended to each field's disk name. Use `""` for the
/// "all" aggregate and `"{type}_"` for per-address-type instances.
/// Field names are suffixed with `_addrs` so the final disk series
/// are e.g. `active_addrs`, `p2tr_bidirectional_addrs`.
pub(crate) fn forced_import(
db: &Database,
name: &str,
prefix: &str,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
Ok(Self {
reactivated: PerBlockRollingAverage::forced_import(
db,
&format!("{name}_reactivated"),
&format!("{prefix}reactivated_addrs"),
version,
indexes,
cached_starts,
)?,
sending: PerBlockRollingAverage::forced_import(
db,
&format!("{name}_sending"),
&format!("{prefix}sending_addrs"),
version,
indexes,
cached_starts,
)?,
receiving: PerBlockRollingAverage::forced_import(
db,
&format!("{name}_receiving"),
&format!("{prefix}receiving_addrs"),
version,
indexes,
cached_starts,
)?,
both: PerBlockRollingAverage::forced_import(
bidirectional: PerBlockRollingAverage::forced_import(
db,
&format!("{name}_both"),
&format!("{prefix}bidirectional_addrs"),
version,
indexes,
cached_starts,
)?,
active: PerBlockRollingAverage::forced_import(
db,
&format!("{prefix}active_addrs"),
version,
indexes,
cached_starts,
@@ -117,7 +134,8 @@ impl ActivityCountVecs {
.len()
.min(self.sending.block.len())
.min(self.receiving.block.len())
.min(self.both.block.len())
.min(self.bidirectional.block.len())
.min(self.active.block.len())
}
pub(crate) fn par_iter_height_mut(
@@ -125,9 +143,10 @@ impl ActivityCountVecs {
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.reactivated.block as &mut dyn AnyStoredVec,
&mut self.sending.block as &mut dyn AnyStoredVec,
&mut self.receiving.block as &mut dyn AnyStoredVec,
&mut self.both.block as &mut dyn AnyStoredVec,
&mut self.sending.block,
&mut self.receiving.block,
&mut self.bidirectional.block,
&mut self.active.block,
]
.into_par_iter()
}
@@ -136,7 +155,8 @@ impl ActivityCountVecs {
self.reactivated.block.reset()?;
self.sending.block.reset()?;
self.receiving.block.reset()?;
self.both.block.reset()?;
self.bidirectional.block.reset()?;
self.active.block.reset()?;
Ok(())
}
@@ -145,14 +165,17 @@ impl ActivityCountVecs {
self.reactivated.block.push(counts.reactivated.into());
self.sending.block.push(counts.sending.into());
self.receiving.block.push(counts.receiving.into());
self.both.block.push(counts.both.into());
self.bidirectional.block.push(counts.bidirectional.into());
let active = counts.sending + counts.receiving - counts.bidirectional;
self.active.block.push(active.into());
}
pub(crate) fn compute_rest(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
self.reactivated.compute_rest(max_from, exit)?;
self.sending.compute_rest(max_from, exit)?;
self.receiving.compute_rest(max_from, exit)?;
self.both.compute_rest(max_from, exit)?;
self.bidirectional.compute_rest(max_from, exit)?;
self.active.compute_rest(max_from, exit)?;
Ok(())
}
}
@@ -171,16 +194,15 @@ impl From<ByAddrType<ActivityCountVecs>> for AddrTypeToActivityCountVecs {
impl AddrTypeToActivityCountVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
Ok(Self::from(ByAddrType::<ActivityCountVecs>::new_with_name(
|type_name| {
ActivityCountVecs::forced_import(
db,
&format!("{type_name}_{name}"),
&format!("{type_name}_"),
version,
indexes,
cached_starts,
@@ -205,7 +227,8 @@ impl AddrTypeToActivityCountVecs {
vecs.push(&mut type_vecs.reactivated.block);
vecs.push(&mut type_vecs.sending.block);
vecs.push(&mut type_vecs.receiving.block);
vecs.push(&mut type_vecs.both.block);
vecs.push(&mut type_vecs.bidirectional.block);
vecs.push(&mut type_vecs.active.block);
}
vecs.into_par_iter()
}
@@ -243,16 +266,14 @@ pub struct AddrActivityVecs<M: StorageMode = Rw> {
impl AddrActivityVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
Ok(Self {
all: ActivityCountVecs::forced_import(db, name, version, indexes, cached_starts)?,
all: ActivityCountVecs::forced_import(db, "", version, indexes, cached_starts)?,
by_addr_type: AddrTypeToActivityCountVecs::forced_import(
db,
name,
version,
indexes,
cached_starts,
@@ -1,183 +0,0 @@
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, Indexes, StoredU64, Version};
use derive_more::{Deref, DerefMut};
use rayon::prelude::*;
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, PcoVec, ReadableVec, Rw, StorageMode,
WritableVec,
};
use crate::{indexes, internal::PerBlock};
#[derive(Deref, DerefMut, Traversable)]
pub struct AddrCountVecs<M: StorageMode = Rw>(#[traversable(flatten)] pub PerBlock<StoredU64, M>);
impl AddrCountVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self(PerBlock::forced_import(db, name, version, indexes)?))
}
}
/// Address count per address type (runtime state).
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddrTypeToAddrCount(ByAddrType<u64>);
impl AddrTypeToAddrCount {
#[inline]
pub(crate) fn sum(&self) -> u64 {
self.0.values().sum()
}
}
impl From<(&AddrTypeToAddrCountVecs, Height)> for AddrTypeToAddrCount {
#[inline]
fn from((groups, starting_height): (&AddrTypeToAddrCountVecs, Height)) -> Self {
if let Some(prev_height) = starting_height.decremented() {
Self(ByAddrType {
p2pk65: groups
.p2pk65
.height
.collect_one(prev_height)
.unwrap()
.into(),
p2pk33: groups
.p2pk33
.height
.collect_one(prev_height)
.unwrap()
.into(),
p2pkh: groups.p2pkh.height.collect_one(prev_height).unwrap().into(),
p2sh: groups.p2sh.height.collect_one(prev_height).unwrap().into(),
p2wpkh: groups
.p2wpkh
.height
.collect_one(prev_height)
.unwrap()
.into(),
p2wsh: groups.p2wsh.height.collect_one(prev_height).unwrap().into(),
p2tr: groups.p2tr.height.collect_one(prev_height).unwrap().into(),
p2a: groups.p2a.height.collect_one(prev_height).unwrap().into(),
})
} else {
Default::default()
}
}
}
/// Address count per address type, with height + derived indexes.
#[derive(Deref, DerefMut, Traversable)]
pub struct AddrTypeToAddrCountVecs<M: StorageMode = Rw>(ByAddrType<AddrCountVecs<M>>);
impl From<ByAddrType<AddrCountVecs>> for AddrTypeToAddrCountVecs {
#[inline]
fn from(value: ByAddrType<AddrCountVecs>) -> Self {
Self(value)
}
}
impl AddrTypeToAddrCountVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self::from(ByAddrType::<AddrCountVecs>::new_with_name(
|type_name| {
AddrCountVecs::forced_import(db, &format!("{type_name}_{name}"), version, indexes)
},
)?))
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.0.values().map(|v| v.height.len()).min().unwrap()
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.0
.par_values_mut()
.map(|v| &mut v.height as &mut dyn AnyStoredVec)
}
#[inline(always)]
pub(crate) fn push_height(&mut self, addr_counts: &AddrTypeToAddrCount) {
for (vecs, &count) in self.0.values_mut().zip(addr_counts.values()) {
vecs.height.push(count.into());
}
}
pub(crate) fn reset_height(&mut self) -> Result<()> {
for v in self.0.values_mut() {
v.height.reset()?;
}
Ok(())
}
pub(crate) fn by_height(&self) -> Vec<&EagerVec<PcoVec<Height, StoredU64>>> {
self.0.values().map(|v| &v.height).collect()
}
}
#[derive(Traversable)]
pub struct AddrCountsVecs<M: StorageMode = Rw> {
pub all: AddrCountVecs<M>,
#[traversable(flatten)]
pub by_addr_type: AddrTypeToAddrCountVecs<M>,
}
impl AddrCountsVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self {
all: AddrCountVecs::forced_import(db, name, version, indexes)?,
by_addr_type: AddrTypeToAddrCountVecs::forced_import(db, name, version, indexes)?,
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.all
.height
.len()
.min(self.by_addr_type.min_stateful_len())
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
rayon::iter::once(&mut self.all.height as &mut dyn AnyStoredVec)
.chain(self.by_addr_type.par_iter_height_mut())
}
pub(crate) fn reset_height(&mut self) -> Result<()> {
self.all.height.reset()?;
self.by_addr_type.reset_height()?;
Ok(())
}
#[inline(always)]
pub(crate) fn push_height(&mut self, total: u64, addr_counts: &AddrTypeToAddrCount) {
self.all.height.push(total.into());
self.by_addr_type.push_height(addr_counts);
}
pub(crate) fn compute_rest(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> {
let sources = self.by_addr_type.by_height();
self.all
.height
.compute_sum_of_others(starting_indexes.height, &sources, exit)?;
Ok(())
}
}
@@ -0,0 +1,38 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{StoredU64, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{Database, Rw, StorageMode};
use crate::{
indexes,
internal::{PerBlock, WithAddrTypes},
};
use super::AddrTypeToAddrCount;
/// Per-block `StoredU64` counts with an aggregate `all` plus a per-address-type
/// breakdown. Shared primitive backing addr-count, empty-addr-count, and the
/// funded/total pairs used by exposed, reused, and respent.
#[derive(Deref, DerefMut, Traversable)]
pub struct AddrCountsVecs<M: StorageMode = Rw>(
#[traversable(flatten)] pub WithAddrTypes<PerBlock<StoredU64, M>>,
);
impl AddrCountsVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self(WithAddrTypes::<PerBlock<StoredU64>>::forced_import(
db, name, version, indexes,
)?))
}
#[inline(always)]
pub(crate) fn push_counts(&mut self, counts: &AddrTypeToAddrCount) {
self.push_height(counts.sum(), counts.values().copied());
}
}
@@ -0,0 +1,79 @@
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use brk_types::Version;
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, Exit, Rw, StorageMode};
use crate::indexes;
use super::{AddrCountsVecs, AddrTypeToAddrCount};
/// Paired funded + cumulative-total address counts, used by exposed, reused,
/// and respent. On-disk naming: `"{name}_addr_count"` (funded) and
/// `"total_{name}_addr_count"` (total).
#[derive(Traversable)]
pub struct AddrCountFundedTotalVecs<M: StorageMode = Rw> {
pub funded: AddrCountsVecs<M>,
pub total: AddrCountsVecs<M>,
}
impl AddrCountFundedTotalVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self {
funded: AddrCountsVecs::forced_import(
db,
&format!("{name}_addr_count"),
version,
indexes,
)?,
total: AddrCountsVecs::forced_import(
db,
&format!("total_{name}_addr_count"),
version,
indexes,
)?,
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.funded
.min_stateful_len()
.min(self.total.min_stateful_len())
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.funded
.par_iter_height_mut()
.chain(self.total.par_iter_height_mut())
}
pub(crate) fn reset_height(&mut self) -> Result<()> {
self.funded.reset_height()?;
self.total.reset_height()?;
Ok(())
}
#[inline(always)]
pub(crate) fn push_counts(
&mut self,
funded: &AddrTypeToAddrCount,
total: &AddrTypeToAddrCount,
) {
self.funded.push_counts(funded);
self.total.push_counts(total);
}
pub(crate) fn compute_rest(&mut self, starting_lengths: &Lengths, exit: &Exit) -> Result<()> {
self.funded.compute_rest(starting_lengths, exit)?;
self.total.compute_rest(starting_lengths, exit)?;
Ok(())
}
}
@@ -0,0 +1,7 @@
mod all_vecs;
mod funded_total_vecs;
mod state;
pub use all_vecs::AddrCountsVecs;
pub use funded_total_vecs::AddrCountFundedTotalVecs;
pub use state::AddrTypeToAddrCount;
@@ -0,0 +1,38 @@
use brk_cohort::ByAddrType;
use brk_types::Height;
use derive_more::{Deref, DerefMut};
use vecdb::ReadableVec;
use super::AddrCountsVecs;
/// Per-addr-type address-count running total. Shared runtime state across
/// funded / empty / exposed / reused / respent counters; paired with
/// [`AddrCountsVecs`] on disk.
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddrTypeToAddrCount(ByAddrType<u64>);
impl AddrTypeToAddrCount {
#[inline]
pub(crate) fn sum(&self) -> u64 {
self.0.values().sum()
}
}
impl From<ByAddrType<u64>> for AddrTypeToAddrCount {
#[inline]
fn from(value: ByAddrType<u64>) -> Self {
Self(value)
}
}
impl From<(&AddrCountsVecs, Height)> for AddrTypeToAddrCount {
#[inline]
fn from((vecs, starting_height): (&AddrCountsVecs, Height)) -> Self {
let Some(prev_height) = starting_height.decremented() else {
return Self::default();
};
vecs.by_addr_type
.map_with_name(|_, v| v.height.collect_one(prev_height).unwrap().into())
.into()
}
}
@@ -1,28 +1,24 @@
use brk_cohort::ByAddrType;
use brk_traversable::Traversable;
use brk_types::{BasisPointsSigned32, StoredI64, StoredU64, Version};
use derive_more::{Deref, DerefMut};
use crate::{
indexes,
internal::{CachedWindowStarts, LazyRollingDeltasFromHeight},
internal::{LazyRollingDeltasFromHeight, WindowStartVec, Windows, WithAddrTypes},
};
use super::AddrCountsVecs;
type AddrDelta = LazyRollingDeltasFromHeight<StoredU64, StoredI64, BasisPointsSigned32>;
#[derive(Clone, Traversable)]
pub struct DeltaVecs {
pub all: AddrDelta,
#[traversable(flatten)]
pub by_addr_type: ByAddrType<AddrDelta>,
}
#[derive(Clone, Deref, DerefMut, Traversable)]
pub struct DeltaVecs(#[traversable(flatten)] pub WithAddrTypes<AddrDelta>);
impl DeltaVecs {
pub(crate) fn new(
version: Version,
addr_count: &AddrCountsVecs,
cached_starts: &CachedWindowStarts,
cached_starts: &Windows<&WindowStartVec>,
indexes: &indexes::Vecs,
) -> Self {
let version = version + Version::TWO;
@@ -30,7 +26,7 @@ impl DeltaVecs {
let all = LazyRollingDeltasFromHeight::new(
"addr_count",
version,
&addr_count.all.0.height,
&addr_count.all.height,
cached_starts,
indexes,
);
@@ -39,12 +35,12 @@ impl DeltaVecs {
LazyRollingDeltasFromHeight::new(
&format!("{name}_addr_count"),
version,
&addr.0.height,
&addr.height,
cached_starts,
indexes,
)
});
Self { all, by_addr_type }
Self(WithAddrTypes { all, by_addr_type })
}
}
@@ -0,0 +1,124 @@
//! Exposed address tracking (quantum / pubkey-exposure sense).
//!
//! An address is "exposed" once its public key is in the blockchain. Once
//! exposed, any funds at that address are at cryptographic risk (e.g. from
//! a quantum attacker capable of recovering the private key from the pubkey).
//!
//! When the pubkey gets exposed depends on the address type:
//!
//! - **P2PK33, P2PK65, P2TR**: the pubkey (or P2TR's tweaked output key) is
//! directly in the locking script of the funding output. These addresses are
//! exposed the moment they receive any funds.
//! - **P2PKH, P2SH, P2WPKH, P2WSH**: the locking script contains a hash of
//! the pubkey/script. The pubkey is only revealed when spending. Note that
//! even the spending tx itself exposes the pubkey while the address still
//! holds funds, during the mempool window between broadcast and confirmation,
//! the pubkey is visible while the UTXO being spent is still unspent on-chain.
//! So every spent address of these types has had at least one moment with
//! funds at quantum risk.
//! - **P2A**: anyone-can-spend, no pubkey at all. Excluded from both counters.
//!
//! Formally, with `is_funding_exposed` = `output_type.pubkey_exposed_at_funding()`:
//! - `funded` (count): `(utxo_count > 0) AND (is_funding_exposed OR spent_txo_count >= 1)`
//! - `total` (count): `(is_funding_exposed AND ever received) OR spent_txo_count >= 1`
//! - `supply` (sats): sum of balances of addresses currently in the funded set
//!
//! For P2PK/P2TR types this means `total ≡ total_addr_count` and
//! `funded ≡ funded_addr_count` (every address of those types is exposed by
//! virtue of existing). For P2PKH/P2SH/P2WPKH/P2WSH it's the strict subset of
//! addresses that have been spent from. The aggregate `all` exposed counter
//! sums these, giving "Bitcoin addresses currently with funds at quantum risk".
//!
//! All metrics are tracked as running counters and require no extra fields
//! on the address data. They're maintained via delta detection in
//! `process_received` and `process_sent`.
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use brk_types::{Height, Sats, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, Exit, ReadableVec, Rw, StorageMode};
use super::{
count::AddrCountFundedTotalVecs,
supply::{AddrSupplyShareVecs, AddrSupplyVecs},
};
use crate::{indexes, prices};
mod state;
pub use state::ExposedAddrState;
/// Top-level container for all exposed address tracking: counts (funded +
/// total), the funded supply, and share of supply.
#[derive(Traversable)]
pub struct ExposedAddrVecs<M: StorageMode = Rw> {
pub count: AddrCountFundedTotalVecs<M>,
pub supply: AddrSupplyVecs<M>,
#[traversable(wrap = "supply", rename = "share")]
pub supply_share: AddrSupplyShareVecs<M>,
}
impl ExposedAddrVecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self {
count: AddrCountFundedTotalVecs::forced_import(db, "exposed", version, indexes)?,
supply: AddrSupplyVecs::forced_import(db, "exposed", version, indexes)?,
supply_share: AddrSupplyShareVecs::forced_import(db, "exposed", version, indexes)?,
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.count
.min_stateful_len()
.min(self.supply.min_stateful_len())
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.count
.par_iter_height_mut()
.chain(self.supply.par_iter_height_mut())
}
pub(crate) fn reset_height(&mut self) -> Result<()> {
self.count.reset_height()?;
self.supply.reset_height()?;
self.supply_share.reset_height()?;
Ok(())
}
#[inline(always)]
pub(crate) fn push_height(&mut self, state: &ExposedAddrState) {
self.count.push_counts(&state.funded, &state.total);
self.supply.push_supply(&state.supply);
}
pub(crate) fn compute_rest(
&mut self,
starting_lengths: &Lengths,
prices: &prices::Vecs,
all_supply_sats: &impl ReadableVec<Height, Sats>,
type_supply_sats: &ByAddrType<&impl ReadableVec<Height, Sats>>,
exit: &Exit,
) -> Result<()> {
self.count.compute_rest(starting_lengths, exit)?;
self.supply
.compute_rest(starting_lengths.height, prices, exit)?;
self.supply_share.compute_rest(
starting_lengths.height,
&self.supply,
all_supply_sats,
type_supply_sats,
exit,
)?;
Ok(())
}
}
@@ -0,0 +1,83 @@
use brk_types::{FundedAddrData, Height, OutputType};
use crate::distribution::{
addr::{AddrReceivePreState, AddrSendPreState, AddrTypeToAddrCount, AddrTypeToSupply},
block::TrackingStatus,
};
use super::ExposedAddrVecs;
/// Runtime running totals for exposed-addr tracking. Mirrors the persistent
/// fields of [`ExposedAddrVecs`]: funded count, total count, funded supply.
/// Recovered from disk at the start of a block-loop run.
#[derive(Debug, Default)]
pub struct ExposedAddrState {
pub funded: AddrTypeToAddrCount,
pub total: AddrTypeToAddrCount,
pub supply: AddrTypeToSupply,
}
impl ExposedAddrState {
/// Apply exposed-addr updates for a received output, AFTER the receive
/// has mutated `addr_data`. `pre` is the snapshot taken before the mutation.
#[inline]
pub(crate) fn on_receive(
&mut self,
output_type: OutputType,
addr_data: &FundedAddrData,
pre: &AddrReceivePreState,
status: TrackingStatus,
) {
// Pubkey-exposure state is unchanged by a receive, so `pre.was_pubkey_exposed`
// equals the post-receive value.
if !pre.was_funded && pre.was_pubkey_exposed {
*self.funded.get_mut_unwrap(output_type) += 1;
}
// Total for pk-exposed-at-funding types fires here on first receive.
// Other types fire on first spend in [`Self::on_send`].
if output_type.pubkey_exposed_at_funding() && matches!(status, TrackingStatus::New) {
*self.total.get_mut_unwrap(output_type) += 1;
}
let after = addr_data.exposed_supply_contribution(output_type);
self.supply
.apply_delta(output_type, pre.exposed_contribution, after);
}
/// Apply exposed-addr updates for a spent UTXO, AFTER the send has mutated
/// `addr_data`. `pre` is the snapshot taken before the mutation.
#[inline]
pub(crate) fn on_send(
&mut self,
output_type: OutputType,
addr_data: &FundedAddrData,
pre: &AddrSendPreState,
will_be_empty: bool,
) {
let after = addr_data.exposed_supply_contribution(output_type);
self.supply
.apply_delta(output_type, pre.exposed_contribution, after);
// First-ever pubkey exposure. Non-pk-exposed types fire on first spend.
// Pk-exposed types never fire here (was already exposed at first receive).
if !pre.was_pubkey_exposed {
*self.total.get_mut_unwrap(output_type) += 1;
if !will_be_empty {
*self.funded.get_mut_unwrap(output_type) += 1;
}
}
// Leaving the funded exposed set: was in it iff pubkey was exposed pre-spend.
if will_be_empty && pre.was_pubkey_exposed {
*self.funded.get_mut_unwrap(output_type) -= 1;
}
}
}
impl From<(&ExposedAddrVecs, Height)> for ExposedAddrState {
#[inline]
fn from((vecs, starting_height): (&ExposedAddrVecs, Height)) -> Self {
Self {
funded: AddrTypeToAddrCount::from((&vecs.count.funded, starting_height)),
total: AddrTypeToAddrCount::from((&vecs.count.total, starting_height)),
supply: AddrTypeToSupply::from((&vecs.supply, starting_height)),
}
}
}
@@ -1,17 +1,25 @@
mod activity;
mod addr_count;
mod count;
mod data;
mod delta;
mod exposed;
mod indexes;
mod new_addr_count;
mod reused;
mod state;
mod supply;
mod total_addr_count;
mod type_map;
pub use activity::{AddrActivityVecs, AddrTypeToActivityCounts};
pub use addr_count::{AddrCountsVecs, AddrTypeToAddrCount};
pub use count::{AddrCountsVecs, AddrTypeToAddrCount};
pub use data::AddrsDataVecs;
pub use delta::DeltaVecs;
pub use exposed::{ExposedAddrState, ExposedAddrVecs};
pub use indexes::AnyAddrIndexesVecs;
pub use new_addr_count::NewAddrCountVecs;
pub use reused::{ReusedAddrState, ReusedAddrVecs};
pub use state::{AddrMetricsState, AddrReceivePreState, AddrSendPreState};
pub use supply::AddrTypeToSupply;
pub use total_addr_count::TotalAddrCountVecs;
pub use type_map::{AddrTypeToTypeIndexMap, AddrTypeToVec, HeightToAddrTypeToVec};
@@ -1,50 +1,38 @@
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, StoredU64, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{Database, Exit, Rw, StorageMode};
use crate::{
indexes,
internal::{CachedWindowStarts, PerBlockCumulativeRolling},
internal::{PerBlockCumulativeRolling, WindowStartVec, Windows, WithAddrTypes},
};
use super::TotalAddrCountVecs;
/// New address count per block (global + per-type)
#[derive(Traversable)]
pub struct NewAddrCountVecs<M: StorageMode = Rw> {
pub all: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
#[traversable(flatten)]
pub by_addr_type: ByAddrType<PerBlockCumulativeRolling<StoredU64, StoredU64, M>>,
}
/// New address count per block (global + per-type).
#[derive(Deref, DerefMut, Traversable)]
pub struct NewAddrCountVecs<M: StorageMode = Rw>(
#[traversable(flatten)] pub WithAddrTypes<PerBlockCumulativeRolling<StoredU64, StoredU64, M>>,
);
impl NewAddrCountVecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
let all = PerBlockCumulativeRolling::forced_import(
Ok(Self(WithAddrTypes::<
PerBlockCumulativeRolling<StoredU64, StoredU64>,
>::forced_import(
db,
"new_addr_count",
version,
indexes,
cached_starts,
)?;
let by_addr_type = ByAddrType::new_with_name(|name| {
PerBlockCumulativeRolling::forced_import(
db,
&format!("{name}_new_addr_count"),
version,
indexes,
cached_starts,
)
})?;
Ok(Self { all, by_addr_type })
)?))
}
pub(crate) fn compute(
@@ -53,11 +41,12 @@ impl NewAddrCountVecs {
total_addr_count: &TotalAddrCountVecs,
exit: &Exit,
) -> Result<()> {
self.all.compute(max_from, exit, |height_vec| {
self.0.all.compute(max_from, exit, |height_vec| {
Ok(height_vec.compute_change(max_from, &total_addr_count.all.height, 1, exit)?)
})?;
for ((_, new), (_, total)) in self
.0
.by_addr_type
.iter_mut()
.zip(total_addr_count.by_addr_type.iter())
@@ -0,0 +1,11 @@
//! Per-block address-reuse event tracking. Holds both the output-side
//! ("an output landed on a previously-used address") and input-side
//! ("an input spent from an address in the reused set") event counters.
//! Shared between reused (receive-based) and respent (spend-based) flavors.
//! See [`vecs::AddrEventsVecs`] for the full description of each metric.
mod state;
mod vecs;
pub use state::AddrTypeToAddrEventCount;
pub use vecs::AddrEventsVecs;
@@ -0,0 +1,27 @@
use brk_cohort::ByAddrType;
use derive_more::{Deref, DerefMut};
/// Per-block running counter of address-reuse events, per address type. Shared
/// across reused (receive-based) and respent (spend-based) flavors, and
/// across output-side ("output landed on a previously-used address") and
/// input-side ("input spent from an address in the set") event kinds.
///
/// Reset at the start of each block; no disk recovery needed since per-block
/// flow is reconstructed deterministically from `process_received` /
/// `process_sent`.
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddrTypeToAddrEventCount(ByAddrType<u64>);
impl AddrTypeToAddrEventCount {
#[inline]
pub(crate) fn sum(&self) -> u64 {
self.0.values().sum()
}
#[inline]
pub(crate) fn reset(&mut self) {
for v in self.0.values_mut() {
*v = 0;
}
}
}
@@ -0,0 +1,268 @@
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use brk_types::{BasisPoints16, OutputType, StoredF32, StoredU32, StoredU64, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, AnyVec, Database, Exit, Rw, StorageMode, WritableVec};
use crate::{
indexes, inputs,
internal::{
PerBlockCumulativeRolling, PerBlockRollingAverage, PercentCumulativeRolling,
WindowStartVec, Windows, WithAddrTypes,
},
outputs,
};
use super::state::AddrTypeToAddrEventCount;
/// Per-block reused-address event metrics. Holds three families of
/// signals: output-level (use), input-level (spend), and address-level
/// (active in block).
///
/// `output_to_reused_addr_count`: every output landing on an address that had
/// already received at least one prior output anywhere in its lifetime,
/// i.e. an output-level reuse event. Outputs are not deduplicated per
/// address within a block: an address receiving N outputs in one block
/// that had `before` lifetime outputs contributes
/// `max(0, N - max(0, 1 - before))` events. Only the very first output
/// an address ever sees is excluded. Every subsequent output counts,
/// matching the standard "% of outputs to previously-used addresses"
/// reuse ratio reported by external sources. `output_to_reused_addr_share`
/// uses `outputs::ByTypeVecs::output_count` (all 12 output types) as
/// denominator. `spendable_output_to_reused_addr_share` uses the
/// op_return-excluded 11-type aggregate (`spendable_output_count`).
///
/// `input_from_reused_addr_count`: every input spending from an address
/// whose lifetime `funded_txo_count > 1` at the time of the spend (i.e.
/// the address is in the same reused set tracked by
/// `reused_addr_count`). Every input is checked independently. If a
/// single address has multiple inputs in one block each one counts.
/// This is a *stable-predicate* signal about the sending address, not
/// an output-level repeat event: the first spend from a reused address
/// counts just as much as the tenth. Denominator
/// (`input_from_reused_addr_share`): `inputs::ByTypeVecs::input_count` (11
/// spendable types, where `p2ms`, `unknown`, `empty` count as true
/// negatives).
///
/// `active_reused_addr_count` / `active_reused_addr_share`: block-level
/// *address* signals (single aggregate, not per-type).
/// `active_reused_addr_count` is the count of distinct addresses
/// involved in this block (sent received) that satisfy `is_reused()`
/// after the block's events, populated inline in `process_received`
/// (each receiver, post-receive) and in `process_sent` (each
/// first-encounter sender, deduped against `received_addrs` so
/// addresses that did both aren't double-counted).
/// `active_reused_addr_share` is the per-block ratio
/// `reused / active * 100` as a percentage in `[0, 100]` (or `0.0` for
/// empty blocks). The denominator (distinct active addrs per block)
/// lives on `ActivityCountVecs::active` (`addrs.activity.all.active`),
/// derived from `sending + receiving - bidirectional`. Both fields
/// use `PerBlockRollingAverage` so their lazy 24h/1w/1m/1y series are
/// rolling *averages* of the per-block values. Sums and cumulatives of
/// distinct-address counts would be misleading because the same
/// address can appear in multiple blocks.
#[derive(Traversable)]
pub struct AddrEventsVecs<M: StorageMode = Rw> {
pub output_to_reused_addr_count:
WithAddrTypes<PerBlockCumulativeRolling<StoredU64, StoredU64, M>>,
pub output_to_reused_addr_share: WithAddrTypes<PercentCumulativeRolling<BasisPoints16, M>>,
pub spendable_output_to_reused_addr_share: PercentCumulativeRolling<BasisPoints16, M>,
pub input_from_reused_addr_count:
WithAddrTypes<PerBlockCumulativeRolling<StoredU64, StoredU64, M>>,
pub input_from_reused_addr_share: WithAddrTypes<PercentCumulativeRolling<BasisPoints16, M>>,
pub active_reused_addr_count: PerBlockRollingAverage<StoredU32, StoredU64, M>,
pub active_reused_addr_share: PerBlockRollingAverage<StoredF32, StoredF32, M>,
}
impl AddrEventsVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
let import_count = |name: &str| {
WithAddrTypes::<PerBlockCumulativeRolling<StoredU64, StoredU64>>::forced_import(
db,
name,
version,
indexes,
cached_starts,
)
};
let import_percent =
|name: &str| -> Result<WithAddrTypes<PercentCumulativeRolling<BasisPoints16>>> {
Ok(WithAddrTypes {
all: PercentCumulativeRolling::forced_import(db, name, version, indexes)?,
by_addr_type: ByAddrType::new_with_name(|type_name| {
PercentCumulativeRolling::forced_import(
db,
&format!("{type_name}_{name}"),
version,
indexes,
)
})?,
})
};
let output_to_reused_addr_count = import_count(&format!("output_to_{name}_addr_count"))?;
let output_to_reused_addr_share = import_percent(&format!("output_to_{name}_addr_share"))?;
let spendable_output_to_reused_addr_share = PercentCumulativeRolling::forced_import(
db,
&format!("spendable_output_to_{name}_addr_share"),
version,
indexes,
)?;
let input_from_reused_addr_count = import_count(&format!("input_from_{name}_addr_count"))?;
let input_from_reused_addr_share =
import_percent(&format!("input_from_{name}_addr_share"))?;
let active_reused_addr_count = PerBlockRollingAverage::forced_import(
db,
&format!("active_{name}_addr_count"),
version,
indexes,
cached_starts,
)?;
let active_reused_addr_share = PerBlockRollingAverage::forced_import(
db,
&format!("active_{name}_addr_share"),
version,
indexes,
cached_starts,
)?;
Ok(Self {
output_to_reused_addr_count,
output_to_reused_addr_share,
spendable_output_to_reused_addr_share,
input_from_reused_addr_count,
input_from_reused_addr_share,
active_reused_addr_count,
active_reused_addr_share,
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.output_to_reused_addr_count
.min_stateful_len()
.min(self.input_from_reused_addr_count.min_stateful_len())
.min(self.active_reused_addr_count.block.len())
.min(self.active_reused_addr_share.block.len())
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.output_to_reused_addr_count
.par_iter_height_mut()
.chain(self.input_from_reused_addr_count.par_iter_height_mut())
.chain([
&mut self.active_reused_addr_count.block as &mut dyn AnyStoredVec,
&mut self.active_reused_addr_share.block as &mut dyn AnyStoredVec,
])
}
pub(crate) fn reset_height(&mut self) -> Result<()> {
self.output_to_reused_addr_count.reset_height()?;
self.input_from_reused_addr_count.reset_height()?;
self.active_reused_addr_count.block.reset()?;
self.active_reused_addr_share.block.reset()?;
Ok(())
}
#[inline(always)]
pub(crate) fn push_height(
&mut self,
uses: &AddrTypeToAddrEventCount,
spends: &AddrTypeToAddrEventCount,
active_addr_count: u32,
active_reused_addr_count: u32,
) {
self.output_to_reused_addr_count
.push_height(uses.sum(), uses.values().copied());
self.input_from_reused_addr_count
.push_height(spends.sum(), spends.values().copied());
self.active_reused_addr_count
.block
.push(StoredU32::from(active_reused_addr_count));
// Stored as a percentage in [0, 100] to match the rest of the
// codebase (Unit.percentage on the website expects 0..100). The
// `active_addr_count` denominator lives on `ActivityCountVecs`
// (`addrs.activity.all.active`), passed in here so we can
// compute the per-block ratio inline.
let share = if active_addr_count > 0 {
100.0 * (active_reused_addr_count as f32 / active_addr_count as f32)
} else {
0.0
};
self.active_reused_addr_share
.block
.push(StoredF32::from(share));
}
pub(crate) fn compute_rest(
&mut self,
starting_lengths: &Lengths,
outputs_by_type: &outputs::ByTypeVecs,
inputs_by_type: &inputs::ByTypeVecs,
exit: &Exit,
) -> Result<()> {
self.output_to_reused_addr_count
.compute_rest(starting_lengths.height, exit)?;
self.input_from_reused_addr_count
.compute_rest(starting_lengths.height, exit)?;
self.active_reused_addr_count
.compute_rest(starting_lengths.height, exit)?;
self.active_reused_addr_share
.compute_rest(starting_lengths.height, exit)?;
self.output_to_reused_addr_share.all.compute_count_ratio(
&self.output_to_reused_addr_count.all,
&outputs_by_type.output_count.all,
starting_lengths.height,
exit,
)?;
self.spendable_output_to_reused_addr_share
.compute_count_ratio(
&self.output_to_reused_addr_count.all,
&outputs_by_type.spendable_output_count,
starting_lengths.height,
exit,
)?;
self.input_from_reused_addr_share.all.compute_count_ratio(
&self.input_from_reused_addr_count.all,
&inputs_by_type.input_count.all,
starting_lengths.height,
exit,
)?;
for otype in OutputType::ADDR_TYPES {
self.output_to_reused_addr_share
.by_addr_type
.get_mut_unwrap(otype)
.compute_count_ratio(
self.output_to_reused_addr_count
.by_addr_type
.get_unwrap(otype),
outputs_by_type.output_count.by_type.get(otype),
starting_lengths.height,
exit,
)?;
self.input_from_reused_addr_share
.by_addr_type
.get_mut_unwrap(otype)
.compute_count_ratio(
self.input_from_reused_addr_count
.by_addr_type
.get_unwrap(otype),
inputs_by_type.input_count.by_type.get(otype),
starting_lengths.height,
exit,
)?;
}
Ok(())
}
}
@@ -0,0 +1,134 @@
//! Reused address tracking.
//!
//! An address is "reused" if its lifetime `funded_txo_count > 1`, i.e.
//! it has received more than one output across its lifetime. This is
//! the simplest output-multiplicity proxy for address linkability.
//!
//! Two facets are tracked here:
//! - [`count`]: how many distinct addresses are currently reused
//! (funded) and how many have *ever* been reused (total). Per address
//! type plus an aggregated `all`.
//! - [`events`]: per-block address-reuse event counts on both sides.
//! Output-side (`output_to_reused_addr_count`, outputs landing on
//! addresses that had already received ≥ 1 prior output) and
//! input-side (`input_from_reused_addr_count`, inputs spending from
//! addresses with lifetime `funded_txo_count > 1`). Each count is
//! paired with a percent over the matching block-level output/input
//! total.
mod events;
pub use events::{AddrEventsVecs, AddrTypeToAddrEventCount};
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_indexer::Lengths;
use brk_traversable::Traversable;
use brk_types::{Height, Sats, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, Exit, ReadableVec, Rw, StorageMode};
use super::{
count::AddrCountFundedTotalVecs,
supply::{AddrSupplyShareVecs, AddrSupplyVecs},
};
use crate::{
indexes, inputs,
internal::{WindowStartVec, Windows},
outputs, prices,
};
mod state;
pub use state::ReusedAddrState;
/// Top-level container for all reused address tracking: counts (funded +
/// total), per-block reuse events (output-side + input-side), and funded
/// supply + share.
#[derive(Traversable)]
pub struct ReusedAddrVecs<M: StorageMode = Rw> {
pub count: AddrCountFundedTotalVecs<M>,
pub events: AddrEventsVecs<M>,
pub supply: AddrSupplyVecs<M>,
#[traversable(wrap = "supply", rename = "share")]
pub supply_share: AddrSupplyShareVecs<M>,
}
impl ReusedAddrVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
Ok(Self {
count: AddrCountFundedTotalVecs::forced_import(db, name, version, indexes)?,
events: AddrEventsVecs::forced_import(db, name, version, indexes, cached_starts)?,
supply: AddrSupplyVecs::forced_import(db, name, version, indexes)?,
supply_share: AddrSupplyShareVecs::forced_import(db, name, version, indexes)?,
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.count
.min_stateful_len()
.min(self.events.min_stateful_len())
.min(self.supply.min_stateful_len())
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.count
.par_iter_height_mut()
.chain(self.events.par_iter_height_mut())
.chain(self.supply.par_iter_height_mut())
}
pub(crate) fn reset_height(&mut self) -> Result<()> {
self.count.reset_height()?;
self.events.reset_height()?;
self.supply.reset_height()?;
self.supply_share.reset_height()?;
Ok(())
}
#[inline(always)]
pub(crate) fn push_height(&mut self, state: &ReusedAddrState, active_addr_count: u32) {
self.count.push_counts(&state.funded, &state.total);
self.supply.push_supply(&state.supply);
self.events.push_height(
&state.output_events,
&state.input_events,
active_addr_count,
u32::try_from(state.active.sum()).unwrap(),
);
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute_rest(
&mut self,
starting_lengths: &Lengths,
outputs_by_type: &outputs::ByTypeVecs,
inputs_by_type: &inputs::ByTypeVecs,
prices: &prices::Vecs,
all_supply_sats: &impl ReadableVec<Height, Sats>,
type_supply_sats: &ByAddrType<&impl ReadableVec<Height, Sats>>,
exit: &Exit,
) -> Result<()> {
self.count.compute_rest(starting_lengths, exit)?;
self.events
.compute_rest(starting_lengths, outputs_by_type, inputs_by_type, exit)?;
self.supply
.compute_rest(starting_lengths.height, prices, exit)?;
self.supply_share.compute_rest(
starting_lengths.height,
&self.supply,
all_supply_sats,
type_supply_sats,
exit,
)?;
Ok(())
}
}
@@ -0,0 +1,201 @@
use brk_types::{FundedAddrData, Height, OutputType};
use crate::distribution::addr::{
AddrReceivePreState, AddrSendPreState, AddrTypeToAddrCount, AddrTypeToSupply,
};
use super::{AddrTypeToAddrEventCount, ReusedAddrVecs};
/// Runtime state for receive-based (reused) or spend-based (respent)
/// address tracking. Mirrors the persistent fields of [`ReusedAddrVecs`]
/// (funded + total counts, funded supply) plus per-block event counters
/// that reset every block.
///
/// `output_events`, `input_events`, and `active` are cleared via
/// [`Self::reset_per_block`] at the start of each block. The three running
/// totals (`funded`, `total`, `supply`) are recovered from disk at the start
/// of a run via [`From<(&ReusedAddrVecs, Height)>`].
#[derive(Debug, Default)]
pub struct ReusedAddrState {
pub funded: AddrTypeToAddrCount,
pub total: AddrTypeToAddrCount,
pub supply: AddrTypeToSupply,
pub output_events: AddrTypeToAddrEventCount,
pub input_events: AddrTypeToAddrEventCount,
pub active: AddrTypeToAddrEventCount,
}
impl ReusedAddrState {
#[inline]
pub(crate) fn reset_per_block(&mut self) {
self.output_events.reset();
self.input_events.reset();
self.active.reset();
}
/// Apply reused-flavor (receive-based: `funded_txo_count > 1`) updates
/// for a received output, AFTER the receive has mutated `addr_data`.
#[inline]
pub(crate) fn on_receive_as_reused(
&mut self,
output_type: OutputType,
addr_data: &FundedAddrData,
pre: &AddrReceivePreState,
output_count: u32,
) {
let is_now_reused = addr_data.is_reused();
// Threshold crossing: the 2nd lifetime receive lands here. The address
// is always funded post-receive.
if is_now_reused && !pre.was_reused {
*self.total.get_mut_unwrap(output_type) += 1;
*self.funded.get_mut_unwrap(output_type) += 1;
} else if pre.was_reused && !pre.was_funded {
// Reactivation: already-reused address was empty, now funded.
*self.funded.get_mut_unwrap(output_type) += 1;
}
// output-to-reused events: outputs landing on addresses that had
// already received >= 1 prior output, i.e. every output except the
// very first one the address ever sees. With `before =
// prev_funded_txo_count` and `N = output_count`: events = N - max(0, 1 - before).
let skip_first = 1u32.saturating_sub(pre.prev_funded_txo_count.min(1));
let reused_events = output_count.saturating_sub(skip_first);
if reused_events > 0 {
*self.output_events.get_mut_unwrap(output_type) += u64::from(reused_events);
}
if is_now_reused {
*self.active.get_mut_unwrap(output_type) += 1;
}
let after = addr_data.reused_supply_contribution();
self.supply
.apply_delta(output_type, pre.reused_contribution, after);
}
/// Apply respent-flavor (spend-based: `spent_txo_count > 1`) updates for a
/// received output, AFTER the receive has mutated `addr_data`. Receives
/// don't cross the respent threshold. The only transition is an
/// already-respent empty address reactivating into the funded set.
#[inline]
pub(crate) fn on_receive_as_respent(
&mut self,
output_type: OutputType,
addr_data: &FundedAddrData,
pre: &AddrReceivePreState,
output_count: u32,
) {
if pre.was_respent && !pre.was_funded {
*self.funded.get_mut_unwrap(output_type) += 1;
}
// Respent status is stable across a receive, so every output lands on
// a respent address iff the address was already respent.
if pre.was_respent {
*self.output_events.get_mut_unwrap(output_type) += u64::from(output_count);
*self.active.get_mut_unwrap(output_type) += 1;
}
let after = addr_data.respent_supply_contribution();
self.supply
.apply_delta(output_type, pre.respent_contribution, after);
}
/// Apply reused-flavor updates for a spent UTXO, AFTER the send has
/// mutated `addr_data`. Sends don't change the reused predicate, so
/// `pre.was_reused == is_reused` post-spend.
#[inline]
pub(crate) fn on_send_as_reused(
&mut self,
output_type: OutputType,
addr_data: &FundedAddrData,
pre: &AddrSendPreState,
is_first_encounter: bool,
also_received: bool,
will_be_empty: bool,
) {
if pre.was_reused {
*self.input_events.get_mut_unwrap(output_type) += 1;
}
// Active reused: first-encounter sender, currently reused, and not
// already counted on the receive side.
if is_first_encounter && pre.was_reused && !also_received {
*self.active.get_mut_unwrap(output_type) += 1;
}
if will_be_empty && pre.was_reused {
*self.funded.get_mut_unwrap(output_type) -= 1;
}
let after = addr_data.reused_supply_contribution();
self.supply
.apply_delta(output_type, pre.reused_contribution, after);
}
/// Apply respent-flavor updates for a spent UTXO, AFTER the send has
/// mutated `addr_data`. Sends CAN cross the respent threshold on the
/// 2nd lifetime spend.
#[inline]
pub(crate) fn on_send_as_respent(
&mut self,
output_type: OutputType,
addr_data: &FundedAddrData,
pre: &AddrSendPreState,
is_first_encounter: bool,
also_received: bool,
will_be_empty: bool,
) {
if pre.was_respent {
*self.input_events.get_mut_unwrap(output_type) += 1;
}
let is_now_respent = addr_data.is_respent();
// Threshold crossing: the 2nd spend ever on this address. Always
// bumps the monotonic total. Bumps the funded count iff the address
// still has a balance. If the crossing spend also empties the
// address, the `will_be_empty` branch below doesn't decrement
// (was_respent is false), so the funded count stays correct.
if is_now_respent && !pre.was_respent {
*self.total.get_mut_unwrap(output_type) += 1;
if !will_be_empty {
*self.funded.get_mut_unwrap(output_type) += 1;
}
}
// Active respent splits cleanly into two disjoint branches (gated on
// `pre.was_respent`):
// - was already respent + active this block, and not also counted
// on the receive side: pure senders on first spend.
// - crosses the respent threshold this block: fires once per
// address ever, on the exact crossing spend.
if (is_first_encounter && pre.was_respent && !also_received)
|| (is_now_respent && !pre.was_respent)
{
*self.active.get_mut_unwrap(output_type) += 1;
}
// Leaving the funded respent set on empty uses pre-spend state: a
// threshold-crossing spend that also empties the address never
// entered the funded set above (gated on !will_be_empty), so we
// don't double-decrement.
if will_be_empty && pre.was_respent {
*self.funded.get_mut_unwrap(output_type) -= 1;
}
let after = addr_data.respent_supply_contribution();
self.supply
.apply_delta(output_type, pre.respent_contribution, after);
}
}
impl From<(&ReusedAddrVecs, Height)> for ReusedAddrState {
#[inline]
fn from((vecs, starting_height): (&ReusedAddrVecs, Height)) -> Self {
Self {
funded: AddrTypeToAddrCount::from((&vecs.count.funded, starting_height)),
total: AddrTypeToAddrCount::from((&vecs.count.total, starting_height)),
supply: AddrTypeToSupply::from((&vecs.supply, starting_height)),
output_events: AddrTypeToAddrEventCount::default(),
input_events: AddrTypeToAddrEventCount::default(),
active: AddrTypeToAddrEventCount::default(),
}
}
}
@@ -0,0 +1,180 @@
use brk_types::{FundedAddrData, Height, OutputType, Sats};
use crate::distribution::{block::TrackingStatus, vecs::AddrMetricsVecs};
use super::{AddrTypeToActivityCounts, AddrTypeToAddrCount, ExposedAddrState, ReusedAddrState};
/// Bundle of per-block runtime state for the full address-metrics pipeline.
/// Feeds `process_received` / `process_sent` and is pushed to [`AddrMetricsVecs`]
/// once per block.
///
/// Recovery: [`From<(&AddrMetricsVecs, Height)>`] reads the prior block from
/// disk to seed all persistent running totals. Per-block counters (activity,
/// and event counts inside each [`ReusedAddrState`]) default to zero and are
/// cleared at the top of each block via [`Self::reset_per_block`].
#[derive(Debug, Default)]
pub struct AddrMetricsState {
pub funded: AddrTypeToAddrCount,
pub empty: AddrTypeToAddrCount,
pub activity: AddrTypeToActivityCounts,
pub reused: ReusedAddrState,
pub respent: ReusedAddrState,
pub exposed: ExposedAddrState,
}
/// Snapshot of [`FundedAddrData`] taken BEFORE a receive mutates it.
/// Feeds delta-based updates in [`AddrMetricsState::on_receive_applied`].
#[derive(Debug)]
pub struct AddrReceivePreState {
pub was_funded: bool,
pub was_reused: bool,
pub was_respent: bool,
pub was_pubkey_exposed: bool,
pub prev_funded_txo_count: u32,
pub exposed_contribution: Sats,
pub reused_contribution: Sats,
pub respent_contribution: Sats,
}
impl AddrReceivePreState {
#[inline]
pub fn capture(addr_data: &FundedAddrData, output_type: OutputType) -> Self {
Self {
was_funded: addr_data.is_funded(),
was_reused: addr_data.is_reused(),
was_respent: addr_data.is_respent(),
was_pubkey_exposed: addr_data.is_pubkey_exposed(output_type),
prev_funded_txo_count: addr_data.funded_txo_count,
exposed_contribution: addr_data.exposed_supply_contribution(output_type),
reused_contribution: addr_data.reused_supply_contribution(),
respent_contribution: addr_data.respent_supply_contribution(),
}
}
}
/// Snapshot of [`FundedAddrData`] taken BEFORE a spend mutates it.
/// Feeds delta-based updates in [`AddrMetricsState::on_send_applied`].
#[derive(Debug)]
pub struct AddrSendPreState {
pub was_reused: bool,
pub was_respent: bool,
pub was_pubkey_exposed: bool,
pub exposed_contribution: Sats,
pub reused_contribution: Sats,
pub respent_contribution: Sats,
}
impl AddrSendPreState {
#[inline]
pub fn capture(addr_data: &FundedAddrData, output_type: OutputType) -> Self {
Self {
was_reused: addr_data.is_reused(),
was_respent: addr_data.is_respent(),
was_pubkey_exposed: addr_data.is_pubkey_exposed(output_type),
exposed_contribution: addr_data.exposed_supply_contribution(output_type),
reused_contribution: addr_data.reused_supply_contribution(),
respent_contribution: addr_data.respent_supply_contribution(),
}
}
}
impl AddrMetricsState {
#[inline]
pub(crate) fn reset_per_block(&mut self) {
self.activity.reset();
self.reused.reset_per_block();
self.respent.reset_per_block();
}
/// Apply all state updates for a received output, AFTER the cohort and
/// `addr_data` have been mutated. `pre` is the snapshot captured before
/// the mutation, `addr_data` is the post-receive view.
#[inline]
pub(crate) fn on_receive_applied(
&mut self,
output_type: OutputType,
status: TrackingStatus,
addr_data: &FundedAddrData,
pre: &AddrReceivePreState,
output_count: u32,
) {
let activity = self.activity.get_mut_unwrap(output_type);
activity.receiving += 1;
match status {
TrackingStatus::New => {
*self.funded.get_mut_unwrap(output_type) += 1;
}
TrackingStatus::WasEmpty => {
activity.reactivated += 1;
*self.funded.get_mut_unwrap(output_type) += 1;
*self.empty.get_mut_unwrap(output_type) -= 1;
}
TrackingStatus::Tracked => {}
}
self.reused
.on_receive_as_reused(output_type, addr_data, pre, output_count);
self.respent
.on_receive_as_respent(output_type, addr_data, pre, output_count);
self.exposed.on_receive(output_type, addr_data, pre, status);
}
/// Apply all state updates for a spent UTXO, AFTER the cohort and
/// `addr_data` have been mutated. `pre` is the snapshot captured before
/// the mutation. `is_first_encounter` / `also_received` come from the
/// caller's per-block seen/received tracking. `will_be_empty` is from
/// the pre-mutation `addr_data.has_1_utxos()`.
#[inline]
pub(crate) fn on_send_applied(
&mut self,
output_type: OutputType,
addr_data: &FundedAddrData,
pre: &AddrSendPreState,
is_first_encounter: bool,
also_received: bool,
will_be_empty: bool,
) {
if is_first_encounter {
let activity = self.activity.get_mut_unwrap(output_type);
activity.sending += 1;
if also_received {
activity.bidirectional += 1;
}
}
if will_be_empty {
*self.funded.get_mut_unwrap(output_type) -= 1;
*self.empty.get_mut_unwrap(output_type) += 1;
}
self.reused.on_send_as_reused(
output_type,
addr_data,
pre,
is_first_encounter,
also_received,
will_be_empty,
);
self.respent.on_send_as_respent(
output_type,
addr_data,
pre,
is_first_encounter,
also_received,
will_be_empty,
);
self.exposed
.on_send(output_type, addr_data, pre, will_be_empty);
}
}
impl From<(&AddrMetricsVecs, Height)> for AddrMetricsState {
#[inline]
fn from((vecs, starting_height): (&AddrMetricsVecs, Height)) -> Self {
Self {
funded: AddrTypeToAddrCount::from((&vecs.funded, starting_height)),
empty: AddrTypeToAddrCount::from((&vecs.empty, starting_height)),
activity: AddrTypeToActivityCounts::default(),
reused: ReusedAddrState::from((&vecs.reused, starting_height)),
respent: ReusedAddrState::from((&vecs.respent, starting_height)),
exposed: ExposedAddrState::from((&vecs.exposed, starting_height)),
}
}
}
@@ -0,0 +1,12 @@
//! Generic per-address-type supply tracking, shared across predicate-based
//! supply categories (exposed, reused, respent). A "category supply" is the
//! running sum of balances held by addresses currently in the funded subset
//! defined by some predicate.
mod share;
mod state;
mod vecs;
pub use share::AddrSupplyShareVecs;
pub use state::AddrTypeToSupply;
pub use vecs::AddrSupplyVecs;
@@ -0,0 +1,69 @@
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{BasisPoints16, Height, Sats, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{Database, Exit, ReadableVec, Rw, StorageMode};
use crate::{
indexes,
internal::{PercentPerBlock, RatioSatsBp16, WithAddrTypes},
};
use super::vecs::AddrSupplyVecs;
/// Share of a predicate-based supply category relative to total supply.
///
/// - `all`: category supply / circulating supply
/// - Per-type: type's category supply / type's total supply
#[derive(Deref, DerefMut, Traversable)]
pub struct AddrSupplyShareVecs<M: StorageMode = Rw>(
#[traversable(flatten)] pub WithAddrTypes<PercentPerBlock<BasisPoints16, M>>,
);
impl AddrSupplyShareVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self(
WithAddrTypes::<PercentPerBlock<BasisPoints16>>::forced_import(
db,
&format!("{name}_addr_supply_share"),
version,
indexes,
)?,
))
}
pub(crate) fn compute_rest(
&mut self,
max_from: Height,
supply: &AddrSupplyVecs,
all_supply_sats: &impl ReadableVec<Height, Sats>,
type_supply_sats: &ByAddrType<&impl ReadableVec<Height, Sats>>,
exit: &Exit,
) -> Result<()> {
self.all.compute_binary::<Sats, Sats, RatioSatsBp16>(
max_from,
&supply.all.sats.height,
all_supply_sats,
exit,
)?;
for ((_, share), ((_, cat), (_, denom))) in self
.by_addr_type
.iter_mut()
.zip(supply.by_addr_type.iter().zip(type_supply_sats.iter()))
{
share.compute_binary::<Sats, Sats, RatioSatsBp16>(
max_from,
&cat.sats.height,
*denom,
exit,
)?;
}
Ok(())
}
}
@@ -0,0 +1,49 @@
use brk_cohort::ByAddrType;
use brk_types::{Height, OutputType, Sats};
use derive_more::{Deref, DerefMut};
use vecdb::ReadableVec;
use super::vecs::AddrSupplyVecs;
/// Per-addr-type running-total of a supply category (sats). Shared across
/// predicate-based supply categories (exposed, reused, respent).
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddrTypeToSupply(ByAddrType<Sats>);
impl AddrTypeToSupply {
#[inline]
pub(crate) fn sum(&self) -> Sats {
self.0.values().copied().sum()
}
/// Apply a signed `after - before` delta to the slot for `output_type`.
/// Sats is unsigned, so branch on sign.
#[inline]
pub(crate) fn apply_delta(&mut self, output_type: OutputType, before: Sats, after: Sats) {
let slot = self.get_mut_unwrap(output_type);
if after >= before {
*slot += after - before;
} else {
*slot -= before - after;
}
}
}
impl From<ByAddrType<Sats>> for AddrTypeToSupply {
#[inline]
fn from(value: ByAddrType<Sats>) -> Self {
Self(value)
}
}
impl From<(&AddrSupplyVecs, Height)> for AddrTypeToSupply {
#[inline]
fn from((vecs, starting_height): (&AddrSupplyVecs, Height)) -> Self {
let Some(prev_height) = starting_height.decremented() else {
return Self::default();
};
vecs.by_addr_type
.map_with_name(|_, v| v.sats.height.collect_one(prev_height).unwrap())
.into()
}
}
@@ -0,0 +1,42 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::Version;
use derive_more::{Deref, DerefMut};
use vecdb::{Database, Rw, StorageMode};
use crate::{
indexes,
internal::{ValuePerBlock, WithAddrTypes},
};
use super::AddrTypeToSupply;
/// Per-addr-type running supply (sats/btc/cents/usd) with an aggregated `all`.
/// Shared across predicate-based supply categories (exposed, reused, respent).
/// Sats are pushed stateful per block; cents/usd are derived post-hoc from
/// sats × spot price.
#[derive(Deref, DerefMut, Traversable)]
pub struct AddrSupplyVecs<M: StorageMode = Rw>(
#[traversable(flatten)] pub WithAddrTypes<ValuePerBlock<M>>,
);
impl AddrSupplyVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self(WithAddrTypes::<ValuePerBlock>::forced_import(
db,
&format!("{name}_addr_supply"),
version,
indexes,
)?))
}
#[inline(always)]
pub(crate) fn push_supply(&mut self, supply: &AddrTypeToSupply) {
self.push_height(supply.sum(), supply.values().copied());
}
}
@@ -1,20 +1,21 @@
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, StoredU64, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{Database, Exit, Rw, StorageMode};
use crate::{indexes, internal::PerBlock};
use crate::{
indexes,
internal::{PerBlock, WithAddrTypes},
};
use super::AddrCountsVecs;
/// Total address count (global + per-type) with all derived indexes
#[derive(Traversable)]
pub struct TotalAddrCountVecs<M: StorageMode = Rw> {
pub all: PerBlock<StoredU64, M>,
#[traversable(flatten)]
pub by_addr_type: ByAddrType<PerBlock<StoredU64, M>>,
}
/// Total address count (global + per-type) with all derived indexes.
#[derive(Deref, DerefMut, Traversable)]
pub struct TotalAddrCountVecs<M: StorageMode = Rw>(
#[traversable(flatten)] pub WithAddrTypes<PerBlock<StoredU64, M>>,
);
impl TotalAddrCountVecs {
pub(crate) fn forced_import(
@@ -22,13 +23,12 @@ impl TotalAddrCountVecs {
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let all = PerBlock::forced_import(db, "total_addr_count", version, indexes)?;
let by_addr_type: ByAddrType<PerBlock<StoredU64>> = ByAddrType::new_with_name(|name| {
PerBlock::forced_import(db, &format!("{name}_total_addr_count"), version, indexes)
})?;
Ok(Self { all, by_addr_type })
Ok(Self(WithAddrTypes::<PerBlock<StoredU64>>::forced_import(
db,
"total_addr_count",
version,
indexes,
)?))
}
/// Eagerly compute total = addr_count + empty_addr_count.
@@ -67,14 +67,13 @@ pub(crate) fn process_funded_addrs(
// Pure pushes - no holes remain
addrs_data.funded.reserve_pushed(pushes_iter.len());
let mut next_index = addrs_data.funded.len();
for (addr_type, type_index, data) in pushes_iter {
for (next_index, (addr_type, type_index, data)) in (addrs_data.funded.len()..).zip(pushes_iter)
{
addrs_data.funded.push(data);
result.get_mut(addr_type).unwrap().insert(
type_index,
AnyAddrIndex::from(FundedAddrIndex::from(next_index)),
);
next_index += 1;
}
Ok(result)
@@ -138,14 +137,12 @@ pub(crate) fn process_empty_addrs(
// Pure pushes - no holes remain
addrs_data.empty.reserve_pushed(pushes_iter.len());
let mut next_index = addrs_data.empty.len();
for (addr_type, type_index, data) in pushes_iter {
for (next_index, (addr_type, type_index, data)) in (addrs_data.empty.len()..).zip(pushes_iter) {
addrs_data.empty.push(data);
result.get_mut(addr_type).unwrap().insert(
type_index,
AnyAddrIndex::from(EmptyAddrIndex::from(next_index)),
);
next_index += 1;
}
Ok(result)

Some files were not shown because too many files have changed in this diff Show More