Compare commits

...

120 Commits

Author SHA1 Message Date
nym21 f5c50e69fc release: v0.3.0-beta.4 2026-04-22 22:30:09 +02:00
nym21 9709c2040d docs: update generated docs 2026-04-22 22:29:43 +02:00
nym21 3faa989691 global: cost basis -> urpd 2026-04-22 22:23:52 +02:00
nym21 84e924b77e global: refactor 2026-04-22 22:23:39 +02:00
nym21 c5b16e7048 release: v0.3.0-beta.3 2026-04-22 15:56:15 +02:00
nym21 c1335cec31 docs: update generated docs 2026-04-22 15:55:53 +02:00
nym21 bdc3ba1df6 mempool: snap 2026-04-22 15:30:08 +02:00
nym21 6afce0bbdc mempool: fixes 2026-04-21 12:43:50 +02:00
nym21 327873d010 website: fixes 2026-04-20 18:11:30 +02:00
nym21 08175009d2 website: snap 2026-04-19 20:28:12 +02:00
nym21 a5d3be465e website: snap 2026-04-19 17:13:47 +02:00
nym21 fd2b93367d global: snap 2026-04-18 17:23:12 +02:00
nym21 2a93f51e81 global: snap 2026-04-17 21:23:11 +02:00
nym21 008143ff00 mempool: fix pending tx info 2026-04-16 23:35:55 +02:00
nym21 d340855c8b global: snap 2026-04-16 22:17:41 +02:00
nym21 78d6d9d6f1 changelog: updated 2026-04-16 10:15:23 +02:00
nym21 5cc85b0619 release: v0.3.0-beta.2 2026-04-15 19:49:13 +02:00
nym21 7433ce0d0e docs: update generated docs 2026-04-15 19:48:50 +02:00
nym21 75a97b4da9 client: js: add cache support to text fetches 2026-04-15 19:19:59 +02:00
nym21 c23e0f2a3c global: snap 2026-04-15 13:04:22 +02:00
nym21 08ba4ad996 global: snap 2026-04-15 12:51:30 +02:00
nym21 39da441d14 global: snapshot 2026-04-14 22:53:10 +02:00
nym21 904ec93668 deps: bumped 2026-04-14 01:42:24 +02:00
nym21 4cd8d9eb56 reader: snap 2026-04-14 01:37:04 +02:00
nym21 283baca848 global: big snapshot part 2 2026-04-13 22:47:08 +02:00
nym21 765261648d global: big snapshot 2026-04-13 22:46:56 +02:00
nym21 c3cef71aa3 global: snap 2026-04-12 18:00:02 +02:00
nym21 18d9c166d8 computer: snap 2026-04-11 22:11:48 +02:00
nym21 286256ebf0 global: veccached change 2026-04-10 11:30:29 +02:00
nym21 12aae503c9 global: snapshot pre cached change 2026-04-10 10:27:07 +02:00
nym21 95e5168244 deps: bumped 2026-04-09 15:08:06 +02:00
nym21 5fd9fff9cf global: speed improvement part4 2026-04-09 15:06:19 +02:00
nym21 db5b3887f9 global: speed improvement part3 2026-04-09 14:58:25 +02:00
nym21 5a3e1b4e6e global: speed improvement part2 2026-04-09 14:02:26 +02:00
nym21 21a0226a19 global: speed improvement 2026-04-09 11:52:01 +02:00
nym21 c5c49f62d1 clients: bump versions 2026-04-08 22:19:28 +02:00
nym21 dac66c988d release: v0.3.0-beta.1 2026-04-08 17:12:32 +02:00
nym21 303d168681 docs: update generated docs 2026-04-08 17:12:03 +02:00
nym21 1ddb3385e2 website: css 2026-04-08 17:06:54 +02:00
nym21 eb75274dbf website: fixes 2026-04-08 13:11:03 +02:00
nym21 3a7887348c global: snapshot 2026-04-08 12:09:35 +02:00
nym21 0a4cb0601f release: v0.3.0-beta.0 2026-04-08 01:46:43 +02:00
nym21 861e29277c docs: update generated docs 2026-04-08 01:46:13 +02:00
nym21 c76b149ef9 website: fixes 2026-04-08 01:43:58 +02:00
nym21 4c4c6fc840 global: snapshot 2026-04-08 01:38:03 +02:00
nym21 0c14dfe924 query: more optimizations 2026-04-07 17:57:57 +02:00
nym21 17e531b4ee query: more optimizations 2026-04-07 17:43:11 +02:00
nym21 f022f62cce global: snap 2026-04-07 13:49:02 +02:00
nym21 e91f1386b1 website: snap 2026-04-06 22:30:02 +02:00
nym21 02f543af38 release: v0.3.0-alpha.6 2026-04-05 22:48:37 +02:00
nym21 20c96fb551 docs: update generated docs 2026-04-05 22:48:10 +02:00
nym21 acd3d6f425 server: cache fixes 2026-04-05 22:43:30 +02:00
nym21 2b15a24b6d website: add pool logos 2026-04-05 19:46:41 +02:00
nym21 7fac0bc613 global: snap 2026-04-04 20:13:03 +02:00
nym21 62f51761ee global: snap 2026-04-04 18:19:11 +02:00
nym21 5340cc288e release: v0.3.0-alpha.5 2026-04-04 13:10:48 +02:00
nym21 befe3c8fb7 docs: update generated docs 2026-04-04 13:10:28 +02:00
nym21 41ec24c81e server: ms endpoint fixes 2026-04-04 13:05:39 +02:00
nym21 42b497ff65 server: ms endpoint fixes 2026-04-04 12:16:15 +02:00
nym21 01d908a560 release: v0.3.0-alpha.4 2026-04-04 11:59:17 +02:00
nym21 42debcce80 docs: update generated docs 2026-04-04 11:58:50 +02:00
nym21 8bc993eceb global: fixes 2026-04-04 11:53:27 +02:00
nym21 366ac33e23 release: v0.3.0-alpha.3 2026-04-04 01:05:04 +02:00
nym21 b5a7023bd3 docs: update generated docs 2026-04-04 01:04:38 +02:00
nym21 883b38c77c global: snap 2026-04-04 00:59:37 +02:00
nym21 59c767a9e2 release: v0.3.0-alpha.2 2026-04-03 17:56:37 +02:00
nym21 9b5bb848f7 docs: update generated docs 2026-04-03 17:56:12 +02:00
nym21 5bf06530ce store: replace fjal reset by dir nuking 2026-04-03 17:49:46 +02:00
nym21 768e6870cb global: snap 2026-04-03 15:51:27 +02:00
nym21 79829ddd53 changelog: updated 2026-04-03 01:19:47 +02:00
nym21 78082801c6 clients: bump versions 2026-04-03 00:52:09 +02:00
nym21 50771ddccc release: v0.3.0-alpha.1 2026-04-03 00:16:03 +02:00
nym21 3a8a9ddecc docs: update generated docs 2026-04-03 00:15:43 +02:00
nym21 6cd45c1f1f deps: bumped 2026-04-02 23:54:12 +02:00
nym21 1a2db43cf5 fmt: global 2026-04-02 23:50:01 +02:00
nym21 4840e564f4 server: moved params from brk_types 2026-04-02 23:49:01 +02:00
nym21 744dce932c types: docs 2026-04-02 23:08:25 +02:00
nym21 8dfc1bc932 server: ms endpoint fixes 2026-04-02 22:37:34 +02:00
nym21 d92cf43c57 server: reorg 2026-04-02 13:19:56 +02:00
nym21 099699872e global: fixes 2026-04-02 12:39:20 +02:00
nym21 5099903043 clients: bump versions 2026-04-01 23:00:31 +02:00
nym21 982fe47a33 pr: #32
v0.3.0-alpha.0
2026-04-01 22:41:39 +02:00
nym21 65d5fadd13 release: v0.3.0-alpha.0 2026-04-01 22:17:00 +02:00
nym21 b55f5255ad docs: update generated docs 2026-04-01 22:16:40 +02:00
nym21 83edef4806 coinbase: lossy to latin 2026-04-01 21:54:02 +02:00
nym21 d4936d889a clients: regened 2026-04-01 21:37:16 +02:00
nym21 c938cc8eae types: coinbase lossy format 2026-04-01 21:15:45 +02:00
nym21 0558834eef global: fixes 2026-04-01 21:11:20 +02:00
nym21 098950fdde deps: bumped 2026-04-01 18:25:11 +02:00
nym21 91e68a1d1e global: fixes 2026-04-01 18:18:56 +02:00
nym21 7172ddb247 global: snapshot 2026-04-01 17:51:50 +02:00
nym21 96f2e058f7 global: snapshot 2026-04-01 15:50:49 +02:00
nym21 8782944191 readme: add supporter 2026-03-31 23:00:15 +02:00
nym21 ae26db6df2 global: snapshot 2026-03-31 22:53:25 +02:00
nym21 d038141a8a global: snapshot 2026-03-29 23:10:31 +02:00
nym21 f6960c61d6 clients: regened 2026-03-29 17:47:47 +02:00
nym21 07fa2d2c9a release: v0.2.5 2026-03-29 17:33:10 +02:00
nym21 82c6d69a0b docs: update generated docs 2026-03-29 17:32:44 +02:00
nym21 d4dc1b9e49 deps: bumped 2026-03-29 17:26:48 +02:00
nym21 24d2b7b142 global: fmt 2026-03-28 11:56:51 +01:00
nym21 b6e56c4e9f Merge pull request #30 from yashbhutwala/codex/brk-website-static-serving-tests
[codex] add brk_website static serving tests
2026-03-27 23:02:39 +01:00
nym21 45c77a4c3b global: delay compaction 2026-03-27 23:02:31 +01:00
Yash Bhutwala 09af190ac0 add brk_website serving tests 2026-03-27 10:00:37 -04:00
nym21 d24f3691cb release: v0.2.4 2026-03-27 12:55:49 +01:00
nym21 daaaa15483 docs: update generated docs 2026-03-27 12:55:25 +01:00
nym21 041652d85d changelog + website: fixes 2026-03-27 12:50:35 +01:00
nym21 17570e12b8 release: v0.2.3 2026-03-26 23:56:58 +01:00
nym21 78172734db docs: update generated docs 2026-03-26 23:56:38 +01:00
nym21 19d4a193ff computer: renames 2026-03-26 23:39:28 +01:00
nym21 66680368b6 deps: bump vecdb 2026-03-26 22:06:16 +01:00
nym21 b4ded21ea3 website: chart fix 2026-03-26 22:02:15 +01:00
nym21 7412373d8a global: snapshot 2026-03-26 21:53:00 +01:00
nym21 259960b80b global: snapshot 2026-03-26 20:23:09 +01:00
nym21 18bb4186a8 global: snapshot 2026-03-26 15:57:22 +01:00
nym21 6d3307c0df docker: update heath check 2026-03-25 10:33:14 +01:00
nym21 6eea20b89a release: v0.2.2 2026-03-23 20:24:48 +01:00
nym21 5077cefda8 docs: update generated docs 2026-03-23 20:24:23 +01:00
nym21 14d7adfdd5 website: better llm files 2026-03-23 20:19:02 +01:00
nym21 000027fab8 changelog: updated 2026-03-23 18:30:32 +01:00
nym21 ade23795b8 clients: bump versions 2026-03-23 16:57:43 +01:00
1384 changed files with 45424 additions and 18853 deletions
+1
View File
@@ -39,6 +39,7 @@ flamegraph.svg
# AI # AI
.claude/settings* .claude/settings*
!CLAUDE.md
# Expand # Expand
expand.rs expand.rs
+1
View File
@@ -0,0 +1 @@
Codex will review your output once you are done.
Generated
+205 -243
View File
File diff suppressed because it is too large Load Diff
+37 -34
View File
@@ -4,7 +4,7 @@ members = ["crates/*"]
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node" package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
package.license = "MIT" package.license = "MIT"
package.edition = "2024" package.edition = "2024"
package.version = "0.2.1" package.version = "0.3.0-beta.4"
package.homepage = "https://bitcoinresearchkit.org" package.homepage = "https://bitcoinresearchkit.org"
package.repository = "https://github.com/bitcoinresearchkit/brk" package.repository = "https://github.com/bitcoinresearchkit/brk"
package.readme = "README.md" package.readme = "README.md"
@@ -36,58 +36,60 @@ inherits = "release"
debug = true debug = true
[workspace.dependencies] [workspace.dependencies]
aide = { version = "0.16.0-alpha.3", features = ["axum-json", "axum-query"] } aide = { version = "0.16.0-alpha.4", features = ["axum-json", "axum-query"] }
axum = { version = "0.8.8", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] } axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] }
bitcoin = { version = "0.32.8", features = ["serde"] } bitcoin = { version = "0.32.8", features = ["serde"] }
bitcoincore-rpc = "0.19.0" bitcoincore-rpc = "0.19.0"
brk_alloc = { version = "0.2.1", path = "crates/brk_alloc" } brk_alloc = { version = "0.3.0-beta.4", path = "crates/brk_alloc" }
brk_bencher = { version = "0.2.1", path = "crates/brk_bencher" } brk_bencher = { version = "0.3.0-beta.4", path = "crates/brk_bencher" }
brk_bindgen = { version = "0.2.1", path = "crates/brk_bindgen" } brk_bindgen = { version = "0.3.0-beta.4", path = "crates/brk_bindgen" }
brk_cli = { version = "0.2.1", path = "crates/brk_cli" } brk_cli = { version = "0.3.0-beta.4", path = "crates/brk_cli" }
brk_client = { version = "0.2.1", path = "crates/brk_client" } brk_client = { version = "0.3.0-beta.4", path = "crates/brk_client" }
brk_cohort = { version = "0.2.1", path = "crates/brk_cohort" } brk_cohort = { version = "0.3.0-beta.4", path = "crates/brk_cohort" }
brk_computer = { version = "0.2.1", path = "crates/brk_computer" } brk_computer = { version = "0.3.0-beta.4", path = "crates/brk_computer" }
brk_error = { version = "0.2.1", path = "crates/brk_error" } brk_error = { version = "0.3.0-beta.4", path = "crates/brk_error" }
brk_fetcher = { version = "0.2.1", path = "crates/brk_fetcher" } brk_fetcher = { version = "0.3.0-beta.4", path = "crates/brk_fetcher" }
brk_indexer = { version = "0.2.1", path = "crates/brk_indexer" } brk_indexer = { version = "0.3.0-beta.4", path = "crates/brk_indexer" }
brk_iterator = { version = "0.2.1", path = "crates/brk_iterator" } brk_iterator = { version = "0.3.0-beta.4", path = "crates/brk_iterator" }
brk_logger = { version = "0.2.1", path = "crates/brk_logger" } brk_logger = { version = "0.3.0-beta.4", path = "crates/brk_logger" }
brk_mempool = { version = "0.2.1", path = "crates/brk_mempool" } brk_mempool = { version = "0.3.0-beta.4", path = "crates/brk_mempool" }
brk_oracle = { version = "0.2.1", path = "crates/brk_oracle" } brk_oracle = { version = "0.3.0-beta.4", path = "crates/brk_oracle" }
brk_query = { version = "0.2.1", path = "crates/brk_query", features = ["tokio"] } brk_query = { version = "0.3.0-beta.4", path = "crates/brk_query", features = ["tokio"] }
brk_reader = { version = "0.2.1", path = "crates/brk_reader" } brk_reader = { version = "0.3.0-beta.4", path = "crates/brk_reader" }
brk_rpc = { version = "0.2.1", path = "crates/brk_rpc" } brk_rpc = { version = "0.3.0-beta.4", path = "crates/brk_rpc" }
brk_server = { version = "0.2.1", path = "crates/brk_server" } brk_server = { version = "0.3.0-beta.4", path = "crates/brk_server" }
brk_store = { version = "0.2.1", path = "crates/brk_store" } brk_store = { version = "0.3.0-beta.4", path = "crates/brk_store" }
brk_traversable = { version = "0.2.1", path = "crates/brk_traversable", features = ["pco", "derive"] } brk_traversable = { version = "0.3.0-beta.4", path = "crates/brk_traversable", features = ["pco", "derive"] }
brk_traversable_derive = { version = "0.2.1", path = "crates/brk_traversable_derive" } brk_traversable_derive = { version = "0.3.0-beta.4", path = "crates/brk_traversable_derive" }
brk_types = { version = "0.2.1", path = "crates/brk_types" } brk_types = { version = "0.3.0-beta.4", path = "crates/brk_types" }
brk_website = { version = "0.2.1", path = "crates/brk_website" } brk_website = { version = "0.3.0-beta.4", path = "crates/brk_website" }
byteview = "0.10.1" byteview = "0.10.1"
color-eyre = "0.6.5" color-eyre = "0.6.5"
corepc-client = { package = "brk-corepc-client", version = "0.11.0", features = ["client-sync"] } corepc-client = { package = "brk-corepc-client", version = "0.11.1", features = ["client-sync"] }
corepc-jsonrpc = { package = "brk-corepc-jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false } # corepc-client = { package = "brk-corepc-client", path = "../corepc/client", features = ["client-sync"] }
corepc-jsonrpc = { package = "brk-corepc-jsonrpc", version = "0.19.1", features = ["simple_http"], default-features = false }
# corepc-jsonrpc = { package = "brk-corepc-jsonrpc", path = "../corepc/jsonrpc", features = ["simple_http"], default-features = false }
derive_more = { version = "2.1.1", features = ["deref", "deref_mut"] } derive_more = { version = "2.1.1", features = ["deref", "deref_mut"] }
fjall = "3.1.2" fjall = "=3.0.4"
indexmap = { version = "2.13.0", features = ["serde"] } indexmap = { version = "2.14.0", features = ["serde"] }
jiff = { version = "0.2.23", features = ["perf-inline", "tz-system"], default-features = false } jiff = { version = "0.2.23", features = ["perf-inline", "tz-system"], default-features = false }
owo-colors = "4.3.0" owo-colors = "4.3.0"
parking_lot = "0.12.5" parking_lot = "0.12.5"
pco = "1.0.1" pco = "1.0.1"
rayon = "1.11.0" rayon = "1.12.0"
rustc-hash = "2.1.1" rustc-hash = "2.1.2"
schemars = { version = "1.2.1", features = ["indexmap2"] } schemars = { version = "1.2.1", features = ["indexmap2"] }
serde = "1.0.228" serde = "1.0.228"
serde_bytes = "0.11.19" serde_bytes = "0.11.19"
serde_derive = "1.0.228" serde_derive = "1.0.228"
serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] } serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] }
smallvec = "1.15.1" smallvec = "1.15.1"
tokio = { version = "1.50.0", features = ["rt-multi-thread"] } tokio = { version = "1.52.1", features = ["rt-multi-thread"] }
tower-http = { version = "0.6.8", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] } tower-http = { version = "0.6.8", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
tower-layer = "0.3" tower-layer = "0.3"
tracing = { version = "0.1", default-features = false, features = ["std"] } tracing = { version = "0.1", default-features = false, features = ["std"] }
ureq = { version = "3.3.0", features = ["json"] } ureq = { version = "3.3.0", features = ["json"] }
vecdb = { version = "0.7.2", features = ["derive", "serde_json", "pco", "schemars"] } vecdb = { version = "0.10.1", features = ["derive", "serde_json", "pco", "schemars"] }
# vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] } # vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] }
[workspace.metadata.release] [workspace.metadata.release]
@@ -95,6 +97,7 @@ shared-version = true
tag-name = "v{{version}}" tag-name = "v{{version}}"
pre-release-commit-message = "release: v{{version}}" pre-release-commit-message = "release: v{{version}}"
tag-message = "release: v{{version}}" tag-message = "release: v{{version}}"
allow-branch = ["main", "next"]
[workspace.metadata.dist] [workspace.metadata.dist]
cargo-dist-version = "0.30.2" cargo-dist-version = "0.30.2"
+2 -2
View File
@@ -8,5 +8,5 @@ homepage.workspace = true
repository.workspace = true repository.workspace = true
[dependencies] [dependencies]
libmimalloc-sys = { version = "0.1.44", features = ["extended"] } libmimalloc-sys = { version = "0.1.47", features = ["extended"] }
mimalloc = { version = "0.1.48", features = ["v3"] } mimalloc = { version = "0.1.50" }
+1 -1
View File
@@ -12,6 +12,6 @@ brk_cohort = { workspace = true }
brk_query = { workspace = true } brk_query = { workspace = true }
brk_types = { workspace = true } brk_types = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
oas3 = "0.20" oas3 = "0.21"
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
+5 -2
View File
@@ -3,7 +3,10 @@
//! This module detects repeating tree structures and analyzes them //! This module detects repeating tree structures and analyzes them
//! using the bottom-up name deconstruction algorithm. //! using the bottom-up name deconstruction algorithm.
use std::collections::{BTreeMap, BTreeSet}; use std::{
cmp::Reverse,
collections::{BTreeMap, BTreeSet},
};
use brk_types::{TreeNode, extract_json_type}; use brk_types::{TreeNode, extract_json_type};
@@ -111,7 +114,7 @@ pub fn detect_structural_patterns(
// Also collects node bases for each tree path // Also collects node bases for each tree path
let node_bases = analyze_pattern_modes(tree, &mut patterns, &pattern_lookup); let node_bases = analyze_pattern_modes(tree, &mut patterns, &pattern_lookup);
patterns.sort_by(|a, b| b.fields.len().cmp(&a.fields.len())); patterns.sort_by_key(|p| Reverse(p.fields.len()));
(patterns, concrete_to_pattern, type_mappings, node_bases) (patterns, concrete_to_pattern, type_mappings, node_bases)
} }
+293 -66
View File
@@ -108,7 +108,13 @@ fn fill_mixed_empty_field_parts(
// Recurse first (bottom-up) // Recurse first (bottom-up)
for (field_name, child_node) in children { for (field_name, child_node) in children {
let child_path = build_child_path(path, field_name); let child_path = build_child_path(path, field_name);
fill_mixed_empty_field_parts(child_node, &child_path, pattern_lookup, patterns, node_bases); fill_mixed_empty_field_parts(
child_node,
&child_path,
pattern_lookup,
patterns,
node_bases,
);
} }
// Check if this node has mixed empty/non-empty field_parts // Check if this node has mixed empty/non-empty field_parts
@@ -351,16 +357,18 @@ fn try_embedded_disc(
} }
/// Strategy 2: suffix discriminator (e.g., all field_parts differ by `_4y` suffix) /// Strategy 2: suffix discriminator (e.g., all field_parts differ by `_4y` suffix)
fn try_suffix_disc( fn try_suffix_disc(majority: &[&InstanceAnalysis], fields: &[PatternField]) -> Option<PatternMode> {
majority: &[&InstanceAnalysis],
fields: &[PatternField],
) -> Option<PatternMode> {
let first = &majority[0]; let first = &majority[0];
// Use a non-empty field to detect the suffix // Use a non-empty field to detect the suffix
let ref_field = fields let ref_field = fields
.iter() .iter()
.find(|f| first.field_parts.get(&f.name).is_some_and(|v| !v.is_empty())) .find(|f| {
first
.field_parts
.get(&f.name)
.is_some_and(|v| !v.is_empty())
})
.map(|f| &f.name)?; .map(|f| &f.name)?;
let ref_first = first.field_parts.get(ref_field)?; let ref_first = first.field_parts.get(ref_field)?;
@@ -763,19 +771,51 @@ mod tests {
fn test_embedded_disc_percentile_bands() { fn test_embedded_disc_percentile_bands() {
use std::collections::BTreeSet; use std::collections::BTreeSet;
let fields = vec![ let fields = vec![
PatternField { name: "bps".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, PatternField {
PatternField { name: "price".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, name: "bps".into(),
PatternField { name: "ratio".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
PatternField {
name: "price".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
PatternField {
name: "ratio".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
]; ];
let pct99 = InstanceAnalysis { let pct99 = InstanceAnalysis {
base: "realized_price".into(), base: "realized_price".into(),
field_parts: [("bps".into(), "ratio_pct99_bps".into()), ("price".into(), "pct99".into()), ("ratio".into(), "ratio_pct99".into())].into_iter().collect(), field_parts: [
is_suffix_mode: true, has_outlier: false, ("bps".into(), "ratio_pct99_bps".into()),
("price".into(), "pct99".into()),
("ratio".into(), "ratio_pct99".into()),
]
.into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: false,
}; };
let pct1 = InstanceAnalysis { let pct1 = InstanceAnalysis {
base: "realized_price".into(), base: "realized_price".into(),
field_parts: [("bps".into(), "ratio_pct1_bps".into()), ("price".into(), "pct1".into()), ("ratio".into(), "ratio_pct1".into())].into_iter().collect(), field_parts: [
is_suffix_mode: true, has_outlier: false, ("bps".into(), "ratio_pct1_bps".into()),
("price".into(), "pct1".into()),
("ratio".into(), "ratio_pct1".into()),
]
.into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: false,
}; };
let mode = determine_pattern_mode(&[pct99, pct1], &fields); let mode = determine_pattern_mode(&[pct99, pct1], &fields);
assert!(mode.is_some()); assert!(mode.is_some());
@@ -793,19 +833,51 @@ mod tests {
fn test_suffix_disc_period_windows() { fn test_suffix_disc_period_windows() {
use std::collections::BTreeSet; use std::collections::BTreeSet;
let fields = vec![ let fields = vec![
PatternField { name: "p1sd".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, PatternField {
PatternField { name: "sd".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, name: "p1sd".into(),
PatternField { name: "zscore".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
PatternField {
name: "sd".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
PatternField {
name: "zscore".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
]; ];
let all_time = InstanceAnalysis { let all_time = InstanceAnalysis {
base: "realized_price".into(), base: "realized_price".into(),
field_parts: [("p1sd".into(), "p1sd".into()), ("sd".into(), "ratio_sd".into()), ("zscore".into(), "ratio_zscore".into())].into_iter().collect(), field_parts: [
is_suffix_mode: true, has_outlier: false, ("p1sd".into(), "p1sd".into()),
("sd".into(), "ratio_sd".into()),
("zscore".into(), "ratio_zscore".into()),
]
.into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: false,
}; };
let four_year = InstanceAnalysis { let four_year = InstanceAnalysis {
base: "realized_price".into(), base: "realized_price".into(),
field_parts: [("p1sd".into(), "p1sd_4y".into()), ("sd".into(), "ratio_sd_4y".into()), ("zscore".into(), "ratio_zscore_4y".into())].into_iter().collect(), field_parts: [
is_suffix_mode: true, has_outlier: false, ("p1sd".into(), "p1sd_4y".into()),
("sd".into(), "ratio_sd_4y".into()),
("zscore".into(), "ratio_zscore_4y".into()),
]
.into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: false,
}; };
let mode = determine_pattern_mode(&[all_time, four_year], &fields); let mode = determine_pattern_mode(&[all_time, four_year], &fields);
assert!(mode.is_some()); assert!(mode.is_some());
@@ -823,18 +895,39 @@ mod tests {
fn test_suffix_disc_with_empty_fields() { fn test_suffix_disc_with_empty_fields() {
use std::collections::BTreeSet; use std::collections::BTreeSet;
let fields = vec![ let fields = vec![
PatternField { name: "band".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, PatternField {
PatternField { name: "sd".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, name: "band".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
PatternField {
name: "sd".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
]; ];
let all_time = InstanceAnalysis { let all_time = InstanceAnalysis {
base: "price".into(), base: "price".into(),
field_parts: [("band".into(), "".into()), ("sd".into(), "ratio_sd".into())].into_iter().collect(), field_parts: [("band".into(), "".into()), ("sd".into(), "ratio_sd".into())]
is_suffix_mode: true, has_outlier: false, .into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: false,
}; };
let four_year = InstanceAnalysis { let four_year = InstanceAnalysis {
base: "price".into(), base: "price".into(),
field_parts: [("band".into(), "".into()), ("sd".into(), "ratio_sd_4y".into())].into_iter().collect(), field_parts: [
is_suffix_mode: true, has_outlier: false, ("band".into(), "".into()),
("sd".into(), "ratio_sd_4y".into()),
]
.into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: false,
}; };
let mode = determine_pattern_mode(&[all_time, four_year], &fields); let mode = determine_pattern_mode(&[all_time, four_year], &fields);
assert!(mode.is_some()); assert!(mode.is_some());
@@ -851,18 +944,39 @@ mod tests {
fn test_suffix_disc_empty_to_nonempty() { fn test_suffix_disc_empty_to_nonempty() {
use std::collections::BTreeSet; use std::collections::BTreeSet;
let fields = vec![ let fields = vec![
PatternField { name: "all".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, PatternField {
PatternField { name: "sth".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, name: "all".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
PatternField {
name: "sth".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
]; ];
let regular = InstanceAnalysis { let regular = InstanceAnalysis {
base: "supply".into(), base: "supply".into(),
field_parts: [("all".into(), "".into()), ("sth".into(), "sth_".into())].into_iter().collect(), field_parts: [("all".into(), "".into()), ("sth".into(), "sth_".into())]
is_suffix_mode: true, has_outlier: false, .into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: false,
}; };
let profitability = InstanceAnalysis { let profitability = InstanceAnalysis {
base: "utxos_in_profit".into(), base: "utxos_in_profit".into(),
field_parts: [("all".into(), "supply".into()), ("sth".into(), "sth_supply".into())].into_iter().collect(), field_parts: [
is_suffix_mode: true, has_outlier: false, ("all".into(), "supply".into()),
("sth".into(), "sth_supply".into()),
]
.into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: false,
}; };
let mode = determine_pattern_mode(&[regular, profitability], &fields); let mode = determine_pattern_mode(&[regular, profitability], &fields);
assert!(mode.is_some()); assert!(mode.is_some());
@@ -879,43 +993,91 @@ mod tests {
fn test_outlier_rejects_pattern() { fn test_outlier_rejects_pattern() {
use std::collections::BTreeSet; use std::collections::BTreeSet;
let fields = vec![ let fields = vec![
PatternField { name: "ratio".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, PatternField {
PatternField { name: "value".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, name: "ratio".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
PatternField {
name: "value".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
]; ];
// SOPR case: one instance has outlier naming (no common prefix) // SOPR case: one instance has outlier naming (no common prefix)
let normal = InstanceAnalysis { let normal = InstanceAnalysis {
base: "series".into(), base: "series".into(),
field_parts: [("ratio".into(), "ratio".into()), ("value".into(), "value".into())].into_iter().collect(), field_parts: [
is_suffix_mode: true, has_outlier: false, ("ratio".into(), "ratio".into()),
("value".into(), "value".into()),
]
.into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: false,
}; };
let outlier = InstanceAnalysis { let outlier = InstanceAnalysis {
base: "".into(), base: "".into(),
field_parts: [("ratio".into(), "asopr".into()), ("value".into(), "adj_value".into())].into_iter().collect(), field_parts: [
is_suffix_mode: true, has_outlier: true, ("ratio".into(), "asopr".into()),
("value".into(), "adj_value".into()),
]
.into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: true,
}; };
let mode = determine_pattern_mode(&[normal, outlier], &fields); let mode = determine_pattern_mode(&[normal, outlier], &fields);
assert!(mode.is_some(), "Outlier should be filtered out, leaving a valid pattern from non-outlier instances"); assert!(
mode.is_some(),
"Outlier should be filtered out, leaving a valid pattern from non-outlier instances"
);
} }
#[test] #[test]
fn test_unanimity_rejects_disagreeing_instances() { fn test_unanimity_rejects_disagreeing_instances() {
use std::collections::BTreeSet; use std::collections::BTreeSet;
let fields = vec![ let fields = vec![
PatternField { name: "a".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, PatternField {
PatternField { name: "b".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, name: "a".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
PatternField {
name: "b".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
]; ];
let inst1 = InstanceAnalysis { let inst1 = InstanceAnalysis {
base: "x".into(), base: "x".into(),
field_parts: [("a".into(), "foo".into()), ("b".into(), "bar".into())].into_iter().collect(), field_parts: [("a".into(), "foo".into()), ("b".into(), "bar".into())]
is_suffix_mode: true, has_outlier: false, .into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: false,
}; };
let inst2 = InstanceAnalysis { let inst2 = InstanceAnalysis {
base: "y".into(), base: "y".into(),
field_parts: [("a".into(), "baz".into()), ("b".into(), "qux".into())].into_iter().collect(), field_parts: [("a".into(), "baz".into()), ("b".into(), "qux".into())]
is_suffix_mode: true, has_outlier: false, .into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: false,
}; };
let mode = determine_pattern_mode(&[inst1, inst2], &fields); let mode = determine_pattern_mode(&[inst1, inst2], &fields);
assert!(mode.is_none(), "Should be non-parameterizable when no pattern detected"); assert!(
mode.is_none(),
"Should be non-parameterizable when no pattern detected"
);
} }
#[test] #[test]
@@ -925,20 +1087,43 @@ mod tests {
// Should keep identity (empty parts) so both children receive acc unchanged. // Should keep identity (empty parts) so both children receive acc unchanged.
use std::collections::BTreeSet; use std::collections::BTreeSet;
let fields = vec![ let fields = vec![
PatternField { name: "absolute".into(), rust_type: "TypeA".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, PatternField {
PatternField { name: "rate".into(), rust_type: "TypeB".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, name: "absolute".into(),
rust_type: "TypeA".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
PatternField {
name: "rate".into(),
rust_type: "TypeB".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
]; ];
let inst = InstanceAnalysis { let inst = InstanceAnalysis {
base: "supply_delta".into(), base: "supply_delta".into(),
field_parts: [("absolute".into(), "".into()), ("rate".into(), "".into())].into_iter().collect(), field_parts: [("absolute".into(), "".into()), ("rate".into(), "".into())]
is_suffix_mode: true, has_outlier: false, .into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: false,
}; };
let mode = determine_pattern_mode(&[inst], &fields); let mode = determine_pattern_mode(&[inst], &fields);
assert!(mode.is_some()); assert!(mode.is_some());
match mode.unwrap() { match mode.unwrap() {
PatternMode::Suffix { relatives } => { PatternMode::Suffix { relatives } => {
assert_eq!(relatives.get("absolute"), Some(&"".to_string()), "absolute should be identity"); assert_eq!(
assert_eq!(relatives.get("rate"), Some(&"".to_string()), "rate should be identity"); relatives.get("absolute"),
Some(&"".to_string()),
"absolute should be identity"
);
assert_eq!(
relatives.get("rate"),
Some(&"".to_string()),
"rate should be identity"
);
} }
other => panic!("Expected Suffix with identity, got {:?}", other), other => panic!("Expected Suffix with identity, got {:?}", other),
} }
@@ -975,16 +1160,26 @@ mod tests {
// Parent patterns containing non-parameterizable children should also // Parent patterns containing non-parameterizable children should also
// be detected via metadata.is_parameterizable (recursive check). // be detected via metadata.is_parameterizable (recursive check).
use std::collections::BTreeSet; use std::collections::BTreeSet;
let fields = vec![ let fields = vec![PatternField {
PatternField { name: "a".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, name: "a".into(),
]; rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
}];
let inst = InstanceAnalysis { let inst = InstanceAnalysis {
base: "".into(), base: "".into(),
field_parts: [("a".into(), "standalone_name".into())].into_iter().collect(), field_parts: [("a".into(), "standalone_name".into())]
is_suffix_mode: true, has_outlier: true, .into_iter()
.collect(),
is_suffix_mode: true,
has_outlier: true,
}; };
let mode = determine_pattern_mode(&[inst], &fields); let mode = determine_pattern_mode(&[inst], &fields);
assert!(mode.is_none(), "Pattern with outlier should be non-parameterizable"); assert!(
mode.is_none(),
"Pattern with outlier should be non-parameterizable"
);
} }
#[test] #[test]
@@ -998,9 +1193,27 @@ mod tests {
let pattern = StructuralPattern { let pattern = StructuralPattern {
name: "TestPattern".into(), name: "TestPattern".into(),
fields: vec![ fields: vec![
PatternField { name: "_0sd".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, PatternField {
PatternField { name: "p1sd".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, name: "_0sd".into(),
PatternField { name: "sd".into(), rust_type: "T".into(), json_type: "n".into(), indexes: BTreeSet::new(), type_param: None }, rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
PatternField {
name: "p1sd".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
PatternField {
name: "sd".into(),
rust_type: "T".into(),
json_type: "n".into(),
indexes: BTreeSet::new(),
type_param: None,
},
], ],
mode: Some(PatternMode::Templated { mode: Some(PatternMode::Templated {
templates: [ templates: [
@@ -1059,9 +1272,15 @@ mod tests {
assert_eq!(analysis.field_parts.get("loss"), Some(&"".to_string())); assert_eq!(analysis.field_parts.get("loss"), Some(&"".to_string()));
assert_eq!(analysis.field_parts.get("supply"), Some(&"".to_string())); assert_eq!(analysis.field_parts.get("supply"), Some(&"".to_string()));
// others should be non-empty // others should be non-empty
assert_eq!(analysis.field_parts.get("cap"), Some(&"realized_cap".to_string())); assert_eq!(
analysis.field_parts.get("cap"),
Some(&"realized_cap".to_string())
);
assert_eq!(analysis.field_parts.get("mvrv"), Some(&"mvrv".to_string())); assert_eq!(analysis.field_parts.get("mvrv"), Some(&"mvrv".to_string()));
assert_eq!(analysis.field_parts.get("price"), Some(&"realized_price".to_string())); assert_eq!(
analysis.field_parts.get("price"),
Some(&"realized_price".to_string())
);
} }
#[test] #[test]
@@ -1111,12 +1330,20 @@ mod tests {
&mut path_to_pattern, &mut path_to_pattern,
); );
let result = node_bases.get("test").expect("should have node_bases entry"); let result = node_bases
.get("test")
.expect("should have node_bases entry");
assert_eq!(result.base, "utxos"); assert_eq!(result.base, "utxos");
assert!(!result.has_outlier); assert!(!result.has_outlier);
assert_eq!(result.field_parts.get("cap"), Some(&"realized_cap".to_string())); assert_eq!(
result.field_parts.get("cap"),
Some(&"realized_cap".to_string())
);
assert_eq!(result.field_parts.get("mvrv"), Some(&"mvrv".to_string())); assert_eq!(result.field_parts.get("mvrv"), Some(&"mvrv".to_string()));
// loss branch returns base "utxos_realized_loss" which yields field_part "realized_loss" // loss branch returns base "utxos_realized_loss" which yields field_part "realized_loss"
assert_eq!(result.field_parts.get("loss"), Some(&"realized_loss".to_string())); assert_eq!(
result.field_parts.get("loss"),
Some(&"realized_loss".to_string())
);
} }
} }
+8 -5
View File
@@ -6,9 +6,9 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use brk_cohort::{ use brk_cohort::{
AGE_RANGE_NAMES, AMOUNT_RANGE_NAMES, CLASS_NAMES, EPOCH_NAMES, LOSS_NAMES, AGE_RANGE_NAMES, AMOUNT_RANGE_NAMES, CLASS_NAMES, EPOCH_NAMES, LOSS_NAMES, OVER_AGE_NAMES,
OVER_AGE_NAMES, OVER_AMOUNT_NAMES, PROFITABILITY_RANGE_NAMES, PROFIT_NAMES, OVER_AMOUNT_NAMES, PROFIT_NAMES, PROFITABILITY_RANGE_NAMES, SPENDABLE_TYPE_NAMES, TERM_NAMES,
SPENDABLE_TYPE_NAMES, TERM_NAMES, UNDER_AGE_NAMES, UNDER_AMOUNT_NAMES, UNDER_AGE_NAMES, UNDER_AMOUNT_NAMES,
}; };
use brk_types::{Index, PoolSlug, pools}; use brk_types::{Index, PoolSlug, pools};
use serde::Serialize; use serde::Serialize;
@@ -31,7 +31,7 @@ impl ClientConstants {
let pools = pools(); let pools = pools();
let mut sorted_pools: Vec<_> = pools.iter().collect(); let mut sorted_pools: Vec<_> = pools.iter().collect();
sorted_pools.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase())); sorted_pools.sort_by_key(|p| p.name.to_lowercase());
let pool_map: BTreeMap<PoolSlug, &'static str> = let pool_map: BTreeMap<PoolSlug, &'static str> =
sorted_pools.iter().map(|p| (p.slug(), p.name)).collect(); sorted_pools.iter().map(|p| (p.slug(), p.name)).collect();
@@ -64,7 +64,10 @@ impl CohortConstants {
("AMOUNT_RANGE_NAMES", to_value(&AMOUNT_RANGE_NAMES)), ("AMOUNT_RANGE_NAMES", to_value(&AMOUNT_RANGE_NAMES)),
("OVER_AMOUNT_NAMES", to_value(&OVER_AMOUNT_NAMES)), ("OVER_AMOUNT_NAMES", to_value(&OVER_AMOUNT_NAMES)),
("UNDER_AMOUNT_NAMES", to_value(&UNDER_AMOUNT_NAMES)), ("UNDER_AMOUNT_NAMES", to_value(&UNDER_AMOUNT_NAMES)),
("PROFITABILITY_RANGE_NAMES", to_value(&PROFITABILITY_RANGE_NAMES)), (
"PROFITABILITY_RANGE_NAMES",
to_value(&PROFITABILITY_RANGE_NAMES),
),
("PROFIT_NAMES", to_value(&PROFIT_NAMES)), ("PROFIT_NAMES", to_value(&PROFIT_NAMES)),
("LOSS_NAMES", to_value(&LOSS_NAMES)), ("LOSS_NAMES", to_value(&LOSS_NAMES)),
] ]
+4 -4
View File
@@ -8,7 +8,9 @@ use std::fmt::Write;
use brk_types::SeriesLeafWithSchema; use brk_types::SeriesLeafWithSchema;
use crate::{ClientMetadata, LanguageSyntax, PatternBaseResult, PatternField, PatternMode, StructuralPattern}; use crate::{
ClientMetadata, LanguageSyntax, PatternBaseResult, PatternField, PatternMode, StructuralPattern,
};
/// Create a path suffix from a name. /// Create a path suffix from a name.
fn path_suffix(name: &str) -> String { fn path_suffix(name: &str) -> String {
@@ -33,9 +35,7 @@ fn compute_parameterized_value<S: LanguageSyntax>(
if let Some(child_pattern) = metadata.find_pattern(&field.rust_type) if let Some(child_pattern) = metadata.find_pattern(&field.rust_type)
&& child_pattern.is_templated() && child_pattern.is_templated()
{ {
let disc_template = pattern let disc_template = pattern.get_field_part(&field.name).unwrap_or(&field.name);
.get_field_part(&field.name)
.unwrap_or(&field.name);
let disc_arg = syntax.disc_arg_expr(disc_template); let disc_arg = syntax.disc_arg_expr(disc_template);
let acc_arg = syntax.owned_expr("acc"); let acc_arg = syntax.owned_expr("acc");
return syntax.constructor(&field.rust_type, &format!("{acc_arg}, {disc_arg}")); return syntax.constructor(&field.rust_type, &format!("{acc_arg}, {disc_arg}"));
+2 -2
View File
@@ -125,8 +125,8 @@ pub fn prepare_tree_node<'a>(
p.is_suffix_mode() == base_result.is_suffix_mode p.is_suffix_mode() == base_result.is_suffix_mode
&& p.field_parts_match(&base_result.field_parts) && p.field_parts_match(&base_result.field_parts)
}); });
let is_parameterizable = matching_pattern let is_parameterizable =
.is_none_or(|p| metadata.is_parameterizable(&p.name)); matching_pattern.is_none_or(|p| metadata.is_parameterizable(&p.name));
// should_inline determines if we generate an inline struct type // should_inline determines if we generate an inline struct type
let should_inline = !is_leaf let should_inline = !is_leaf
@@ -69,31 +69,49 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
.unwrap(); .unwrap();
} }
writeln!(
output,
" * @param {{{{ signal?: AbortSignal, onUpdate?: (value: {}) => void }}}} [options]",
return_type
)
.unwrap();
writeln!(output, " * @returns {{Promise<{}>}}", return_type).unwrap(); writeln!(output, " * @returns {{Promise<{}>}}", return_type).unwrap();
writeln!(output, " */").unwrap(); writeln!(output, " */").unwrap();
let params = build_method_params(endpoint); let params = build_method_params(endpoint);
writeln!(output, " async {}({}) {{", method_name, params).unwrap(); let params_with_opts = if params.is_empty() {
"{ signal, onUpdate } = {}".to_string()
} else {
format!("{}, {{ signal, onUpdate }} = {{}}", params)
};
writeln!(output, " async {}({}) {{", method_name, params_with_opts).unwrap();
let path = build_path_template(&endpoint.path, &endpoint.path_params); let path = build_path_template(&endpoint.path, &endpoint.path_params);
let fetch_call = if endpoint.returns_json() {
"this.getJson(path, { signal, onUpdate })"
} else {
"this.getText(path, { signal, onUpdate })"
};
if endpoint.query_params.is_empty() { if endpoint.query_params.is_empty() {
writeln!(output, " return this.getJson(`{}`);", path).unwrap(); writeln!(output, " const path = `{}`;", path).unwrap();
} else { } else {
writeln!(output, " const params = new URLSearchParams();").unwrap(); writeln!(output, " const params = new URLSearchParams();").unwrap();
for param in &endpoint.query_params { for param in &endpoint.query_params {
let ident = sanitize_ident(&param.name);
if param.required { if param.required {
writeln!( writeln!(
output, output,
" params.set('{}', String({}));", " params.set('{}', String({}));",
param.name, param.name param.name, ident
) )
.unwrap(); .unwrap();
} else { } else {
writeln!( writeln!(
output, output,
" if ({} !== undefined) params.set('{}', String({}));", " if ({} !== undefined) params.set('{}', String({}));",
param.name, param.name, param.name ident, param.name, ident
) )
.unwrap(); .unwrap();
} }
@@ -105,17 +123,13 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
path path
) )
.unwrap(); .unwrap();
if endpoint.supports_csv {
writeln!(output, " if (format === 'csv') {{").unwrap();
writeln!(output, " return this.getText(path);").unwrap();
writeln!(output, " }}").unwrap();
writeln!(output, " return this.getJson(path);").unwrap();
} else {
writeln!(output, " return this.getJson(path);").unwrap();
}
} }
if endpoint.supports_csv {
writeln!(output, " if (format === 'csv') return this.getText(path, {{ signal, onUpdate }});").unwrap();
}
writeln!(output, " return {};", fetch_call).unwrap();
writeln!(output, " }}\n").unwrap(); writeln!(output, " }}\n").unwrap();
} }
} }
@@ -127,14 +141,19 @@ fn endpoint_to_method_name(endpoint: &Endpoint) -> String {
fn build_method_params(endpoint: &Endpoint) -> String { fn build_method_params(endpoint: &Endpoint) -> String {
let mut params = Vec::new(); let mut params = Vec::new();
for param in &endpoint.path_params { for param in &endpoint.path_params {
params.push(param.name.clone()); params.push(sanitize_ident(&param.name));
} }
for param in &endpoint.query_params { for param in &endpoint.query_params {
params.push(param.name.clone()); params.push(sanitize_ident(&param.name));
} }
params.join(", ") params.join(", ")
} }
/// Strip characters invalid in JS identifiers (e.g. `[]` from `txId[]`).
fn sanitize_ident(name: &str) -> String {
name.replace(['[', ']'], "")
}
fn build_path_template(path: &str, path_params: &[Parameter]) -> String { fn build_path_template(path: &str, path_params: &[Parameter]) -> String {
let mut result = path.to_string(); let mut result = path.to_string();
for param in path_params { for param in path_params {
@@ -22,6 +22,20 @@ pub fn generate_base_client(output: &mut String) {
const _isBrowser = typeof window !== 'undefined' && 'caches' in window; const _isBrowser = typeof window !== 'undefined' && 'caches' in window;
const _runIdle = (/** @type {{VoidFunction}} */ fn) => (globalThis.requestIdleCallback ?? setTimeout)(fn); const _runIdle = (/** @type {{VoidFunction}} */ fn) => (globalThis.requestIdleCallback ?? setTimeout)(fn);
const _defaultCacheName = '__BRK_CLIENT__'; const _defaultCacheName = '__BRK_CLIENT__';
/** @param {{*}} v */
const _addCamelGetters = (v) => {{
if (Array.isArray(v)) {{ v.forEach(_addCamelGetters); return v; }}
if (v && typeof v === 'object' && v.constructor === Object) {{
for (const k in v) {{
if (k.includes('_')) {{
const c = k.replace(/_([a-z])/g, (_, l) => l.toUpperCase());
if (!(c in v)) Object.defineProperty(v, c, {{ get() {{ return this[k]; }} }});
}}
_addCamelGetters(v[k]);
}}
}}
return v;
}};
/** /**
* @param {{string|boolean|undefined}} cache * @param {{string|boolean|undefined}} cache
@@ -390,23 +404,28 @@ class BrkClientBase {{
/** /**
* @param {{string}} path * @param {{string}} path
* @param {{{{ signal?: AbortSignal }}}} [options]
* @returns {{Promise<Response>}} * @returns {{Promise<Response>}}
*/ */
async get(path) {{ async get(path, {{ signal }} = {{}}) {{
const url = `${{this.baseUrl}}${{path}}`; const url = `${{this.baseUrl}}${{path}}`;
const res = await fetch(url, {{ signal: AbortSignal.timeout(this.timeout) }}); const signals = [AbortSignal.timeout(this.timeout)];
if (signal) signals.push(signal);
const res = await fetch(url, {{ signal: AbortSignal.any(signals) }});
if (!res.ok) throw new BrkError(`HTTP ${{res.status}}: ${{url}}`, res.status); if (!res.ok) throw new BrkError(`HTTP ${{res.status}}: ${{url}}`, res.status);
return res; return res;
}} }}
/** /**
* Make a GET request - races cache vs network, first to resolve calls onUpdate * Make a GET request - races cache vs network, first to resolve calls onUpdate.
* Shared implementation backing `getJson` and `getText`.
* @template T * @template T
* @param {{string}} path * @param {{string}} path
* @param {{(value: T) => void}} [onUpdate] - Called when data is available (may be called twice: cache then network) * @param {{(res: Response) => Promise<T>}} parse - Response body reader
* @param {{{{ onUpdate?: (value: T) => void, signal?: AbortSignal }}}} [options]
* @returns {{Promise<T>}} * @returns {{Promise<T>}}
*/ */
async getJson(path, onUpdate) {{ async _getCached(path, parse, {{ onUpdate, signal }} = {{}}) {{
const url = `${{this.baseUrl}}${{path}}`; const url = `${{this.baseUrl}}${{path}}`;
const cache = this._cache ?? await this._cachePromise; const cache = this._cache ?? await this._cachePromise;
@@ -418,51 +437,61 @@ class BrkClientBase {{
const cachePromise = cache?.match(url).then(async (res) => {{ const cachePromise = cache?.match(url).then(async (res) => {{
cachedRes = res ?? null; cachedRes = res ?? null;
if (!res) return null; if (!res) return null;
const json = await res.json(); const value = await parse(res);
if (!resolved && onUpdate) {{ if (!resolved && onUpdate) {{
resolved = true; resolved = true;
onUpdate(json); onUpdate(value);
}} }}
return json; return value;
}}); }});
const networkPromise = this.get(path).then(async (res) => {{ const networkPromise = this.get(path, {{ signal }}).then(async (res) => {{
const cloned = res.clone(); const cloned = res.clone();
const json = await res.json(); const value = await parse(res);
// Skip update if ETag matches and cache already delivered // Skip update if ETag matches and cache already delivered
if (cachedRes?.headers.get('ETag') === res.headers.get('ETag')) {{ if (cachedRes?.headers.get('ETag') === res.headers.get('ETag')) {{
if (!resolved && onUpdate) {{ if (!resolved && onUpdate) {{
resolved = true; resolved = true;
onUpdate(json); onUpdate(value);
}} }}
return json; return value;
}} }}
resolved = true; resolved = true;
if (onUpdate) {{ if (onUpdate) onUpdate(value);
onUpdate(json);
}}
if (cache) _runIdle(() => cache.put(url, cloned)); if (cache) _runIdle(() => cache.put(url, cloned));
return json; return value;
}}); }});
try {{ try {{
return await networkPromise; return await networkPromise;
}} catch (e) {{ }} catch (e) {{
// Network failed - wait for cache // Network failed - wait for cache
const cachedJson = await cachePromise?.catch(() => null); const cachedValue = await cachePromise?.catch(() => null);
if (cachedJson) return cachedJson; if (cachedValue != null) return cachedValue;
throw e; throw e;
}} }}
}} }}
/** /**
* Make a GET request and return raw text (for CSV responses) * Make a GET request expecting a JSON response. Cached and supports `onUpdate`.
* @template T
* @param {{string}} path * @param {{string}} path
* @param {{{{ onUpdate?: (value: T) => void, signal?: AbortSignal }}}} [options]
* @returns {{Promise<T>}}
*/
getJson(path, options) {{
return this._getCached(path, async (res) => _addCamelGetters(await res.json()), options);
}}
/**
* Make a GET request expecting a text response (text/plain, text/csv, ...).
* Cached and supports `onUpdate`, same as `getJson`.
* @param {{string}} path
* @param {{{{ onUpdate?: (value: string) => void, signal?: AbortSignal }}}} [options]
* @returns {{Promise<string>}} * @returns {{Promise<string>}}
*/ */
async getText(path) {{ getText(path, options) {{
const res = await this.get(path); return this._getCached(path, (res) => res.text(), options);
return res.text();
}} }}
/** /**
@@ -474,7 +503,7 @@ class BrkClientBase {{
*/ */
async _fetchSeriesData(path, onUpdate) {{ async _fetchSeriesData(path, onUpdate) {{
const wrappedOnUpdate = onUpdate ? (/** @type {{SeriesData<T>}} */ raw) => onUpdate(_wrapSeriesData(raw)) : undefined; const wrappedOnUpdate = onUpdate ? (/** @type {{SeriesData<T>}} */ raw) => onUpdate(_wrapSeriesData(raw)) : undefined;
const raw = await this.getJson(path, wrappedOnUpdate); const raw = await this.getJson(path, {{ onUpdate: wrappedOnUpdate }});
return _wrapSeriesData(raw); return _wrapSeriesData(raw);
}} }}
}} }}
@@ -726,7 +755,12 @@ pub fn generate_structural_patterns(
writeln!(output, " */").unwrap(); writeln!(output, " */").unwrap();
if pattern.is_templated() { if pattern.is_templated() {
writeln!(output, "function create{}(client, acc, disc) {{", pattern.name).unwrap(); writeln!(
output,
"function create{}(client, acc, disc) {{",
pattern.name
)
.unwrap();
} else { } else {
writeln!(output, "function create{}(client, acc) {{", pattern.name).unwrap(); writeln!(output, "function create{}(client, acc) {{", pattern.name).unwrap();
} }
@@ -108,15 +108,14 @@ pub fn generate_main_client(
writeln!(output, " constructor(options) {{").unwrap(); writeln!(output, " constructor(options) {{").unwrap();
writeln!(output, " super(options);").unwrap(); writeln!(output, " super(options);").unwrap();
writeln!(output, " /** @type {{SeriesTree}} */").unwrap(); writeln!(output, " /** @type {{SeriesTree}} */").unwrap();
writeln!(output, " this.series = this._buildTree('');").unwrap(); writeln!(output, " this.series = this._buildTree();").unwrap();
writeln!(output, " }}\n").unwrap(); writeln!(output, " }}\n").unwrap();
writeln!(output, " /**").unwrap(); writeln!(output, " /**").unwrap();
writeln!(output, " * @private").unwrap(); writeln!(output, " * @private").unwrap();
writeln!(output, " * @param {{string}} basePath").unwrap();
writeln!(output, " * @returns {{SeriesTree}}").unwrap(); writeln!(output, " * @returns {{SeriesTree}}").unwrap();
writeln!(output, " */").unwrap(); writeln!(output, " */").unwrap();
writeln!(output, " _buildTree(basePath) {{").unwrap(); writeln!(output, " _buildTree() {{").unwrap();
writeln!(output, " return {{").unwrap(); writeln!(output, " return {{").unwrap();
let mut generated = BTreeSet::new(); let mut generated = BTreeSet::new();
generate_tree_initializer( generate_tree_initializer(
+12 -10
View File
@@ -56,11 +56,7 @@ pub fn generate_main_client(output: &mut String, endpoints: &[Endpoint]) {
) )
.unwrap(); .unwrap();
writeln!(output, " \"\"\"").unwrap(); writeln!(output, " \"\"\"").unwrap();
writeln!( writeln!(output, " return SeriesEndpoint(self, series, index)").unwrap();
output,
" return SeriesEndpoint(self, series, index)"
)
.unwrap();
writeln!(output).unwrap(); writeln!(output).unwrap();
// Generate helper methods // Generate helper methods
@@ -105,7 +101,7 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
.response_type .response_type
.as_deref() .as_deref()
.map(js_type_to_python) .map(js_type_to_python)
.unwrap_or_else(|| "Any".to_string()), .unwrap_or_else(|| "str".to_string()),
); );
let return_type = if endpoint.supports_csv { let return_type = if endpoint.supports_csv {
@@ -163,11 +159,17 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
// Build path // Build path
let path = build_path_template(&endpoint.path, &endpoint.path_params); let path = build_path_template(&endpoint.path, &endpoint.path_params);
let fetch_method = if endpoint.returns_json() {
"get_json"
} else {
"get_text"
};
if endpoint.query_params.is_empty() { if endpoint.query_params.is_empty() {
if endpoint.path_params.is_empty() { if endpoint.path_params.is_empty() {
writeln!(output, " return self.get_json('{}')", path).unwrap(); writeln!(output, " return self.{}('{}')", fetch_method, path).unwrap();
} else { } else {
writeln!(output, " return self.get_json(f'{}')", path).unwrap(); writeln!(output, " return self.{}(f'{}')", fetch_method, path).unwrap();
} }
} else { } else {
writeln!(output, " params = []").unwrap(); writeln!(output, " params = []").unwrap();
@@ -201,9 +203,9 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
if endpoint.supports_csv { if endpoint.supports_csv {
writeln!(output, " if format == 'csv':").unwrap(); writeln!(output, " if format == 'csv':").unwrap();
writeln!(output, " return self.get_text(path)").unwrap(); writeln!(output, " return self.get_text(path)").unwrap();
writeln!(output, " return self.get_json(path)").unwrap(); writeln!(output, " return self.{}(path)", fetch_method).unwrap();
} else { } else {
writeln!(output, " return self.get_json(path)").unwrap(); writeln!(output, " return self.{}(path)", fetch_method).unwrap();
} }
} }
@@ -684,7 +684,6 @@ pub fn generate_structural_patterns(
writeln!(output, "# Reusable structural pattern classes\n").unwrap(); writeln!(output, "# Reusable structural pattern classes\n").unwrap();
for pattern in patterns { for pattern in patterns {
// Generate class // Generate class
if pattern.is_generic { if pattern.is_generic {
writeln!(output, "class {}(Generic[T]):", pattern.name).unwrap(); writeln!(output, "class {}(Generic[T]):", pattern.name).unwrap();
+36 -12
View File
@@ -93,7 +93,7 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
.response_type .response_type
.as_deref() .as_deref()
.map(js_type_to_rust) .map(js_type_to_rust)
.unwrap_or_else(|| "serde_json::Value".to_string()); .unwrap_or_else(|| "String".to_string());
let return_type = if endpoint.supports_csv { let return_type = if endpoint.supports_csv {
format!("FormatResponse<{}>", base_return_type) format!("FormatResponse<{}>", base_return_type)
@@ -132,29 +132,43 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
.unwrap(); .unwrap();
let (path, index_arg) = build_path_template(endpoint); let (path, index_arg) = build_path_template(endpoint);
let fetch_method = if endpoint.returns_json() {
"get_json"
} else {
"get_text"
};
if endpoint.query_params.is_empty() { if endpoint.query_params.is_empty() {
writeln!( writeln!(
output, output,
" self.base.get_json(&format!(\"{}\"{}))", " self.base.{}(&format!(\"{}\"{}))",
path, index_arg fetch_method, path, index_arg
) )
.unwrap(); .unwrap();
} else { } else {
writeln!(output, " let mut query = Vec::new();").unwrap(); writeln!(output, " let mut query = Vec::new();").unwrap();
for param in &endpoint.query_params { for param in &endpoint.query_params {
if param.required { let ident = sanitize_ident(&param.name);
let is_array = param.param_type.ends_with("[]");
if is_array {
writeln!(
output,
" for v in {} {{ query.push(format!(\"{}={{}}\", v)); }}",
ident, param.name
)
.unwrap();
} else if param.required {
writeln!( writeln!(
output, output,
" query.push(format!(\"{}={{}}\", {}));", " query.push(format!(\"{}={{}}\", {}));",
param.name, param.name param.name, ident
) )
.unwrap(); .unwrap();
} else { } else {
writeln!( writeln!(
output, output,
" if let Some(v) = {} {{ query.push(format!(\"{}={{}}\", v)); }}", " if let Some(v) = {} {{ query.push(format!(\"{}={{}}\", v)); }}",
param.name, param.name ident, param.name
) )
.unwrap(); .unwrap();
} }
@@ -177,12 +191,13 @@ pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
writeln!(output, " }} else {{").unwrap(); writeln!(output, " }} else {{").unwrap();
writeln!( writeln!(
output, output,
" self.base.get_json(&path).map(FormatResponse::Json)" " self.base.{}(&path).map(FormatResponse::Json)",
fetch_method
) )
.unwrap(); .unwrap();
writeln!(output, " }}").unwrap(); writeln!(output, " }}").unwrap();
} else { } else {
writeln!(output, " self.base.get_json(&path)").unwrap(); writeln!(output, " self.base.{}(&path)", fetch_method).unwrap();
} }
} }
@@ -198,26 +213,35 @@ fn build_method_params(endpoint: &Endpoint) -> String {
let mut params = Vec::new(); let mut params = Vec::new();
for param in &endpoint.path_params { for param in &endpoint.path_params {
let rust_type = param_type_to_rust(&param.param_type); let rust_type = param_type_to_rust(&param.param_type);
params.push(format!(", {}: {}", param.name, rust_type)); params.push(format!(", {}: {}", sanitize_ident(&param.name), rust_type));
} }
for param in &endpoint.query_params { for param in &endpoint.query_params {
let rust_type = param_type_to_rust(&param.param_type); let rust_type = param_type_to_rust(&param.param_type);
let name = sanitize_ident(&param.name);
if param.required { if param.required {
params.push(format!(", {}: {}", param.name, rust_type)); params.push(format!(", {}: {}", name, rust_type));
} else { } else {
params.push(format!(", {}: Option<{}>", param.name, rust_type)); params.push(format!(", {}: Option<{}>", name, rust_type));
} }
} }
params.join("") params.join("")
} }
/// Strip characters invalid in Rust identifiers (e.g. `[]` from `txId[]`).
fn sanitize_ident(name: &str) -> String {
name.replace(['[', ']'], "")
}
/// Convert parameter type to Rust type for function signatures. /// Convert parameter type to Rust type for function signatures.
fn param_type_to_rust(param_type: &str) -> String { fn param_type_to_rust(param_type: &str) -> String {
if let Some(inner) = param_type.strip_suffix("[]") {
return format!("&[{}]", param_type_to_rust(inner));
}
match param_type { match param_type {
"string" | "*" => "&str".to_string(), "string" | "*" => "&str".to_string(),
"integer" | "number" => "i64".to_string(), "integer" | "number" => "i64".to_string(),
"boolean" => "bool".to_string(), "boolean" => "bool".to_string(),
other => other.to_string(), // Domain types like Index, SeriesName, Format other => other.to_string(),
} }
} }
+5
View File
@@ -43,6 +43,11 @@ impl Endpoint {
self.method == "GET" && !self.deprecated self.method == "GET" && !self.deprecated
} }
/// Returns true if this endpoint returns JSON (has a response_type extracted from application/json).
pub fn returns_json(&self) -> bool {
self.response_type.is_some()
}
/// Returns the operation ID or generates one from the path. /// Returns the operation ID or generates one from the path.
/// The returned string uses the raw case from the spec (typically camelCase). /// The returned string uses the raw case from the spec (typically camelCase).
pub fn operation_name(&self) -> String { pub fn operation_name(&self) -> String {
+3
View File
@@ -74,6 +74,9 @@ pub fn escape_python_keyword(name: &str) -> String {
"try", "while", "with", "yield", "try", "while", "with", "yield",
]; ];
// Strip characters invalid in identifiers (e.g. `[]` from `txId[]`)
let name = name.replace(['[', ']'], "");
// Prefix with underscore if starts with digit // Prefix with underscore if starts with digit
let name = if name.starts_with(|c: char| c.is_ascii_digit()) { let name = if name.starts_with(|c: char| c.is_ascii_digit()) {
format!("_{}", name) format!("_{}", name)
+1 -2
View File
@@ -13,7 +13,6 @@ brk_alloc = { workspace = true }
brk_computer = { workspace = true } brk_computer = { workspace = true }
brk_error = { workspace = true, features = ["tokio", "vecdb"] } brk_error = { workspace = true, features = ["tokio", "vecdb"] }
brk_indexer = { workspace = true } brk_indexer = { workspace = true }
brk_iterator = { workspace = true }
brk_logger = { workspace = true } brk_logger = { workspace = true }
brk_mempool = { workspace = true } brk_mempool = { workspace = true }
brk_query = { workspace = true } brk_query = { workspace = true }
@@ -26,7 +25,7 @@ owo-colors = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
toml = "1.0.7" toml = "1.1.2"
vecdb = { workspace = true } vecdb = { workspace = true }
[[bin]] [[bin]]
+2 -5
View File
@@ -1,11 +1,8 @@
# BRK CLI # BRK CLI
Command-line interface for running a Bitcoin Research Kit instance. Run your own Bitcoin Research Kit instance. One binary, one command. Full sync in ~4-7h depending on hardware. ~44% disk overhead vs 250% for mempool/electrs.
## Demo [bitview.space](https://bitview.space) is the official free hosted instance.
- [bitview.space](https://bitview.space) - web interface
- [bitview.space/api](https://bitview.space/api) - API docs
## Requirements ## Requirements
+4 -7
View File
@@ -10,7 +10,6 @@ use brk_alloc::Mimalloc;
use brk_computer::Computer; use brk_computer::Computer;
use brk_error::Result; use brk_error::Result;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_iterator::Blocks;
use brk_mempool::Mempool; use brk_mempool::Mempool;
use brk_query::AsyncQuery; use brk_query::AsyncQuery;
use brk_reader::Reader; use brk_reader::Reader;
@@ -37,8 +36,6 @@ pub fn main() -> anyhow::Result<()> {
let reader = Reader::new(config.blocksdir(), &client); let reader = Reader::new(config.blocksdir(), &client);
let blocks = Blocks::new(&client, &reader);
let mut indexer = Indexer::forced_import(&config.brkdir())?; let mut indexer = Indexer::forced_import(&config.brkdir())?;
#[cfg(not(debug_assertions))] #[cfg(not(debug_assertions))]
@@ -52,7 +49,7 @@ pub fn main() -> anyhow::Result<()> {
info!("Indexing {blocks_behind} blocks before starting server..."); info!("Indexing {blocks_behind} blocks before starting server...");
info!("---"); info!("---");
sleep(Duration::from_secs(10)); sleep(Duration::from_secs(10));
indexer.index(&blocks, &client, &exit)?; indexer.index(&reader, &client, &exit)?;
drop(indexer); drop(indexer);
Mimalloc::collect(); Mimalloc::collect();
indexer = Indexer::forced_import(&config.brkdir())?; indexer = Indexer::forced_import(&config.brkdir())?;
@@ -102,14 +99,14 @@ pub fn main() -> anyhow::Result<()> {
let total_start = Instant::now(); let total_start = Instant::now();
let starting_indexes = if cfg!(debug_assertions) { let starting_indexes = if cfg!(debug_assertions) {
indexer.checked_index(&blocks, &client, &exit)? indexer.checked_index(&reader, &client, &exit)?
} else { } else {
indexer.index(&blocks, &client, &exit)? indexer.index(&reader, &client, &exit)?
}; };
Mimalloc::collect(); Mimalloc::collect();
computer.compute(&indexer, starting_indexes, &reader, &exit)?; computer.compute(&indexer, starting_indexes, &exit)?;
info!("Total time: {:?}", total_start.elapsed()); info!("Total time: {:?}", total_start.elapsed());
info!("Waiting for new blocks..."); info!("Waiting for new blocks...");
+1675 -809
View File
File diff suppressed because it is too large Load Diff
+3
View File
@@ -14,3 +14,6 @@ brk_traversable = { workspace = true }
vecdb = { workspace = true } vecdb = { workspace = true }
rayon = { workspace = true } rayon = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
[package.metadata.cargo-machete]
ignored = ["vecdb"]
+3 -2
View File
@@ -75,7 +75,8 @@ impl<T> AddrGroups<T> {
} }
pub fn iter_overlapping_mut(&mut self) -> impl Iterator<Item = &mut T> { pub fn iter_overlapping_mut(&mut self) -> impl Iterator<Item = &mut T> {
self.under_amount.iter_mut().chain(self.over_amount.iter_mut()) self.under_amount
.iter_mut()
.chain(self.over_amount.iter_mut())
} }
} }
+23 -1
View File
@@ -201,7 +201,29 @@ impl<T> AgeRange<T> {
} }
pub fn from_array(arr: [T; 21]) -> Self { pub fn from_array(arr: [T; 21]) -> Self {
let [a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20] = arr; let [
a0,
a1,
a2,
a3,
a4,
a5,
a6,
a7,
a8,
a9,
a10,
a11,
a12,
a13,
a14,
a15,
a16,
a17,
a18,
a19,
a20,
] = arr;
Self { Self {
under_1h: a0, under_1h: a0,
_1h_to_1d: a1, _1h_to_1d: a1,
+5 -1
View File
@@ -84,7 +84,11 @@ pub const AMOUNT_RANGE_NAMES: AmountRange<CohortName> = AmountRange {
_10sats_to_100sats: CohortName::new("10sats_to_100sats", "10-100 sats", "10-100 Sats"), _10sats_to_100sats: CohortName::new("10sats_to_100sats", "10-100 sats", "10-100 Sats"),
_100sats_to_1k_sats: CohortName::new("100sats_to_1k_sats", "100-1k sats", "100-1K Sats"), _100sats_to_1k_sats: CohortName::new("100sats_to_1k_sats", "100-1k sats", "100-1K Sats"),
_1k_sats_to_10k_sats: CohortName::new("1k_sats_to_10k_sats", "1k-10k sats", "1K-10K Sats"), _1k_sats_to_10k_sats: CohortName::new("1k_sats_to_10k_sats", "1k-10k sats", "1K-10K Sats"),
_10k_sats_to_100k_sats: CohortName::new("10k_sats_to_100k_sats", "10k-100k sats", "10K-100K Sats"), _10k_sats_to_100k_sats: CohortName::new(
"10k_sats_to_100k_sats",
"10k-100k sats",
"10K-100K Sats",
),
_100k_sats_to_1m_sats: CohortName::new("100k_sats_to_1m_sats", "100k-1M sats", "100K-1M Sats"), _100k_sats_to_1m_sats: CohortName::new("100k_sats_to_1m_sats", "100k-1M sats", "100K-1M Sats"),
_1m_sats_to_10m_sats: CohortName::new("1m_sats_to_10m_sats", "1M-10M sats", "1M-10M Sats"), _1m_sats_to_10m_sats: CohortName::new("1m_sats_to_10m_sats", "1M-10M sats", "1M-10M Sats"),
_10m_sats_to_1btc: CohortName::new("10m_sats_to_1btc", "0.1-1 BTC", "0.1-1 BTC"), _10m_sats_to_1btc: CohortName::new("10m_sats_to_1btc", "0.1-1 BTC", "0.1-1 BTC"),
+63 -2
View File
@@ -1,16 +1,34 @@
use std::ops::{Add, AddAssign}; use std::ops::{Add, AddAssign};
use brk_traversable::Traversable;
use brk_types::OutputType; use brk_types::OutputType;
use rayon::prelude::*;
use super::{SpendableType, UnspendableType}; use super::{Filter, SpendableType, UnspendableType};
#[derive(Default, Clone, Debug)] pub const OP_RETURN: &str = "op_return";
#[derive(Default, Clone, Debug, Traversable)]
pub struct ByType<T> { pub struct ByType<T> {
#[traversable(flatten)]
pub spendable: SpendableType<T>, pub spendable: SpendableType<T>,
#[traversable(flatten)]
pub unspendable: UnspendableType<T>, pub unspendable: UnspendableType<T>,
} }
impl<T> ByType<T> { impl<T> ByType<T> {
pub fn try_new<F, E>(mut create: F) -> Result<Self, E>
where
F: FnMut(Filter, &'static str) -> Result<T, E>,
{
Ok(Self {
spendable: SpendableType::try_new(&mut create)?,
unspendable: UnspendableType {
op_return: create(Filter::Type(OutputType::OpReturn), OP_RETURN)?,
},
})
}
pub fn get(&self, output_type: OutputType) -> &T { pub fn get(&self, output_type: OutputType) -> &T {
match output_type { match output_type {
OutputType::P2PK65 => &self.spendable.p2pk65, OutputType::P2PK65 => &self.spendable.p2pk65,
@@ -44,6 +62,49 @@ impl<T> ByType<T> {
OutputType::OpReturn => &mut self.unspendable.op_return, OutputType::OpReturn => &mut self.unspendable.op_return,
} }
} }
pub fn iter(&self) -> impl Iterator<Item = &T> {
self.spendable
.iter()
.chain(std::iter::once(&self.unspendable.op_return))
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
self.spendable
.iter_mut()
.chain(std::iter::once(&mut self.unspendable.op_return))
}
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
where
T: Send + Sync,
{
let Self {
spendable,
unspendable,
} = self;
spendable
.par_iter_mut()
.chain([&mut unspendable.op_return].into_par_iter())
}
pub fn iter_typed(&self) -> impl Iterator<Item = (OutputType, &T)> {
self.spendable
.iter_typed()
.chain(std::iter::once((
OutputType::OpReturn,
&self.unspendable.op_return,
)))
}
pub fn iter_typed_mut(&mut self) -> impl Iterator<Item = (OutputType, &mut T)> {
self.spendable
.iter_typed_mut()
.chain(std::iter::once((
OutputType::OpReturn,
&mut self.unspendable.op_return,
)))
}
} }
impl<T> Add for ByType<T> impl<T> Add for ByType<T>
+2 -2
View File
@@ -1,8 +1,8 @@
#![doc = include_str!("../README.md")] #![doc = include_str!("../README.md")]
mod addr; mod addr;
mod amount_filter;
mod age_range; mod age_range;
mod amount_filter;
mod amount_range; mod amount_range;
mod by_addr_type; mod by_addr_type;
mod by_any_addr; mod by_any_addr;
@@ -30,8 +30,8 @@ mod utxo;
pub use brk_types::{Age, Term}; pub use brk_types::{Age, Term};
pub use addr::*; pub use addr::*;
pub use amount_filter::*;
pub use age_range::*; pub use age_range::*;
pub use amount_filter::*;
pub use amount_range::*; pub use amount_range::*;
pub use by_addr_type::*; pub use by_addr_type::*;
pub use by_any_addr::*; pub use by_any_addr::*;
+1 -1
View File
@@ -153,6 +153,6 @@ impl<T> Loss<T> {
.into_iter() .into_iter()
.rev() .rev()
.enumerate() .enumerate()
.map(move |(n, threshold)| (threshold, &ranges[len - 1 - n..])) .map(move |(n, threshold)| (threshold, &ranges[len - 2 - n..]))
} }
} }
+18 -6
View File
@@ -43,19 +43,31 @@ pub const OVER_AMOUNT_NAMES: OverAmount<CohortName> = OverAmount {
pub const OVER_AMOUNT_FILTERS: OverAmount<Filter> = OverAmount { pub const OVER_AMOUNT_FILTERS: OverAmount<Filter> = OverAmount {
_1sat: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._1sat)), _1sat: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._1sat)),
_10sats: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._10sats)), _10sats: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._10sats)),
_100sats: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._100sats)), _100sats: Filter::Amount(AmountFilter::GreaterOrEqual(
_1k_sats: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._1k_sats)), OVER_AMOUNT_THRESHOLDS._100sats,
_10k_sats: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._10k_sats)), )),
_1k_sats: Filter::Amount(AmountFilter::GreaterOrEqual(
OVER_AMOUNT_THRESHOLDS._1k_sats,
)),
_10k_sats: Filter::Amount(AmountFilter::GreaterOrEqual(
OVER_AMOUNT_THRESHOLDS._10k_sats,
)),
_100k_sats: Filter::Amount(AmountFilter::GreaterOrEqual( _100k_sats: Filter::Amount(AmountFilter::GreaterOrEqual(
OVER_AMOUNT_THRESHOLDS._100k_sats, OVER_AMOUNT_THRESHOLDS._100k_sats,
)), )),
_1m_sats: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._1m_sats)), _1m_sats: Filter::Amount(AmountFilter::GreaterOrEqual(
_10m_sats: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._10m_sats)), OVER_AMOUNT_THRESHOLDS._1m_sats,
)),
_10m_sats: Filter::Amount(AmountFilter::GreaterOrEqual(
OVER_AMOUNT_THRESHOLDS._10m_sats,
)),
_1btc: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._1btc)), _1btc: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._1btc)),
_10btc: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._10btc)), _10btc: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._10btc)),
_100btc: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._100btc)), _100btc: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._100btc)),
_1k_btc: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._1k_btc)), _1k_btc: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._1k_btc)),
_10k_btc: Filter::Amount(AmountFilter::GreaterOrEqual(OVER_AMOUNT_THRESHOLDS._10k_btc)), _10k_btc: Filter::Amount(AmountFilter::GreaterOrEqual(
OVER_AMOUNT_THRESHOLDS._10k_btc,
)),
}; };
#[derive(Default, Clone, Traversable, Serialize)] #[derive(Default, Clone, Traversable, Serialize)]
+21 -5
View File
@@ -16,10 +16,26 @@ pub const PROFIT_NAMES: Profit<CohortName> = Profit {
_70pct: CohortName::new("utxos_over_70pct_in_profit", ">=70%", "Over 70% in Profit"), _70pct: CohortName::new("utxos_over_70pct_in_profit", ">=70%", "Over 70% in Profit"),
_80pct: CohortName::new("utxos_over_80pct_in_profit", ">=80%", "Over 80% in Profit"), _80pct: CohortName::new("utxos_over_80pct_in_profit", ">=80%", "Over 80% in Profit"),
_90pct: CohortName::new("utxos_over_90pct_in_profit", ">=90%", "Over 90% in Profit"), _90pct: CohortName::new("utxos_over_90pct_in_profit", ">=90%", "Over 90% in Profit"),
_100pct: CohortName::new("utxos_over_100pct_in_profit", ">=100%", "Over 100% in Profit"), _100pct: CohortName::new(
_200pct: CohortName::new("utxos_over_200pct_in_profit", ">=200%", "Over 200% in Profit"), "utxos_over_100pct_in_profit",
_300pct: CohortName::new("utxos_over_300pct_in_profit", ">=300%", "Over 300% in Profit"), ">=100%",
_500pct: CohortName::new("utxos_over_500pct_in_profit", ">=500%", "Over 500% in Profit"), "Over 100% in Profit",
),
_200pct: CohortName::new(
"utxos_over_200pct_in_profit",
">=200%",
"Over 200% in Profit",
),
_300pct: CohortName::new(
"utxos_over_300pct_in_profit",
">=300%",
"Over 300% in Profit",
),
_500pct: CohortName::new(
"utxos_over_500pct_in_profit",
">=500%",
"Over 500% in Profit",
),
}; };
/// Number of profit thresholds. /// Number of profit thresholds.
@@ -192,6 +208,6 @@ impl<T> Profit<T> {
.into_iter() .into_iter()
.rev() .rev()
.enumerate() .enumerate()
.map(move |(n, threshold)| (threshold, &ranges[..n + 1])) .map(move |(n, threshold)| (threshold, &ranges[..n + 2]))
} }
} }
+125 -25
View File
@@ -83,31 +83,131 @@ pub fn compute_profitability_boundaries(spot: Cents) -> [Cents; PROFITABILITY_BO
/// Profitability range names (25 ranges, from most profitable to most in loss) /// Profitability range names (25 ranges, from most profitable to most in loss)
pub const PROFITABILITY_RANGE_NAMES: ProfitabilityRange<CohortName> = ProfitabilityRange { pub const PROFITABILITY_RANGE_NAMES: ProfitabilityRange<CohortName> = ProfitabilityRange {
over_1000pct_in_profit: CohortName::new("utxos_over_1000pct_in_profit", "+>1000%", "Over 1000% in Profit"), over_1000pct_in_profit: CohortName::new(
_500pct_to_1000pct_in_profit: CohortName::new("utxos_500pct_to_1000pct_in_profit", "+500-1000%", "500-1000% in Profit"), "utxos_over_1000pct_in_profit",
_300pct_to_500pct_in_profit: CohortName::new("utxos_300pct_to_500pct_in_profit", "+300-500%", "300-500% in Profit"), "+>1000%",
_200pct_to_300pct_in_profit: CohortName::new("utxos_200pct_to_300pct_in_profit", "+200-300%", "200-300% in Profit"), "Over 1000% in Profit",
_100pct_to_200pct_in_profit: CohortName::new("utxos_100pct_to_200pct_in_profit", "+100-200%", "100-200% in Profit"), ),
_90pct_to_100pct_in_profit: CohortName::new("utxos_90pct_to_100pct_in_profit", "+90-100%", "90-100% in Profit"), _500pct_to_1000pct_in_profit: CohortName::new(
_80pct_to_90pct_in_profit: CohortName::new("utxos_80pct_to_90pct_in_profit", "+80-90%", "80-90% in Profit"), "utxos_500pct_to_1000pct_in_profit",
_70pct_to_80pct_in_profit: CohortName::new("utxos_70pct_to_80pct_in_profit", "+70-80%", "70-80% in Profit"), "+500-1000%",
_60pct_to_70pct_in_profit: CohortName::new("utxos_60pct_to_70pct_in_profit", "+60-70%", "60-70% in Profit"), "500-1000% in Profit",
_50pct_to_60pct_in_profit: CohortName::new("utxos_50pct_to_60pct_in_profit", "+50-60%", "50-60% in Profit"), ),
_40pct_to_50pct_in_profit: CohortName::new("utxos_40pct_to_50pct_in_profit", "+40-50%", "40-50% in Profit"), _300pct_to_500pct_in_profit: CohortName::new(
_30pct_to_40pct_in_profit: CohortName::new("utxos_30pct_to_40pct_in_profit", "+30-40%", "30-40% in Profit"), "utxos_300pct_to_500pct_in_profit",
_20pct_to_30pct_in_profit: CohortName::new("utxos_20pct_to_30pct_in_profit", "+20-30%", "20-30% in Profit"), "+300-500%",
_10pct_to_20pct_in_profit: CohortName::new("utxos_10pct_to_20pct_in_profit", "+10-20%", "10-20% in Profit"), "300-500% in Profit",
_0pct_to_10pct_in_profit: CohortName::new("utxos_0pct_to_10pct_in_profit", "+0-10%", "0-10% in Profit"), ),
_0pct_to_10pct_in_loss: CohortName::new("utxos_0pct_to_10pct_in_loss", "-0-10%", "0-10% in Loss"), _200pct_to_300pct_in_profit: CohortName::new(
_10pct_to_20pct_in_loss: CohortName::new("utxos_10pct_to_20pct_in_loss", "-10-20%", "10-20% in Loss"), "utxos_200pct_to_300pct_in_profit",
_20pct_to_30pct_in_loss: CohortName::new("utxos_20pct_to_30pct_in_loss", "-20-30%", "20-30% in Loss"), "+200-300%",
_30pct_to_40pct_in_loss: CohortName::new("utxos_30pct_to_40pct_in_loss", "-30-40%", "30-40% in Loss"), "200-300% in Profit",
_40pct_to_50pct_in_loss: CohortName::new("utxos_40pct_to_50pct_in_loss", "-40-50%", "40-50% in Loss"), ),
_50pct_to_60pct_in_loss: CohortName::new("utxos_50pct_to_60pct_in_loss", "-50-60%", "50-60% in Loss"), _100pct_to_200pct_in_profit: CohortName::new(
_60pct_to_70pct_in_loss: CohortName::new("utxos_60pct_to_70pct_in_loss", "-60-70%", "60-70% in Loss"), "utxos_100pct_to_200pct_in_profit",
_70pct_to_80pct_in_loss: CohortName::new("utxos_70pct_to_80pct_in_loss", "-70-80%", "70-80% in Loss"), "+100-200%",
_80pct_to_90pct_in_loss: CohortName::new("utxos_80pct_to_90pct_in_loss", "-80-90%", "80-90% in Loss"), "100-200% in Profit",
_90pct_to_100pct_in_loss: CohortName::new("utxos_90pct_to_100pct_in_loss", "-90-100%", "90-100% in Loss"), ),
_90pct_to_100pct_in_profit: CohortName::new(
"utxos_90pct_to_100pct_in_profit",
"+90-100%",
"90-100% in Profit",
),
_80pct_to_90pct_in_profit: CohortName::new(
"utxos_80pct_to_90pct_in_profit",
"+80-90%",
"80-90% in Profit",
),
_70pct_to_80pct_in_profit: CohortName::new(
"utxos_70pct_to_80pct_in_profit",
"+70-80%",
"70-80% in Profit",
),
_60pct_to_70pct_in_profit: CohortName::new(
"utxos_60pct_to_70pct_in_profit",
"+60-70%",
"60-70% in Profit",
),
_50pct_to_60pct_in_profit: CohortName::new(
"utxos_50pct_to_60pct_in_profit",
"+50-60%",
"50-60% in Profit",
),
_40pct_to_50pct_in_profit: CohortName::new(
"utxos_40pct_to_50pct_in_profit",
"+40-50%",
"40-50% in Profit",
),
_30pct_to_40pct_in_profit: CohortName::new(
"utxos_30pct_to_40pct_in_profit",
"+30-40%",
"30-40% in Profit",
),
_20pct_to_30pct_in_profit: CohortName::new(
"utxos_20pct_to_30pct_in_profit",
"+20-30%",
"20-30% in Profit",
),
_10pct_to_20pct_in_profit: CohortName::new(
"utxos_10pct_to_20pct_in_profit",
"+10-20%",
"10-20% in Profit",
),
_0pct_to_10pct_in_profit: CohortName::new(
"utxos_0pct_to_10pct_in_profit",
"+0-10%",
"0-10% in Profit",
),
_0pct_to_10pct_in_loss: CohortName::new(
"utxos_0pct_to_10pct_in_loss",
"-0-10%",
"0-10% in Loss",
),
_10pct_to_20pct_in_loss: CohortName::new(
"utxos_10pct_to_20pct_in_loss",
"-10-20%",
"10-20% in Loss",
),
_20pct_to_30pct_in_loss: CohortName::new(
"utxos_20pct_to_30pct_in_loss",
"-20-30%",
"20-30% in Loss",
),
_30pct_to_40pct_in_loss: CohortName::new(
"utxos_30pct_to_40pct_in_loss",
"-30-40%",
"30-40% in Loss",
),
_40pct_to_50pct_in_loss: CohortName::new(
"utxos_40pct_to_50pct_in_loss",
"-40-50%",
"40-50% in Loss",
),
_50pct_to_60pct_in_loss: CohortName::new(
"utxos_50pct_to_60pct_in_loss",
"-50-60%",
"50-60% in Loss",
),
_60pct_to_70pct_in_loss: CohortName::new(
"utxos_60pct_to_70pct_in_loss",
"-60-70%",
"60-70% in Loss",
),
_70pct_to_80pct_in_loss: CohortName::new(
"utxos_70pct_to_80pct_in_loss",
"-70-80%",
"70-80% in Loss",
),
_80pct_to_90pct_in_loss: CohortName::new(
"utxos_80pct_to_90pct_in_loss",
"-80-90%",
"80-90% in Loss",
),
_90pct_to_100pct_in_loss: CohortName::new(
"utxos_90pct_to_100pct_in_loss",
"-90-100%",
"90-100% in Loss",
),
}; };
impl ProfitabilityRange<CohortName> { impl ProfitabilityRange<CohortName> {
+17
View File
@@ -116,6 +116,23 @@ impl<T> SpendableType<T> {
}) })
} }
pub fn get(&self, output_type: OutputType) -> &T {
match output_type {
OutputType::P2PK65 => &self.p2pk65,
OutputType::P2PK33 => &self.p2pk33,
OutputType::P2PKH => &self.p2pkh,
OutputType::P2MS => &self.p2ms,
OutputType::P2SH => &self.p2sh,
OutputType::P2WPKH => &self.p2wpkh,
OutputType::P2WSH => &self.p2wsh,
OutputType::P2TR => &self.p2tr,
OutputType::P2A => &self.p2a,
OutputType::Unknown => &self.unknown,
OutputType::Empty => &self.empty,
_ => unreachable!(),
}
}
pub fn get_mut(&mut self, output_type: OutputType) -> &mut T { pub fn get_mut(&mut self, output_type: OutputType) -> &mut T {
match output_type { match output_type {
OutputType::P2PK65 => &mut self.p2pk65, OutputType::P2PK65 => &mut self.p2pk65,
+2 -2
View File
@@ -2,8 +2,8 @@ use brk_traversable::Traversable;
use rayon::prelude::*; use rayon::prelude::*;
use crate::{ use crate::{
AgeRange, AmountRange, ByEpoch, OverAmount, UnderAmount, UnderAge, OverAge, AgeRange, AmountRange, ByEpoch, ByTerm, Class, Filter, OverAge, OverAmount, SpendableType,
Class, SpendableType, ByTerm, Filter, UnderAge, UnderAmount,
}; };
#[derive(Default, Clone, Traversable)] #[derive(Default, Clone, Traversable)]
+1 -3
View File
@@ -14,11 +14,8 @@ brk_error = { workspace = true, features = ["vecdb"] }
brk_cohort = { workspace = true } brk_cohort = { workspace = true }
brk_indexer = { workspace = true } brk_indexer = { workspace = true }
brk_oracle = { workspace = true } brk_oracle = { workspace = true }
brk_iterator = { workspace = true }
brk_logger = { workspace = true } brk_logger = { workspace = true }
brk_reader = { workspace = true }
brk_rpc = { workspace = true, features = ["corepc"] } brk_rpc = { workspace = true, features = ["corepc"] }
brk_store = { workspace = true }
brk_traversable = { workspace = true } brk_traversable = { workspace = true }
brk_types = { workspace = true } brk_types = { workspace = true }
derive_more = { workspace = true } derive_more = { workspace = true }
@@ -33,6 +30,7 @@ smallvec = { workspace = true }
vecdb = { workspace = true } vecdb = { workspace = true }
[dev-dependencies] [dev-dependencies]
brk_reader = { workspace = true }
brk_alloc = { workspace = true } brk_alloc = { workspace = true }
brk_bencher = { workspace = true } brk_bencher = { workspace = true }
color-eyre = { workspace = true } color-eyre = { workspace = true }
+3 -6
View File
@@ -8,7 +8,6 @@ use std::{
use brk_alloc::Mimalloc; use brk_alloc::Mimalloc;
use brk_computer::Computer; use brk_computer::Computer;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_iterator::Blocks;
use brk_reader::Reader; use brk_reader::Reader;
use brk_rpc::{Auth, Client}; use brk_rpc::{Auth, Client};
use vecdb::Exit; use vecdb::Exit;
@@ -31,8 +30,6 @@ pub fn main() -> color_eyre::Result<()> {
let reader = Reader::new(bitcoin_dir.join("blocks"), &client); let reader = Reader::new(bitcoin_dir.join("blocks"), &client);
let blocks = Blocks::new(&client, &reader);
let mut indexer = Indexer::forced_import(&outputs_dir)?; let mut indexer = Indexer::forced_import(&outputs_dir)?;
let exit = Exit::new(); let exit = Exit::new();
@@ -42,7 +39,7 @@ pub fn main() -> color_eyre::Result<()> {
let chain_height = client.get_last_height()?; let chain_height = client.get_last_height()?;
let indexed_height = indexer.vecs.starting_height(); let indexed_height = indexer.vecs.starting_height();
if u32::from(chain_height).saturating_sub(u32::from(indexed_height)) > 1000 { if u32::from(chain_height).saturating_sub(u32::from(indexed_height)) > 1000 {
indexer.checked_index(&blocks, &client, &exit)?; indexer.checked_index(&reader, &client, &exit)?;
drop(indexer); drop(indexer);
Mimalloc::collect(); Mimalloc::collect();
indexer = Indexer::forced_import(&outputs_dir)?; indexer = Indexer::forced_import(&outputs_dir)?;
@@ -52,11 +49,11 @@ pub fn main() -> color_eyre::Result<()> {
loop { loop {
let i = Instant::now(); let i = Instant::now();
let starting_indexes = indexer.checked_index(&blocks, &client, &exit)?; let starting_indexes = indexer.checked_index(&reader, &client, &exit)?;
Mimalloc::collect(); Mimalloc::collect();
computer.compute(&indexer, starting_indexes, &reader, &exit)?; computer.compute(&indexer, starting_indexes, &exit)?;
dbg!(i.elapsed()); dbg!(i.elapsed());
sleep(Duration::from_secs(10)); sleep(Duration::from_secs(10));
} }
@@ -5,7 +5,6 @@ use brk_bencher::Bencher;
use brk_computer::Computer; use brk_computer::Computer;
use brk_error::Result; use brk_error::Result;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_iterator::Blocks;
use brk_reader::Reader; use brk_reader::Reader;
use brk_rpc::{Auth, Client}; use brk_rpc::{Auth, Client};
use tracing::{debug, info}; use tracing::{debug, info};
@@ -28,8 +27,6 @@ pub fn main() -> Result<()> {
let reader = Reader::new(bitcoin_dir.join("blocks"), &client); let reader = Reader::new(bitcoin_dir.join("blocks"), &client);
let blocks = Blocks::new(&client, &reader);
let mut indexer = Indexer::forced_import(&outputs_dir)?; let mut indexer = Indexer::forced_import(&outputs_dir)?;
let mut computer = Computer::forced_import(&outputs_benches_dir, &indexer)?; let mut computer = Computer::forced_import(&outputs_benches_dir, &indexer)?;
@@ -47,13 +44,13 @@ pub fn main() -> Result<()> {
}); });
let i = Instant::now(); let i = Instant::now();
let starting_indexes = indexer.index(&blocks, &client, &exit)?; let starting_indexes = indexer.index(&reader, &client, &exit)?;
info!("Done in {:?}", i.elapsed()); info!("Done in {:?}", i.elapsed());
Mimalloc::collect(); Mimalloc::collect();
let i = Instant::now(); let i = Instant::now();
computer.compute(&indexer, starting_indexes, &reader, &exit)?; computer.compute(&indexer, starting_indexes, &exit)?;
info!("Done in {:?}", i.elapsed()); info!("Done in {:?}", i.elapsed());
// We want to benchmark the drop too // We want to benchmark the drop too
+3 -6
View File
@@ -9,7 +9,6 @@ use brk_alloc::Mimalloc;
use brk_bencher::Bencher; use brk_bencher::Bencher;
use brk_computer::Computer; use brk_computer::Computer;
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_iterator::Blocks;
use brk_reader::Reader; use brk_reader::Reader;
use brk_rpc::{Auth, Client}; use brk_rpc::{Auth, Client};
use tracing::{debug, info}; use tracing::{debug, info};
@@ -45,15 +44,13 @@ pub fn main() -> color_eyre::Result<()> {
let reader = Reader::new(bitcoin_dir.join("blocks"), &client); let reader = Reader::new(bitcoin_dir.join("blocks"), &client);
let blocks = Blocks::new(&client, &reader);
let mut indexer = Indexer::forced_import(&outputs_dir)?; let mut indexer = Indexer::forced_import(&outputs_dir)?;
// Pre-run indexer if too far behind, then drop and reimport to reduce memory // Pre-run indexer if too far behind, then drop and reimport to reduce memory
let chain_height = client.get_last_height()?; let chain_height = client.get_last_height()?;
let indexed_height = indexer.vecs.starting_height(); let indexed_height = indexer.vecs.starting_height();
if chain_height.saturating_sub(*indexed_height) > 1000 { if chain_height.saturating_sub(*indexed_height) > 1000 {
indexer.index(&blocks, &client, &exit)?; indexer.index(&reader, &client, &exit)?;
drop(indexer); drop(indexer);
Mimalloc::collect(); Mimalloc::collect();
indexer = Indexer::forced_import(&outputs_dir)?; indexer = Indexer::forced_import(&outputs_dir)?;
@@ -63,13 +60,13 @@ pub fn main() -> color_eyre::Result<()> {
loop { loop {
let i = Instant::now(); let i = Instant::now();
let starting_indexes = indexer.index(&blocks, &client, &exit)?; let starting_indexes = indexer.index(&reader, &client, &exit)?;
info!("Done in {:?}", i.elapsed()); info!("Done in {:?}", i.elapsed());
Mimalloc::collect(); Mimalloc::collect();
let i = Instant::now(); let i = Instant::now();
computer.compute(&indexer, starting_indexes, &reader, &exit)?; computer.compute(&indexer, starting_indexes, &exit)?;
info!("Done in {:?}", i.elapsed()); info!("Done in {:?}", i.elapsed());
sleep(Duration::from_secs(60)); sleep(Duration::from_secs(60));
+10 -10
View File
@@ -17,12 +17,10 @@ impl Vecs {
starting_indexes: &Indexes, starting_indexes: &Indexes,
exit: &Exit, exit: &Exit,
) -> Result<()> { ) -> Result<()> {
// Sequential: time → lookback (dependency chain) self.db.sync_bg_tasks()?;
self.time
.timestamp // lookback depends on indexes.timestamp.monotonic
.compute(indexer, indexes, starting_indexes, exit)?; self.lookback.compute(indexes, starting_indexes, exit)?;
self.lookback
.compute(&self.time, starting_indexes, exit)?;
// Parallel: remaining sub-modules are independent of each other. // Parallel: remaining sub-modules are independent of each other.
// size depends on lookback (already computed above). // size depends on lookback (already computed above).
@@ -40,8 +38,7 @@ impl Vecs {
let r1 = s.spawn(|| count.compute(indexer, starting_indexes, exit)); let r1 = s.spawn(|| count.compute(indexer, starting_indexes, exit));
let r2 = s.spawn(|| interval.compute(indexer, starting_indexes, exit)); let r2 = s.spawn(|| interval.compute(indexer, starting_indexes, exit));
let r3 = s.spawn(|| weight.compute(indexer, starting_indexes, exit)); let r3 = s.spawn(|| weight.compute(indexer, starting_indexes, exit));
let r4 = let r4 = s.spawn(|| difficulty.compute(indexer, indexes, starting_indexes, exit));
s.spawn(|| difficulty.compute(indexer, indexes, starting_indexes, exit));
let r5 = s.spawn(|| halving.compute(indexes, starting_indexes, exit)); let r5 = s.spawn(|| halving.compute(indexes, starting_indexes, exit));
size.compute(indexer, &*lookback, starting_indexes, exit)?; size.compute(indexer, &*lookback, starting_indexes, exit)?;
r1.join().unwrap()?; r1.join().unwrap()?;
@@ -52,8 +49,11 @@ impl Vecs {
Ok(()) Ok(())
})?; })?;
let _lock = exit.lock(); let exit = exit.clone();
self.db.compact()?; self.db.run_bg(move |db| {
let _lock = exit.lock();
db.compact_deferred_default()
});
Ok(()) Ok(())
} }
} }
+23 -7
View File
@@ -6,8 +6,8 @@ use super::Vecs;
use crate::{ use crate::{
indexes, indexes,
internal::{ internal::{
BlockCountTarget24h, BlockCountTarget1w, BlockCountTarget1m, BlockCountTarget1y, BlockCountTarget1m, BlockCountTarget1w, BlockCountTarget1y, BlockCountTarget24h,
CachedWindowStarts, PerBlockCumulativeRolling, ConstantVecs, Windows, ConstantVecs, PerBlockCumulativeRolling, WindowStartVec, Windows,
}, },
}; };
@@ -16,14 +16,30 @@ impl Vecs {
db: &Database, db: &Database,
version: Version, version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts, cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> { ) -> Result<Self> {
Ok(Self { Ok(Self {
target: Windows { target: Windows {
_24h: ConstantVecs::new::<BlockCountTarget24h>("block_count_target_24h", version, indexes), _24h: ConstantVecs::new::<BlockCountTarget24h>(
_1w: ConstantVecs::new::<BlockCountTarget1w>("block_count_target_1w", version, indexes), "block_count_target_24h",
_1m: ConstantVecs::new::<BlockCountTarget1m>("block_count_target_1m", version, indexes), version,
_1y: ConstantVecs::new::<BlockCountTarget1y>("block_count_target_1y", version, indexes), indexes,
),
_1w: ConstantVecs::new::<BlockCountTarget1w>(
"block_count_target_1w",
version,
indexes,
),
_1m: ConstantVecs::new::<BlockCountTarget1m>(
"block_count_target_1m",
version,
indexes,
),
_1y: ConstantVecs::new::<BlockCountTarget1y>(
"block_count_target_1y",
version,
indexes,
),
}, },
total: PerBlockCumulativeRolling::forced_import( total: PerBlockCumulativeRolling::forced_import(
db, db,
+1 -1
View File
@@ -2,7 +2,7 @@ use brk_traversable::Traversable;
use brk_types::{StoredU32, StoredU64}; use brk_types::{StoredU32, StoredU64};
use vecdb::{Rw, StorageMode}; use vecdb::{Rw, StorageMode};
use crate::internal::{PerBlockCumulativeRolling, ConstantVecs, Windows}; use crate::internal::{ConstantVecs, PerBlockCumulativeRolling, Windows};
#[derive(Traversable)] #[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> { pub struct Vecs<M: StorageMode = Rw> {
@@ -30,7 +30,7 @@ impl Vecs {
self.blocks_to_retarget.height.compute_transform( self.blocks_to_retarget.height.compute_transform(
starting_indexes.height, starting_indexes.height,
&indexes.height.identity, &indexes.height.epoch,
|(h, ..)| (h, StoredU32::from(h.left_before_next_diff_adj())), |(h, ..)| (h, StoredU32::from(h.left_before_next_diff_adj())),
exit, exit,
)?; )?;
@@ -20,19 +20,15 @@ impl Vecs {
) -> Result<Self> { ) -> Result<Self> {
let v2 = Version::TWO; let v2 = Version::TWO;
let hashrate = LazyPerBlock::from_height_source::<DifficultyToHashF64>( let hashrate = LazyPerBlock::from_height_source::<DifficultyToHashF64, _>(
"difficulty_hashrate", "difficulty_hashrate",
version, version,
indexer.vecs.blocks.difficulty.read_only_boxed_clone(), indexer.vecs.blocks.difficulty.read_only_clone(),
indexes, indexes,
); );
let blocks_to_retarget = PerBlock::forced_import( let blocks_to_retarget =
db, PerBlock::forced_import(db, "blocks_to_retarget", version + v2, indexes)?;
"blocks_to_retarget",
version + v2,
indexes,
)?;
let days_to_retarget = LazyPerBlock::from_computed::<BlocksToDaysF32>( let days_to_retarget = LazyPerBlock::from_computed::<BlocksToDaysF32>(
"days_to_retarget", "days_to_retarget",
@@ -44,7 +40,7 @@ impl Vecs {
Ok(Self { Ok(Self {
value: Resolutions::forced_import( value: Resolutions::forced_import(
"difficulty", "difficulty",
indexer.vecs.blocks.difficulty.read_only_boxed_clone(), indexer.vecs.blocks.difficulty.read_only_clone(),
version, version,
indexes, indexes,
), ),
@@ -2,7 +2,7 @@ use brk_traversable::Traversable;
use brk_types::{BasisPointsSigned32, Epoch, StoredF32, StoredF64, StoredU32}; use brk_types::{BasisPointsSigned32, Epoch, StoredF32, StoredF64, StoredU32};
use vecdb::{Rw, StorageMode}; use vecdb::{Rw, StorageMode};
use crate::internal::{LazyPerBlock, PerBlock, Resolutions, PercentPerBlock}; use crate::internal::{LazyPerBlock, PerBlock, PercentPerBlock, Resolutions};
#[derive(Traversable)] #[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> { pub struct Vecs<M: StorageMode = Rw> {
pub value: Resolutions<StoredF64>, pub value: Resolutions<StoredF64>,
@@ -21,7 +21,7 @@ impl Vecs {
self.blocks_to_halving.height.compute_transform( self.blocks_to_halving.height.compute_transform(
starting_indexes.height, starting_indexes.height,
&indexes.height.identity, &indexes.height.halving,
|(h, ..)| (h, StoredU32::from(h.left_before_next_halving())), |(h, ..)| (h, StoredU32::from(h.left_before_next_halving())),
exit, exit,
)?; )?;
@@ -16,9 +16,8 @@ impl Vecs {
) -> Result<Self> { ) -> Result<Self> {
let v2 = Version::TWO; let v2 = Version::TWO;
let blocks_to_halving = PerBlock::forced_import( let blocks_to_halving =
db, "blocks_to_halving", version + v2, indexes, PerBlock::forced_import(db, "blocks_to_halving", version + v2, indexes)?;
)?;
let days_to_halving = LazyPerBlock::from_computed::<BlocksToDaysF32>( let days_to_halving = LazyPerBlock::from_computed::<BlocksToDaysF32>(
"days_to_halving", "days_to_halving",
+6 -9
View File
@@ -10,8 +10,7 @@ use crate::{
}; };
use super::{ use super::{
CountVecs, DifficultyVecs, HalvingVecs, IntervalVecs, LookbackVecs, SizeVecs, TimeVecs, Vecs, CountVecs, DifficultyVecs, HalvingVecs, IntervalVecs, LookbackVecs, SizeVecs, Vecs, WeightVecs,
WeightVecs,
}; };
impl Vecs { impl Vecs {
@@ -25,12 +24,11 @@ impl Vecs {
let version = parent_version; let version = parent_version;
let lookback = LookbackVecs::forced_import(&db, version)?; let lookback = LookbackVecs::forced_import(&db, version)?;
let cached_starts = &lookback.cached_window_starts; let cached_starts = lookback.cached_window_starts();
let count = CountVecs::forced_import(&db, version, indexes, cached_starts)?; let count = CountVecs::forced_import(&db, version, indexes, &cached_starts)?;
let interval = IntervalVecs::forced_import(&db, version, indexes, cached_starts)?; let interval = IntervalVecs::forced_import(&db, version, indexes, &cached_starts)?;
let size = SizeVecs::forced_import(&db, version, indexes, cached_starts)?; let size = SizeVecs::forced_import(&db, version, indexes, &cached_starts)?;
let weight = WeightVecs::forced_import(&db, version, indexes, cached_starts, &size)?; let weight = WeightVecs::forced_import(&db, version, indexes, &cached_starts, &size)?;
let time = TimeVecs::forced_import(&db, version, indexes)?;
let difficulty = DifficultyVecs::forced_import(&db, version, indexer, indexes)?; let difficulty = DifficultyVecs::forced_import(&db, version, indexer, indexes)?;
let halving = HalvingVecs::forced_import(&db, version, indexes)?; let halving = HalvingVecs::forced_import(&db, version, indexes)?;
@@ -41,7 +39,6 @@ impl Vecs {
interval, interval,
size, size,
weight, weight,
time,
difficulty, difficulty,
halving, halving,
}; };
@@ -13,27 +13,26 @@ impl Vecs {
exit: &Exit, exit: &Exit,
) -> Result<()> { ) -> Result<()> {
let mut prev_timestamp = None; let mut prev_timestamp = None;
self.0 self.0.compute(starting_indexes.height, exit, |vec| {
.compute(starting_indexes.height, exit, |vec| { vec.compute_transform(
vec.compute_transform( starting_indexes.height,
starting_indexes.height, &indexer.vecs.blocks.timestamp,
&indexer.vecs.blocks.timestamp, |(h, timestamp, ..)| {
|(h, timestamp, ..)| { let interval = if let Some(prev_h) = h.decremented() {
let interval = if let Some(prev_h) = h.decremented() { let prev = prev_timestamp.unwrap_or_else(|| {
let prev = prev_timestamp.unwrap_or_else(|| { indexer.vecs.blocks.timestamp.collect_one(prev_h).unwrap()
indexer.vecs.blocks.timestamp.collect_one(prev_h).unwrap() });
}); timestamp.checked_sub(prev).unwrap_or(Timestamp::ZERO)
timestamp.checked_sub(prev).unwrap_or(Timestamp::ZERO) } else {
} else { Timestamp::ZERO
Timestamp::ZERO };
}; prev_timestamp = Some(timestamp);
prev_timestamp = Some(timestamp); (h, interval)
(h, interval) },
}, exit,
exit, )?;
)?; Ok(())
Ok(()) })?;
})?;
Ok(()) Ok(())
} }
@@ -3,14 +3,17 @@ use brk_types::Version;
use vecdb::Database; use vecdb::Database;
use super::Vecs; use super::Vecs;
use crate::{indexes, internal::{CachedWindowStarts, PerBlockRollingAverage}}; use crate::{
indexes,
internal::{PerBlockRollingAverage, WindowStartVec, Windows},
};
impl Vecs { impl Vecs {
pub(crate) fn forced_import( pub(crate) fn forced_import(
db: &Database, db: &Database,
version: Version, version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts, cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> { ) -> Result<Self> {
let interval = PerBlockRollingAverage::forced_import( let interval = PerBlockRollingAverage::forced_import(
db, db,
@@ -8,5 +8,5 @@ use crate::internal::PerBlockRollingAverage;
#[derive(Deref, DerefMut, Traversable)] #[derive(Deref, DerefMut, Traversable)]
pub struct Vecs<M: StorageMode = Rw>( pub struct Vecs<M: StorageMode = Rw>(
#[traversable(flatten)] pub PerBlockRollingAverage<Timestamp, M>, #[traversable(flatten)] pub PerBlockRollingAverage<Timestamp, Timestamp, M>,
); );
+132 -103
View File
@@ -1,20 +1,22 @@
use brk_error::Result; use brk_error::Result;
use brk_traversable::Traversable; use brk_traversable::Traversable;
use brk_types::{Height, Indexes, Timestamp, Version}; use brk_types::{Height, Indexes, Timestamp, Version};
use vecdb::{AnyVec, CachedVec, Cursor, Database, EagerVec, Exit, ImportableVec, PcoVec, ReadableVec, Rw, StorageMode, VecIndex}; use vecdb::{
AnyVec, CachedVec, Cursor, Database, EagerVec, Exit, ImportableVec, PcoVec, ReadableVec, Rw,
StorageMode, VecIndex,
};
use crate::internal::{CachedWindowStarts, Windows, WindowStarts}; use crate::{
indexes,
use super::time; internal::{WindowStartVec, WindowStarts, Windows},
};
#[derive(Traversable)] #[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> { pub struct Vecs<M: StorageMode = Rw> {
#[traversable(skip)]
pub cached_window_starts: CachedWindowStarts,
pub _1h: M::Stored<EagerVec<PcoVec<Height, Height>>>, pub _1h: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _24h: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 1d pub _24h: CachedVec<M::Stored<EagerVec<PcoVec<Height, Height>>>>, // 1d
pub _3d: M::Stored<EagerVec<PcoVec<Height, Height>>>, pub _3d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _1w: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 7d pub _1w: CachedVec<M::Stored<EagerVec<PcoVec<Height, Height>>>>, // 7d
pub _8d: M::Stored<EagerVec<PcoVec<Height, Height>>>, pub _8d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _9d: M::Stored<EagerVec<PcoVec<Height, Height>>>, pub _9d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _12d: M::Stored<EagerVec<PcoVec<Height, Height>>>, pub _12d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
@@ -22,7 +24,7 @@ pub struct Vecs<M: StorageMode = Rw> {
pub _2w: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 14d pub _2w: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 14d
pub _21d: M::Stored<EagerVec<PcoVec<Height, Height>>>, pub _21d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _26d: M::Stored<EagerVec<PcoVec<Height, Height>>>, pub _26d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _1m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 30d pub _1m: CachedVec<M::Stored<EagerVec<PcoVec<Height, Height>>>>, // 30d
pub _34d: M::Stored<EagerVec<PcoVec<Height, Height>>>, pub _34d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _55d: M::Stored<EagerVec<PcoVec<Height, Height>>>, pub _55d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _2m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 60d pub _2m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 60d
@@ -39,7 +41,7 @@ pub struct Vecs<M: StorageMode = Rw> {
pub _9m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 270d pub _9m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 270d
pub _350d: M::Stored<EagerVec<PcoVec<Height, Height>>>, pub _350d: M::Stored<EagerVec<PcoVec<Height, Height>>>,
pub _12m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 360d pub _12m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 360d
pub _1y: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 365d pub _1y: CachedVec<M::Stored<EagerVec<PcoVec<Height, Height>>>>, // 365d
pub _14m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 420d pub _14m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 420d
pub _2y: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 730d pub _2y: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 730d
pub _26m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 780d pub _26m: M::Stored<EagerVec<PcoVec<Height, Height>>>, // 780d
@@ -102,24 +104,55 @@ impl Vecs {
let _14y = ImportableVec::forced_import(db, "height_14y_ago", version)?; let _14y = ImportableVec::forced_import(db, "height_14y_ago", version)?;
let _26y = ImportableVec::forced_import(db, "height_26y_ago", version)?; let _26y = ImportableVec::forced_import(db, "height_26y_ago", version)?;
let cached_window_starts = CachedWindowStarts(Windows {
_24h: CachedVec::new(&_24h),
_1w: CachedVec::new(&_1w),
_1m: CachedVec::new(&_1m),
_1y: CachedVec::new(&_1y),
});
Ok(Self { Ok(Self {
cached_window_starts, _1h,
_1h, _24h, _3d, _1w, _8d, _9d, _12d, _13d, _2w, _21d, _26d, _24h: CachedVec::wrap(_24h),
_1m, _34d, _55d, _2m, _9w, _12w, _89d, _3m, _14w, _111d, _144d, _3d,
_6m, _26w, _200d, _9m, _350d, _12m, _1y, _14m, _2y, _26m, _3y, _1w: CachedVec::wrap(_1w),
_200w, _4y, _5y, _6y, _8y, _9y, _10y, _12y, _14y, _26y, _8d,
_9d,
_12d,
_13d,
_2w,
_21d,
_26d,
_1m: CachedVec::wrap(_1m),
_34d,
_55d,
_2m,
_9w,
_12w,
_89d,
_3m,
_14w,
_111d,
_144d,
_6m,
_26w,
_200d,
_9m,
_350d,
_12m,
_1y: CachedVec::wrap(_1y),
_14m,
_2y,
_26m,
_3y,
_200w,
_4y,
_5y,
_6y,
_8y,
_9y,
_10y,
_12y,
_14y,
_26y,
}) })
} }
pub fn window_starts(&self) -> WindowStarts<'_> { pub fn cached_window_starts(&self) -> Windows<&WindowStartVec> {
WindowStarts { Windows {
_24h: &self._24h, _24h: &self._24h,
_1w: &self._1w, _1w: &self._1w,
_1m: &self._1m, _1m: &self._1m,
@@ -127,12 +160,20 @@ impl Vecs {
} }
} }
pub fn window_starts(&self) -> WindowStarts<'_> {
WindowStarts {
_24h: &self._24h.inner,
_1w: &self._1w.inner,
_1m: &self._1m.inner,
_1y: &self._1y.inner,
}
}
pub fn start_vec(&self, days: usize) -> &EagerVec<PcoVec<Height, Height>> { pub fn start_vec(&self, days: usize) -> &EagerVec<PcoVec<Height, Height>> {
match days { match days {
1 => &self._24h, 1 => &self._24h.inner,
3 => &self._3d, 3 => &self._3d,
7 => &self._1w, 7 => &self._1w.inner,
8 => &self._8d, 8 => &self._8d,
9 => &self._9d, 9 => &self._9d,
12 => &self._12d, 12 => &self._12d,
@@ -140,7 +181,7 @@ impl Vecs {
14 => &self._2w, 14 => &self._2w,
21 => &self._21d, 21 => &self._21d,
26 => &self._26d, 26 => &self._26d,
30 => &self._1m, 30 => &self._1m.inner,
34 => &self._34d, 34 => &self._34d,
55 => &self._55d, 55 => &self._55d,
60 => &self._2m, 60 => &self._2m,
@@ -157,7 +198,7 @@ impl Vecs {
270 => &self._9m, 270 => &self._9m,
350 => &self._350d, 350 => &self._350d,
360 => &self._12m, 360 => &self._12m,
365 => &self._1y, 365 => &self._1y.inner,
420 => &self._14m, 420 => &self._14m,
730 => &self._2y, 730 => &self._2y,
780 => &self._26m, 780 => &self._26m,
@@ -178,80 +219,60 @@ impl Vecs {
pub(crate) fn compute( pub(crate) fn compute(
&mut self, &mut self,
time: &time::Vecs, indexes: &indexes::Vecs,
starting_indexes: &Indexes, starting_indexes: &Indexes,
exit: &Exit, exit: &Exit,
) -> Result<()> { ) -> Result<()> {
self.compute_rolling_start_hours(time, starting_indexes, exit, 1, |s| { self.compute_rolling_start_hours(indexes, starting_indexes, exit, 1, |s| &mut s._1h)?;
&mut s._1h self.compute_rolling_start(indexes, starting_indexes, exit, 1, |s| &mut s._24h.inner)?;
})?; self.compute_rolling_start(indexes, starting_indexes, exit, 3, |s| &mut s._3d)?;
self.compute_rolling_start(time, starting_indexes, exit, 1, |s| &mut s._24h)?; self.compute_rolling_start(indexes, starting_indexes, exit, 7, |s| &mut s._1w.inner)?;
self.compute_rolling_start(time, starting_indexes, exit, 3, |s| &mut s._3d)?; self.compute_rolling_start(indexes, starting_indexes, exit, 8, |s| &mut s._8d)?;
self.compute_rolling_start(time, starting_indexes, exit, 7, |s| &mut s._1w)?; self.compute_rolling_start(indexes, starting_indexes, exit, 9, |s| &mut s._9d)?;
self.compute_rolling_start(time, starting_indexes, exit, 8, |s| &mut s._8d)?; self.compute_rolling_start(indexes, starting_indexes, exit, 12, |s| &mut s._12d)?;
self.compute_rolling_start(time, starting_indexes, exit, 9, |s| &mut s._9d)?; self.compute_rolling_start(indexes, starting_indexes, exit, 13, |s| &mut s._13d)?;
self.compute_rolling_start(time, starting_indexes, exit, 12, |s| &mut s._12d)?; self.compute_rolling_start(indexes, starting_indexes, exit, 14, |s| &mut s._2w)?;
self.compute_rolling_start(time, starting_indexes, exit, 13, |s| &mut s._13d)?; self.compute_rolling_start(indexes, starting_indexes, exit, 21, |s| &mut s._21d)?;
self.compute_rolling_start(time, starting_indexes, exit, 14, |s| &mut s._2w)?; self.compute_rolling_start(indexes, starting_indexes, exit, 26, |s| &mut s._26d)?;
self.compute_rolling_start(time, starting_indexes, exit, 21, |s| &mut s._21d)?; self.compute_rolling_start(indexes, starting_indexes, exit, 30, |s| &mut s._1m.inner)?;
self.compute_rolling_start(time, starting_indexes, exit, 26, |s| &mut s._26d)?; self.compute_rolling_start(indexes, starting_indexes, exit, 34, |s| &mut s._34d)?;
self.compute_rolling_start(time, starting_indexes, exit, 30, |s| &mut s._1m)?; self.compute_rolling_start(indexes, starting_indexes, exit, 55, |s| &mut s._55d)?;
self.compute_rolling_start(time, starting_indexes, exit, 34, |s| &mut s._34d)?; self.compute_rolling_start(indexes, starting_indexes, exit, 60, |s| &mut s._2m)?;
self.compute_rolling_start(time, starting_indexes, exit, 55, |s| &mut s._55d)?; self.compute_rolling_start(indexes, starting_indexes, exit, 63, |s| &mut s._9w)?;
self.compute_rolling_start(time, starting_indexes, exit, 60, |s| &mut s._2m)?; self.compute_rolling_start(indexes, starting_indexes, exit, 84, |s| &mut s._12w)?;
self.compute_rolling_start(time, starting_indexes, exit, 63, |s| &mut s._9w)?; self.compute_rolling_start(indexes, starting_indexes, exit, 89, |s| &mut s._89d)?;
self.compute_rolling_start(time, starting_indexes, exit, 84, |s| &mut s._12w)?; self.compute_rolling_start(indexes, starting_indexes, exit, 90, |s| &mut s._3m)?;
self.compute_rolling_start(time, starting_indexes, exit, 89, |s| &mut s._89d)?; self.compute_rolling_start(indexes, starting_indexes, exit, 98, |s| &mut s._14w)?;
self.compute_rolling_start(time, starting_indexes, exit, 90, |s| &mut s._3m)?; self.compute_rolling_start(indexes, starting_indexes, exit, 111, |s| &mut s._111d)?;
self.compute_rolling_start(time, starting_indexes, exit, 98, |s| &mut s._14w)?; self.compute_rolling_start(indexes, starting_indexes, exit, 144, |s| &mut s._144d)?;
self.compute_rolling_start(time, starting_indexes, exit, 111, |s| { self.compute_rolling_start(indexes, starting_indexes, exit, 180, |s| &mut s._6m)?;
&mut s._111d self.compute_rolling_start(indexes, starting_indexes, exit, 182, |s| &mut s._26w)?;
})?; self.compute_rolling_start(indexes, starting_indexes, exit, 200, |s| &mut s._200d)?;
self.compute_rolling_start(time, starting_indexes, exit, 144, |s| { self.compute_rolling_start(indexes, starting_indexes, exit, 270, |s| &mut s._9m)?;
&mut s._144d self.compute_rolling_start(indexes, starting_indexes, exit, 350, |s| &mut s._350d)?;
})?; self.compute_rolling_start(indexes, starting_indexes, exit, 360, |s| &mut s._12m)?;
self.compute_rolling_start(time, starting_indexes, exit, 180, |s| &mut s._6m)?; self.compute_rolling_start(indexes, starting_indexes, exit, 365, |s| &mut s._1y.inner)?;
self.compute_rolling_start(time, starting_indexes, exit, 182, |s| &mut s._26w)?; self.compute_rolling_start(indexes, starting_indexes, exit, 420, |s| &mut s._14m)?;
self.compute_rolling_start(time, starting_indexes, exit, 200, |s| { self.compute_rolling_start(indexes, starting_indexes, exit, 730, |s| &mut s._2y)?;
&mut s._200d self.compute_rolling_start(indexes, starting_indexes, exit, 780, |s| &mut s._26m)?;
})?; self.compute_rolling_start(indexes, starting_indexes, exit, 1095, |s| &mut s._3y)?;
self.compute_rolling_start(time, starting_indexes, exit, 270, |s| &mut s._9m)?; self.compute_rolling_start(indexes, starting_indexes, exit, 1400, |s| &mut s._200w)?;
self.compute_rolling_start(time, starting_indexes, exit, 350, |s| { self.compute_rolling_start(indexes, starting_indexes, exit, 1460, |s| &mut s._4y)?;
&mut s._350d self.compute_rolling_start(indexes, starting_indexes, exit, 1825, |s| &mut s._5y)?;
})?; self.compute_rolling_start(indexes, starting_indexes, exit, 2190, |s| &mut s._6y)?;
self.compute_rolling_start(time, starting_indexes, exit, 360, |s| &mut s._12m)?; self.compute_rolling_start(indexes, starting_indexes, exit, 2920, |s| &mut s._8y)?;
self.compute_rolling_start(time, starting_indexes, exit, 365, |s| &mut s._1y)?; self.compute_rolling_start(indexes, starting_indexes, exit, 3285, |s| &mut s._9y)?;
self.compute_rolling_start(time, starting_indexes, exit, 420, |s| &mut s._14m)?; self.compute_rolling_start(indexes, starting_indexes, exit, 3650, |s| &mut s._10y)?;
self.compute_rolling_start(time, starting_indexes, exit, 730, |s| &mut s._2y)?; self.compute_rolling_start(indexes, starting_indexes, exit, 4380, |s| &mut s._12y)?;
self.compute_rolling_start(time, starting_indexes, exit, 780, |s| &mut s._26m)?; self.compute_rolling_start(indexes, starting_indexes, exit, 5110, |s| &mut s._14y)?;
self.compute_rolling_start(time, starting_indexes, exit, 1095, |s| &mut s._3y)?; self.compute_rolling_start(indexes, starting_indexes, exit, 9490, |s| &mut s._26y)?;
self.compute_rolling_start(time, starting_indexes, exit, 1400, |s| {
&mut s._200w
})?;
self.compute_rolling_start(time, starting_indexes, exit, 1460, |s| &mut s._4y)?;
self.compute_rolling_start(time, starting_indexes, exit, 1825, |s| &mut s._5y)?;
self.compute_rolling_start(time, starting_indexes, exit, 2190, |s| &mut s._6y)?;
self.compute_rolling_start(time, starting_indexes, exit, 2920, |s| &mut s._8y)?;
self.compute_rolling_start(time, starting_indexes, exit, 3285, |s| &mut s._9y)?;
self.compute_rolling_start(time, starting_indexes, exit, 3650, |s| {
&mut s._10y
})?;
self.compute_rolling_start(time, starting_indexes, exit, 4380, |s| {
&mut s._12y
})?;
self.compute_rolling_start(time, starting_indexes, exit, 5110, |s| {
&mut s._14y
})?;
self.compute_rolling_start(time, starting_indexes, exit, 9490, |s| {
&mut s._26y
})?;
Ok(()) Ok(())
} }
fn compute_rolling_start<F>( fn compute_rolling_start<F>(
&mut self, &mut self,
time: &time::Vecs, indexes: &indexes::Vecs,
starting_indexes: &Indexes, starting_indexes: &Indexes,
exit: &Exit, exit: &Exit,
days: usize, days: usize,
@@ -260,14 +281,18 @@ impl Vecs {
where where
F: FnOnce(&mut Self) -> &mut EagerVec<PcoVec<Height, Height>>, F: FnOnce(&mut Self) -> &mut EagerVec<PcoVec<Height, Height>>,
{ {
self.compute_rolling_start_inner(time, starting_indexes, exit, get_field, |t, prev_ts| { self.compute_rolling_start_inner(
t.difference_in_days_between(prev_ts) >= days indexes,
}) starting_indexes,
exit,
get_field,
|t, prev_ts| t.difference_in_days_between(prev_ts) >= days,
)
} }
fn compute_rolling_start_hours<F>( fn compute_rolling_start_hours<F>(
&mut self, &mut self,
time: &time::Vecs, indexes: &indexes::Vecs,
starting_indexes: &Indexes, starting_indexes: &Indexes,
exit: &Exit, exit: &Exit,
hours: usize, hours: usize,
@@ -276,14 +301,18 @@ impl Vecs {
where where
F: FnOnce(&mut Self) -> &mut EagerVec<PcoVec<Height, Height>>, F: FnOnce(&mut Self) -> &mut EagerVec<PcoVec<Height, Height>>,
{ {
self.compute_rolling_start_inner(time, starting_indexes, exit, get_field, |t, prev_ts| { self.compute_rolling_start_inner(
t.difference_in_hours_between(prev_ts) >= hours indexes,
}) starting_indexes,
exit,
get_field,
|t, prev_ts| t.difference_in_hours_between(prev_ts) >= hours,
)
} }
fn compute_rolling_start_inner<F, D>( fn compute_rolling_start_inner<F, D>(
&mut self, &mut self,
time: &time::Vecs, indexes: &indexes::Vecs,
starting_indexes: &Indexes, starting_indexes: &Indexes,
exit: &Exit, exit: &Exit,
get_field: F, get_field: F,
@@ -300,12 +329,12 @@ impl Vecs {
} else { } else {
Height::ZERO Height::ZERO
}; };
let mut cursor = Cursor::new(&time.timestamp_monotonic); let mut cursor = Cursor::new(&indexes.timestamp.monotonic);
cursor.advance(prev.to_usize()); cursor.advance(prev.to_usize());
let mut prev_ts = cursor.next().unwrap(); let mut prev_ts = cursor.next().unwrap();
Ok(field.compute_transform( Ok(field.compute_transform(
starting_indexes.height, starting_indexes.height,
&time.timestamp_monotonic, &indexes.timestamp.monotonic,
|(h, t, ..)| { |(h, t, ..)| {
while expired(t, prev_ts) { while expired(t, prev_ts) {
prev.increment(); prev.increment();
+1 -4
View File
@@ -4,7 +4,6 @@ pub mod halving;
pub mod interval; pub mod interval;
pub mod lookback; pub mod lookback;
pub mod size; pub mod size;
pub mod time;
pub mod weight; pub mod weight;
mod compute; mod compute;
@@ -19,7 +18,6 @@ pub use halving::Vecs as HalvingVecs;
pub use interval::Vecs as IntervalVecs; pub use interval::Vecs as IntervalVecs;
pub use lookback::Vecs as LookbackVecs; pub use lookback::Vecs as LookbackVecs;
pub use size::Vecs as SizeVecs; pub use size::Vecs as SizeVecs;
pub use time::Vecs as TimeVecs;
pub use weight::Vecs as WeightVecs; pub use weight::Vecs as WeightVecs;
pub const DB_NAME: &str = "blocks"; pub const DB_NAME: &str = "blocks";
@@ -37,7 +35,7 @@ pub(crate) const ONE_TERA_HASH: f64 = 1_000_000_000_000.0;
#[derive(Traversable)] #[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> { pub struct Vecs<M: StorageMode = Rw> {
#[traversable(skip)] #[traversable(skip)]
pub(crate) db: Database, pub db: Database,
pub count: CountVecs<M>, pub count: CountVecs<M>,
pub lookback: LookbackVecs<M>, pub lookback: LookbackVecs<M>,
@@ -46,7 +44,6 @@ pub struct Vecs<M: StorageMode = Rw> {
pub size: SizeVecs<M>, pub size: SizeVecs<M>,
#[traversable(flatten)] #[traversable(flatten)]
pub weight: WeightVecs<M>, pub weight: WeightVecs<M>,
pub time: TimeVecs<M>,
pub difficulty: DifficultyVecs<M>, pub difficulty: DifficultyVecs<M>,
pub halving: HalvingVecs<M>, pub halving: HalvingVecs<M>,
} }
@@ -5,7 +5,7 @@ use vecdb::Database;
use super::Vecs; use super::Vecs;
use crate::{ use crate::{
indexes, indexes,
internal::{CachedWindowStarts, PerBlockFull, PerBlockRolling}, internal::{PerBlockFull, PerBlockRolling, WindowStartVec, Windows},
}; };
impl Vecs { impl Vecs {
@@ -13,7 +13,7 @@ impl Vecs {
db: &Database, db: &Database,
version: Version, version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts, cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> { ) -> Result<Self> {
Ok(Self { Ok(Self {
vbytes: PerBlockFull::forced_import( vbytes: PerBlockFull::forced_import(
@@ -1,34 +0,0 @@
use brk_error::Result;
use brk_indexer::Indexer;
use vecdb::{Exit, ReadableVec};
use super::Vecs;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
starting_height: brk_types::Height,
exit: &Exit,
) -> Result<()> {
let mut prev_timestamp_monotonic = None;
self.timestamp_monotonic.compute_transform(
starting_height,
&indexer.vecs.blocks.timestamp,
|(h, timestamp, this)| {
if prev_timestamp_monotonic.is_none()
&& let Some(prev_h) = h.decremented()
{
prev_timestamp_monotonic.replace(this.collect_one(prev_h).unwrap());
}
let timestamp_monotonic =
prev_timestamp_monotonic.map_or(timestamp, |prev_d| prev_d.max(timestamp));
prev_timestamp_monotonic.replace(timestamp_monotonic);
(h, timestamp_monotonic)
},
exit,
)?;
Ok(())
}
}
@@ -1,66 +0,0 @@
use brk_error::Result;
use brk_types::{Date, Height, Version};
use vecdb::{Database, EagerVec, ImportableVec, LazyVecFrom1, ReadableCloneableVec};
use super::{TimestampIndexes, Vecs};
use crate::indexes;
impl Vecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let timestamp_monotonic = EagerVec::forced_import(db, "timestamp_monotonic", version)?;
Ok(Self {
date: LazyVecFrom1::init(
"date",
version,
timestamp_monotonic.read_only_boxed_clone(),
|_height: Height, timestamp| Date::from(timestamp),
),
timestamp_monotonic,
timestamp: TimestampIndexes::forced_import(db, version, indexes)?,
})
}
}
impl TimestampIndexes {
fn forced_import(db: &Database, version: Version, indexes: &indexes::Vecs) -> Result<Self> {
macro_rules! period {
($field:ident) => {
LazyVecFrom1::init(
"timestamp",
version,
indexes.$field.first_height.read_only_boxed_clone(),
|idx, _: Height| idx.to_timestamp(),
)
};
}
macro_rules! epoch {
($field:ident) => {
ImportableVec::forced_import(db, "timestamp", version)?
};
}
Ok(Self(crate::internal::PerResolution {
minute10: period!(minute10),
minute30: period!(minute30),
hour1: period!(hour1),
hour4: period!(hour4),
hour12: period!(hour12),
day1: period!(day1),
day3: period!(day3),
week1: period!(week1),
month1: period!(month1),
month3: period!(month3),
month6: period!(month6),
year1: period!(year1),
year10: period!(year10),
halving: epoch!(halving),
epoch: epoch!(difficulty),
}))
}
}
@@ -1,5 +0,0 @@
mod compute;
mod import;
mod vecs;
pub use vecs::{TimestampIndexes, Vecs};
@@ -1,80 +0,0 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{
Date, Day1, Day3, Epoch, Halving, Height, Hour1, Hour4, Hour12, Indexes,
Minute10, Minute30, Month1, Month3, Month6, Timestamp, Week1, Year1, Year10,
};
use derive_more::{Deref, DerefMut};
use vecdb::{EagerVec, Exit, LazyVecFrom1, PcoVec, ReadableVec, Rw, StorageMode};
use crate::{indexes, internal::PerResolution};
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub date: LazyVecFrom1<Height, Date, Height, Timestamp>,
pub timestamp_monotonic: M::Stored<EagerVec<PcoVec<Height, Timestamp>>>,
pub timestamp: TimestampIndexes<M>,
}
/// Per-period timestamp indexes.
///
/// Time-based periods (minute10year10) are lazy: `idx.to_timestamp()` is a pure
/// function of the index, so no storage or decompression is needed.
/// Epoch-based periods (halving, difficulty) are eager: their timestamps
/// come from block data via `compute_indirect_sequential`.
#[derive(Deref, DerefMut, Traversable)]
#[traversable(transparent)]
pub struct TimestampIndexes<M: StorageMode = Rw>(
#[allow(clippy::type_complexity)]
pub PerResolution<
LazyVecFrom1<Minute10, Timestamp, Minute10, Height>,
LazyVecFrom1<Minute30, Timestamp, Minute30, Height>,
LazyVecFrom1<Hour1, Timestamp, Hour1, Height>,
LazyVecFrom1<Hour4, Timestamp, Hour4, Height>,
LazyVecFrom1<Hour12, Timestamp, Hour12, Height>,
LazyVecFrom1<Day1, Timestamp, Day1, Height>,
LazyVecFrom1<Day3, Timestamp, Day3, Height>,
LazyVecFrom1<Week1, Timestamp, Week1, Height>,
LazyVecFrom1<Month1, Timestamp, Month1, Height>,
LazyVecFrom1<Month3, Timestamp, Month3, Height>,
LazyVecFrom1<Month6, Timestamp, Month6, Height>,
LazyVecFrom1<Year1, Timestamp, Year1, Height>,
LazyVecFrom1<Year10, Timestamp, Year10, Height>,
M::Stored<EagerVec<PcoVec<Halving, Timestamp>>>,
M::Stored<EagerVec<PcoVec<Epoch, Timestamp>>>,
>,
);
impl TimestampIndexes {
/// Compute epoch timestamps via indirect lookup from block timestamps.
/// Time-based periods are lazy (idx.to_timestamp()) and need no compute.
pub(crate) fn compute(
&mut self,
indexer: &brk_indexer::Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let prev_height = starting_indexes.height.decremented().unwrap_or_default();
self.halving.compute_indirect_sequential(
indexes
.height
.halving
.collect_one(prev_height)
.unwrap_or_default(),
&indexes.halving.first_height,
&indexer.vecs.blocks.timestamp,
exit,
)?;
self.epoch.compute_indirect_sequential(
indexes
.height
.epoch
.collect_one(prev_height)
.unwrap_or_default(),
&indexes.epoch.first_height,
&indexer.vecs.blocks.timestamp,
exit,
)?;
Ok(())
}
}
@@ -6,7 +6,7 @@ use super::Vecs;
use crate::{ use crate::{
blocks::SizeVecs, blocks::SizeVecs,
indexes, indexes,
internal::{CachedWindowStarts, LazyPerBlockRolling, PercentVec, VBytesToWeight}, internal::{LazyPerBlockRolling, PercentVec, VBytesToWeight, WindowStartVec, Windows},
}; };
impl Vecs { impl Vecs {
@@ -14,7 +14,7 @@ impl Vecs {
db: &Database, db: &Database,
version: Version, version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts, cached_starts: &Windows<&WindowStartVec>,
size: &SizeVecs, size: &SizeVecs,
) -> Result<Self> { ) -> Result<Self> {
let weight = LazyPerBlockRolling::from_per_block_full::<VBytesToWeight>( let weight = LazyPerBlockRolling::from_per_block_full::<VBytesToWeight>(
@@ -6,7 +6,7 @@ use super::Vecs;
use crate::{ use crate::{
indexes, indexes,
internal::{ internal::{
CachedWindowStarts, LazyPerBlock, OneMinusF64, PerBlock, PerBlockCumulativeRolling, LazyPerBlock, OneMinusF64, PerBlock, PerBlockCumulativeRolling, WindowStartVec, Windows,
}, },
}; };
@@ -15,7 +15,7 @@ impl Vecs {
db: &Database, db: &Database,
version: Version, version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts, cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> { ) -> Result<Self> {
let liveliness = PerBlock::forced_import(db, "liveliness", version, indexes)?; let liveliness = PerBlock::forced_import(db, "liveliness", version, indexes)?;
@@ -28,10 +28,18 @@ impl Vecs {
Ok(Self { Ok(Self {
coinblocks_created: PerBlockCumulativeRolling::forced_import( coinblocks_created: PerBlockCumulativeRolling::forced_import(
db, "coinblocks_created", version, indexes, cached_starts, db,
"coinblocks_created",
version,
indexes,
cached_starts,
)?, )?,
coinblocks_stored: PerBlockCumulativeRolling::forced_import( coinblocks_stored: PerBlockCumulativeRolling::forced_import(
db, "coinblocks_stored", version, indexes, cached_starts, db,
"coinblocks_stored",
version,
indexes,
cached_starts,
)?, )?,
liveliness, liveliness,
vaultedness, vaultedness,
+7 -2
View File
@@ -17,6 +17,8 @@ impl Vecs {
distribution: &distribution::Vecs, distribution: &distribution::Vecs,
exit: &Exit, exit: &Exit,
) -> Result<()> { ) -> Result<()> {
self.db.sync_bg_tasks()?;
// Activity computes first (liveliness, vaultedness, etc.) // Activity computes first (liveliness, vaultedness, etc.)
self.activity self.activity
.compute(starting_indexes, distribution, exit)?; .compute(starting_indexes, distribution, exit)?;
@@ -80,8 +82,11 @@ impl Vecs {
r3?; r3?;
r4?; r4?;
let _lock = exit.lock(); let exit = exit.clone();
self.db.compact()?; self.db.run_bg(move |db| {
let _lock = exit.lock();
db.compact_deferred_default()
});
Ok(()) Ok(())
} }
} }
+2 -2
View File
@@ -13,14 +13,14 @@ use super::{
ValueVecs, Vecs, ValueVecs, Vecs,
}; };
use crate::internal::CachedWindowStarts; use crate::internal::{WindowStartVec, Windows};
impl Vecs { impl Vecs {
pub(crate) fn forced_import( pub(crate) fn forced_import(
parent_path: &Path, parent_path: &Path,
parent_version: Version, parent_version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts, cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> { ) -> Result<Self> {
let db = open_db(parent_path, DB_NAME, 250_000)?; let db = open_db(parent_path, DB_NAME, 250_000)?;
let version = parent_version; let version = parent_version;
@@ -22,11 +22,8 @@ impl Vecs {
let circulating_supply = &all_metrics.supply.total.btc.height; let circulating_supply = &all_metrics.supply.total.btc.height;
let realized_price = &all_metrics.realized.price.cents.height; let realized_price = &all_metrics.realized.price.cents.height;
self.vaulted.compute_all( self.vaulted
prices, .compute_all(prices, starting_indexes, exit, |v| {
starting_indexes,
exit,
|v| {
Ok(v.compute_transform2( Ok(v.compute_transform2(
starting_indexes.height, starting_indexes.height,
realized_price, realized_price,
@@ -36,14 +33,10 @@ impl Vecs {
}, },
exit, exit,
)?) )?)
}, })?;
)?;
self.active.compute_all( self.active
prices, .compute_all(prices, starting_indexes, exit, |v| {
starting_indexes,
exit,
|v| {
Ok(v.compute_transform2( Ok(v.compute_transform2(
starting_indexes.height, starting_indexes.height,
realized_price, realized_price,
@@ -53,14 +46,10 @@ impl Vecs {
}, },
exit, exit,
)?) )?)
}, })?;
)?;
self.true_market_mean.compute_all( self.true_market_mean
prices, .compute_all(prices, starting_indexes, exit, |v| {
starting_indexes,
exit,
|v| {
Ok(v.compute_transform2( Ok(v.compute_transform2(
starting_indexes.height, starting_indexes.height,
&cap.investor.cents.height, &cap.investor.cents.height,
@@ -70,14 +59,10 @@ impl Vecs {
}, },
exit, exit,
)?) )?)
}, })?;
)?;
self.cointime.compute_all( self.cointime
prices, .compute_all(prices, starting_indexes, exit, |v| {
starting_indexes,
exit,
|v| {
Ok(v.compute_transform2( Ok(v.compute_transform2(
starting_indexes.height, starting_indexes.height,
&cap.cointime.cents.height, &cap.cointime.cents.height,
@@ -87,8 +72,7 @@ impl Vecs {
}, },
exit, exit,
)?) )?)
}, })?;
)?;
Ok(()) Ok(())
} }
@@ -3,10 +3,7 @@ use brk_types::Version;
use vecdb::Database; use vecdb::Database;
use super::Vecs; use super::Vecs;
use crate::{ use crate::{indexes, internal::PriceWithRatioExtendedPerBlock};
indexes,
internal::PriceWithRatioExtendedPerBlock,
};
impl Vecs { impl Vecs {
pub(crate) fn forced_import( pub(crate) fn forced_import(
@@ -38,7 +38,8 @@ impl Vecs {
exit, exit,
)?; )?;
self.vaulted.compute(prices, starting_indexes.height, exit)?; self.vaulted
.compute(prices, starting_indexes.height, exit)?;
self.active.compute(prices, starting_indexes.height, exit)?; self.active.compute(prices, starting_indexes.height, exit)?;
Ok(()) Ok(())
@@ -3,7 +3,7 @@ use brk_types::Version;
use vecdb::Database; use vecdb::Database;
use super::Vecs; use super::Vecs;
use crate::{indexes, internal::AmountPerBlock}; use crate::{indexes, internal::ValuePerBlock};
impl Vecs { impl Vecs {
pub(crate) fn forced_import( pub(crate) fn forced_import(
@@ -12,13 +12,8 @@ impl Vecs {
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
) -> Result<Self> { ) -> Result<Self> {
Ok(Self { Ok(Self {
vaulted: AmountPerBlock::forced_import( vaulted: ValuePerBlock::forced_import(db, "vaulted_supply", version, indexes)?,
db, active: ValuePerBlock::forced_import(db, "active_supply", version, indexes)?,
"vaulted_supply",
version,
indexes,
)?,
active: AmountPerBlock::forced_import(db, "active_supply", version, indexes)?,
}) })
} }
} }
@@ -1,10 +1,10 @@
use brk_traversable::Traversable; use brk_traversable::Traversable;
use vecdb::{Rw, StorageMode}; use vecdb::{Rw, StorageMode};
use crate::internal::AmountPerBlock; use crate::internal::ValuePerBlock;
#[derive(Traversable)] #[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> { pub struct Vecs<M: StorageMode = Rw> {
pub vaulted: AmountPerBlock<M>, pub vaulted: ValuePerBlock<M>,
pub active: AmountPerBlock<M>, pub active: ValuePerBlock<M>,
} }
@@ -31,52 +31,49 @@ impl Vecs {
Ok(()) Ok(())
})?; })?;
self.created self.created.compute(starting_indexes.height, exit, |vec| {
.compute(starting_indexes.height, exit, |vec| { vec.compute_multiply(
vec.compute_multiply( starting_indexes.height,
starting_indexes.height, &prices.spot.usd.height,
&prices.spot.usd.height, &activity.coinblocks_created.block,
&activity.coinblocks_created.block, exit,
exit, )?;
)?; Ok(())
Ok(()) })?;
})?;
self.stored self.stored.compute(starting_indexes.height, exit, |vec| {
.compute(starting_indexes.height, exit, |vec| { vec.compute_multiply(
vec.compute_multiply( starting_indexes.height,
starting_indexes.height, &prices.spot.usd.height,
&prices.spot.usd.height, &activity.coinblocks_stored.block,
&activity.coinblocks_stored.block, exit,
exit, )?;
)?; Ok(())
Ok(()) })?;
})?;
// VOCDD: Value of Coin Days Destroyed = price × (CDD / circulating_supply) // VOCDD: Value of Coin Days Destroyed = price × (CDD / circulating_supply)
// Supply-adjusted to account for growing supply over time // Supply-adjusted to account for growing supply over time
// This is a key input for Reserve Risk / HODL Bank calculation // This is a key input for Reserve Risk / HODL Bank calculation
self.vocdd self.vocdd.compute(starting_indexes.height, exit, |vec| {
.compute(starting_indexes.height, exit, |vec| { vec.compute_transform3(
vec.compute_transform3( starting_indexes.height,
starting_indexes.height, &prices.spot.usd.height,
&prices.spot.usd.height, &coindays_destroyed.block,
&coindays_destroyed.block, circulating_supply,
circulating_supply, |(i, price, cdd, supply, _): (_, Dollars, StoredF64, Bitcoin, _)| {
|(i, price, cdd, supply, _): (_, Dollars, StoredF64, Bitcoin, _)| { let supply_f64 = f64::from(supply);
let supply_f64 = f64::from(supply); if supply_f64 == 0.0 {
if supply_f64 == 0.0 { (i, StoredF64::from(0.0))
(i, StoredF64::from(0.0)) } else {
} else { // VOCDD = price × (CDD / supply)
// VOCDD = price × (CDD / supply) let vocdd = f64::from(price) * f64::from(cdd) / supply_f64;
let vocdd = f64::from(price) * f64::from(cdd) / supply_f64; (i, StoredF64::from(vocdd))
(i, StoredF64::from(vocdd)) }
} },
}, exit,
exit, )?;
)?; Ok(())
Ok(()) })?;
})?;
Ok(()) Ok(())
} }
@@ -5,7 +5,7 @@ use vecdb::Database;
use super::Vecs; use super::Vecs;
use crate::{ use crate::{
indexes, indexes,
internal::{CachedWindowStarts, PerBlockCumulativeRolling}, internal::{PerBlockCumulativeRolling, WindowStartVec, Windows},
}; };
impl Vecs { impl Vecs {
@@ -13,7 +13,7 @@ impl Vecs {
db: &Database, db: &Database,
version: Version, version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts, cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> { ) -> Result<Self> {
Ok(Self { Ok(Self {
destroyed: PerBlockCumulativeRolling::forced_import( destroyed: PerBlockCumulativeRolling::forced_import(
@@ -7,19 +7,20 @@
//! | `receiving` | Unique addresses that received this block | //! | `receiving` | Unique addresses that received this block |
//! | `sending` | Unique addresses that sent this block | //! | `sending` | Unique addresses that sent this block |
//! | `reactivated` | Addresses that were empty and now have funds | //! | `reactivated` | Addresses that were empty and now have funds |
//! | `both` | Addresses that both sent AND received same block | //! | `bidirectional` | Addresses that both sent AND received in same block |
//! | `active` | Distinct addresses involved (sent received) |
use brk_cohort::ByAddrType; use brk_cohort::ByAddrType;
use brk_error::Result; use brk_error::Result;
use brk_traversable::Traversable; use brk_traversable::Traversable;
use brk_types::{Height, StoredU32, Version}; use brk_types::{Height, StoredU32, StoredU64, Version};
use derive_more::{Deref, DerefMut}; use derive_more::{Deref, DerefMut};
use rayon::prelude::*; use rayon::prelude::*;
use vecdb::{AnyStoredVec, AnyVec, Database, Exit, Rw, StorageMode, WritableVec}; use vecdb::{AnyStoredVec, AnyVec, Database, Exit, Rw, StorageMode, WritableVec};
use crate::{ use crate::{
indexes, indexes,
internal::{CachedWindowStarts, PerBlockRollingAverage}, internal::{PerBlockRollingAverage, WindowStartVec, Windows},
}; };
/// Per-block activity counts - reset each block. /// Per-block activity counts - reset each block.
@@ -28,7 +29,7 @@ pub struct BlockActivityCounts {
pub reactivated: u32, pub reactivated: u32,
pub sending: u32, pub sending: u32,
pub receiving: u32, pub receiving: u32,
pub both: u32, pub bidirectional: u32,
} }
impl BlockActivityCounts { impl BlockActivityCounts {
@@ -56,7 +57,7 @@ impl AddrTypeToActivityCounts {
total.reactivated += counts.reactivated; total.reactivated += counts.reactivated;
total.sending += counts.sending; total.sending += counts.sending;
total.receiving += counts.receiving; total.receiving += counts.receiving;
total.both += counts.both; total.bidirectional += counts.bidirectional;
} }
total total
} }
@@ -65,45 +66,61 @@ impl AddrTypeToActivityCounts {
/// Activity count vectors for a single category (e.g., one address type or "all"). /// Activity count vectors for a single category (e.g., one address type or "all").
#[derive(Traversable)] #[derive(Traversable)]
pub struct ActivityCountVecs<M: StorageMode = Rw> { pub struct ActivityCountVecs<M: StorageMode = Rw> {
pub reactivated: PerBlockRollingAverage<StoredU32, M>, pub reactivated: PerBlockRollingAverage<StoredU32, StoredU64, M>,
pub sending: PerBlockRollingAverage<StoredU32, M>, pub sending: PerBlockRollingAverage<StoredU32, StoredU64, M>,
pub receiving: PerBlockRollingAverage<StoredU32, M>, pub receiving: PerBlockRollingAverage<StoredU32, StoredU64, M>,
pub both: PerBlockRollingAverage<StoredU32, M>, pub bidirectional: PerBlockRollingAverage<StoredU32, StoredU64, M>,
/// Distinct addresses involved in this block (sent received),
/// computed at push time as `sending + receiving - bidirectional`
/// via inclusion-exclusion. For per-type instances this is
/// per-type. For the `all` aggregate it's the cross-type total.
pub active: PerBlockRollingAverage<StoredU32, StoredU64, M>,
} }
impl ActivityCountVecs { impl ActivityCountVecs {
/// `prefix` is prepended to each field's disk name. Use `""` for the
/// "all" aggregate and `"{type}_"` for per-address-type instances.
/// Field names are suffixed with `_addrs` so the final disk series
/// are e.g. `active_addrs`, `p2tr_bidirectional_addrs`.
pub(crate) fn forced_import( pub(crate) fn forced_import(
db: &Database, db: &Database,
name: &str, prefix: &str,
version: Version, version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts, cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> { ) -> Result<Self> {
Ok(Self { Ok(Self {
reactivated: PerBlockRollingAverage::forced_import( reactivated: PerBlockRollingAverage::forced_import(
db, db,
&format!("{name}_reactivated"), &format!("{prefix}reactivated_addrs"),
version, version,
indexes, indexes,
cached_starts, cached_starts,
)?, )?,
sending: PerBlockRollingAverage::forced_import( sending: PerBlockRollingAverage::forced_import(
db, db,
&format!("{name}_sending"), &format!("{prefix}sending_addrs"),
version, version,
indexes, indexes,
cached_starts, cached_starts,
)?, )?,
receiving: PerBlockRollingAverage::forced_import( receiving: PerBlockRollingAverage::forced_import(
db, db,
&format!("{name}_receiving"), &format!("{prefix}receiving_addrs"),
version, version,
indexes, indexes,
cached_starts, cached_starts,
)?, )?,
both: PerBlockRollingAverage::forced_import( bidirectional: PerBlockRollingAverage::forced_import(
db, db,
&format!("{name}_both"), &format!("{prefix}bidirectional_addrs"),
version,
indexes,
cached_starts,
)?,
active: PerBlockRollingAverage::forced_import(
db,
&format!("{prefix}active_addrs"),
version, version,
indexes, indexes,
cached_starts, cached_starts,
@@ -117,7 +134,8 @@ impl ActivityCountVecs {
.len() .len()
.min(self.sending.block.len()) .min(self.sending.block.len())
.min(self.receiving.block.len()) .min(self.receiving.block.len())
.min(self.both.block.len()) .min(self.bidirectional.block.len())
.min(self.active.block.len())
} }
pub(crate) fn par_iter_height_mut( pub(crate) fn par_iter_height_mut(
@@ -125,9 +143,10 @@ impl ActivityCountVecs {
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> { ) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[ [
&mut self.reactivated.block as &mut dyn AnyStoredVec, &mut self.reactivated.block as &mut dyn AnyStoredVec,
&mut self.sending.block as &mut dyn AnyStoredVec, &mut self.sending.block,
&mut self.receiving.block as &mut dyn AnyStoredVec, &mut self.receiving.block,
&mut self.both.block as &mut dyn AnyStoredVec, &mut self.bidirectional.block,
&mut self.active.block,
] ]
.into_par_iter() .into_par_iter()
} }
@@ -136,7 +155,8 @@ impl ActivityCountVecs {
self.reactivated.block.reset()?; self.reactivated.block.reset()?;
self.sending.block.reset()?; self.sending.block.reset()?;
self.receiving.block.reset()?; self.receiving.block.reset()?;
self.both.block.reset()?; self.bidirectional.block.reset()?;
self.active.block.reset()?;
Ok(()) Ok(())
} }
@@ -145,18 +165,19 @@ impl ActivityCountVecs {
self.reactivated.block.push(counts.reactivated.into()); self.reactivated.block.push(counts.reactivated.into());
self.sending.block.push(counts.sending.into()); self.sending.block.push(counts.sending.into());
self.receiving.block.push(counts.receiving.into()); self.receiving.block.push(counts.receiving.into());
self.both.block.push(counts.both.into()); self.bidirectional
.block
.push(counts.bidirectional.into());
let active = counts.sending + counts.receiving - counts.bidirectional;
self.active.block.push(active.into());
} }
pub(crate) fn compute_rest( pub(crate) fn compute_rest(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
&mut self,
max_from: Height,
exit: &Exit,
) -> Result<()> {
self.reactivated.compute_rest(max_from, exit)?; self.reactivated.compute_rest(max_from, exit)?;
self.sending.compute_rest(max_from, exit)?; self.sending.compute_rest(max_from, exit)?;
self.receiving.compute_rest(max_from, exit)?; self.receiving.compute_rest(max_from, exit)?;
self.both.compute_rest(max_from, exit)?; self.bidirectional.compute_rest(max_from, exit)?;
self.active.compute_rest(max_from, exit)?;
Ok(()) Ok(())
} }
} }
@@ -175,22 +196,21 @@ impl From<ByAddrType<ActivityCountVecs>> for AddrTypeToActivityCountVecs {
impl AddrTypeToActivityCountVecs { impl AddrTypeToActivityCountVecs {
pub(crate) fn forced_import( pub(crate) fn forced_import(
db: &Database, db: &Database,
name: &str,
version: Version, version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts, cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> { ) -> Result<Self> {
Ok(Self::from( Ok(Self::from(ByAddrType::<ActivityCountVecs>::new_with_name(
ByAddrType::<ActivityCountVecs>::new_with_name(|type_name| { |type_name| {
ActivityCountVecs::forced_import( ActivityCountVecs::forced_import(
db, db,
&format!("{type_name}_{name}"), &format!("{type_name}_"),
version, version,
indexes, indexes,
cached_starts, cached_starts,
) )
})?, },
)) )?))
} }
pub(crate) fn min_stateful_len(&self) -> usize { pub(crate) fn min_stateful_len(&self) -> usize {
@@ -209,7 +229,8 @@ impl AddrTypeToActivityCountVecs {
vecs.push(&mut type_vecs.reactivated.block); vecs.push(&mut type_vecs.reactivated.block);
vecs.push(&mut type_vecs.sending.block); vecs.push(&mut type_vecs.sending.block);
vecs.push(&mut type_vecs.receiving.block); vecs.push(&mut type_vecs.receiving.block);
vecs.push(&mut type_vecs.both.block); vecs.push(&mut type_vecs.bidirectional.block);
vecs.push(&mut type_vecs.active.block);
} }
vecs.into_par_iter() vecs.into_par_iter()
} }
@@ -221,11 +242,7 @@ impl AddrTypeToActivityCountVecs {
Ok(()) Ok(())
} }
pub(crate) fn compute_rest( pub(crate) fn compute_rest(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
&mut self,
max_from: Height,
exit: &Exit,
) -> Result<()> {
for type_vecs in self.0.values_mut() { for type_vecs in self.0.values_mut() {
type_vecs.compute_rest(max_from, exit)?; type_vecs.compute_rest(max_from, exit)?;
} }
@@ -251,15 +268,17 @@ pub struct AddrActivityVecs<M: StorageMode = Rw> {
impl AddrActivityVecs { impl AddrActivityVecs {
pub(crate) fn forced_import( pub(crate) fn forced_import(
db: &Database, db: &Database,
name: &str,
version: Version, version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts, cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> { ) -> Result<Self> {
Ok(Self { Ok(Self {
all: ActivityCountVecs::forced_import(db, name, version, indexes, cached_starts)?, all: ActivityCountVecs::forced_import(db, "", version, indexes, cached_starts)?,
by_addr_type: AddrTypeToActivityCountVecs::forced_import( by_addr_type: AddrTypeToActivityCountVecs::forced_import(
db, name, version, indexes, cached_starts, db,
version,
indexes,
cached_starts,
)?, )?,
}) })
} }
@@ -284,11 +303,7 @@ impl AddrActivityVecs {
Ok(()) Ok(())
} }
pub(crate) fn compute_rest( pub(crate) fn compute_rest(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
&mut self,
max_from: Height,
exit: &Exit,
) -> Result<()> {
self.all.compute_rest(max_from, exit)?; self.all.compute_rest(max_from, exit)?;
self.by_addr_type.compute_rest(max_from, exit)?; self.by_addr_type.compute_rest(max_from, exit)?;
Ok(()) Ok(())
@@ -12,9 +12,7 @@ use vecdb::{
use crate::{indexes, internal::PerBlock}; use crate::{indexes, internal::PerBlock};
#[derive(Deref, DerefMut, Traversable)] #[derive(Deref, DerefMut, Traversable)]
pub struct AddrCountVecs<M: StorageMode = Rw>( pub struct AddrCountVecs<M: StorageMode = Rw>(#[traversable(flatten)] pub PerBlock<StoredU64, M>);
#[traversable(flatten)] pub PerBlock<StoredU64, M>,
);
impl AddrCountVecs { impl AddrCountVecs {
pub(crate) fn forced_import( pub(crate) fn forced_import(
@@ -23,9 +21,7 @@ impl AddrCountVecs {
version: Version, version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
) -> Result<Self> { ) -> Result<Self> {
Ok(Self(PerBlock::forced_import( Ok(Self(PerBlock::forced_import(db, name, version, indexes)?))
db, name, version, indexes,
)?))
} }
} }
@@ -57,42 +53,17 @@ impl From<(&AddrTypeToAddrCountVecs, Height)> for AddrTypeToAddrCount {
.collect_one(prev_height) .collect_one(prev_height)
.unwrap() .unwrap()
.into(), .into(),
p2pkh: groups p2pkh: groups.p2pkh.height.collect_one(prev_height).unwrap().into(),
.p2pkh p2sh: groups.p2sh.height.collect_one(prev_height).unwrap().into(),
.height
.collect_one(prev_height)
.unwrap()
.into(),
p2sh: groups
.p2sh
.height
.collect_one(prev_height)
.unwrap()
.into(),
p2wpkh: groups p2wpkh: groups
.p2wpkh .p2wpkh
.height .height
.collect_one(prev_height) .collect_one(prev_height)
.unwrap() .unwrap()
.into(), .into(),
p2wsh: groups p2wsh: groups.p2wsh.height.collect_one(prev_height).unwrap().into(),
.p2wsh p2tr: groups.p2tr.height.collect_one(prev_height).unwrap().into(),
.height p2a: groups.p2a.height.collect_one(prev_height).unwrap().into(),
.collect_one(prev_height)
.unwrap()
.into(),
p2tr: groups
.p2tr
.height
.collect_one(prev_height)
.unwrap()
.into(),
p2a: groups
.p2a
.height
.collect_one(prev_height)
.unwrap()
.into(),
}) })
} else { } else {
Default::default() Default::default()
@@ -177,7 +148,10 @@ impl AddrCountsVecs {
} }
pub(crate) fn min_stateful_len(&self) -> usize { pub(crate) fn min_stateful_len(&self) -> usize {
self.all.height.len().min(self.by_addr_type.min_stateful_len()) self.all
.height
.len()
.min(self.by_addr_type.min_stateful_len())
} }
pub(crate) fn par_iter_height_mut( pub(crate) fn par_iter_height_mut(
@@ -199,11 +173,7 @@ impl AddrCountsVecs {
self.by_addr_type.push_height(addr_counts); self.by_addr_type.push_height(addr_counts);
} }
pub(crate) fn compute_rest( pub(crate) fn compute_rest(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> {
&mut self,
starting_indexes: &Indexes,
exit: &Exit,
) -> Result<()> {
let sources = self.by_addr_type.by_height(); let sources = self.by_addr_type.by_height();
self.all self.all
.height .height
@@ -1,8 +1,6 @@
use brk_error::Result; use brk_error::Result;
use brk_traversable::Traversable; use brk_traversable::Traversable;
use brk_types::{ use brk_types::{EmptyAddrData, EmptyAddrIndex, FundedAddrData, FundedAddrIndex, Height};
EmptyAddrData, EmptyAddrIndex, FundedAddrData, FundedAddrIndex, Height,
};
use rayon::prelude::*; use rayon::prelude::*;
use vecdb::{AnyStoredVec, BytesVec, Rw, Stamp, StorageMode, WritableVec}; use vecdb::{AnyStoredVec, BytesVec, Rw, Stamp, StorageMode, WritableVec};
@@ -1,28 +1,24 @@
use brk_cohort::ByAddrType;
use brk_traversable::Traversable; use brk_traversable::Traversable;
use brk_types::{BasisPointsSigned32, StoredI64, StoredU64, Version}; use brk_types::{BasisPointsSigned32, StoredI64, StoredU64, Version};
use derive_more::{Deref, DerefMut};
use crate::{ use crate::{
indexes, indexes,
internal::{CachedWindowStarts, LazyRollingDeltasFromHeight}, internal::{LazyRollingDeltasFromHeight, WindowStartVec, Windows, WithAddrTypes},
}; };
use super::AddrCountsVecs; use super::AddrCountsVecs;
type AddrDelta = LazyRollingDeltasFromHeight<StoredU64, StoredI64, BasisPointsSigned32>; type AddrDelta = LazyRollingDeltasFromHeight<StoredU64, StoredI64, BasisPointsSigned32>;
#[derive(Clone, Traversable)] #[derive(Clone, Deref, DerefMut, Traversable)]
pub struct DeltaVecs { pub struct DeltaVecs(#[traversable(flatten)] pub WithAddrTypes<AddrDelta>);
pub all: AddrDelta,
#[traversable(flatten)]
pub by_addr_type: ByAddrType<AddrDelta>,
}
impl DeltaVecs { impl DeltaVecs {
pub(crate) fn new( pub(crate) fn new(
version: Version, version: Version,
addr_count: &AddrCountsVecs, addr_count: &AddrCountsVecs,
cached_starts: &CachedWindowStarts, cached_starts: &Windows<&WindowStartVec>,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
) -> Self { ) -> Self {
let version = version + Version::TWO; let version = version + Version::TWO;
@@ -45,9 +41,6 @@ impl DeltaVecs {
) )
}); });
Self { Self(WithAddrTypes { all, by_addr_type })
all,
by_addr_type,
}
} }
} }
@@ -0,0 +1,74 @@
//! Exposed address count tracking — running counters of how many addresses
//! are currently in (or have ever been in) the exposed set, per address type
//! plus an aggregated `all`. See the parent [`super`] module for the
//! definition of "exposed" and how it varies by address type.
mod state;
mod vecs;
pub use state::AddrTypeToExposedAddrCount;
pub use vecs::ExposedAddrCountAllVecs;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Indexes, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, Exit, Rw, StorageMode};
use crate::indexes;
/// Exposed address counts: funded (currently at-risk) and total (ever at-risk).
#[derive(Traversable)]
pub struct ExposedAddrCountsVecs<M: StorageMode = Rw> {
pub funded: ExposedAddrCountAllVecs<M>,
pub total: ExposedAddrCountAllVecs<M>,
}
impl ExposedAddrCountsVecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self {
funded: ExposedAddrCountAllVecs::forced_import(
db,
"exposed_addr_count",
version,
indexes,
)?,
total: ExposedAddrCountAllVecs::forced_import(
db,
"total_exposed_addr_count",
version,
indexes,
)?,
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.funded
.min_stateful_len()
.min(self.total.min_stateful_len())
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.funded
.par_iter_height_mut()
.chain(self.total.par_iter_height_mut())
}
pub(crate) fn reset_height(&mut self) -> Result<()> {
self.funded.reset_height()?;
self.total.reset_height()?;
Ok(())
}
pub(crate) fn compute_rest(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> {
self.funded.compute_rest(starting_indexes, exit)?;
self.total.compute_rest(starting_indexes, exit)?;
Ok(())
}
}
@@ -0,0 +1,42 @@
use brk_cohort::ByAddrType;
use brk_types::{Height, StoredU64};
use derive_more::{Deref, DerefMut};
use vecdb::ReadableVec;
use crate::internal::PerBlock;
use super::vecs::ExposedAddrCountAllVecs;
/// Runtime counter for exposed address counts per address type.
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddrTypeToExposedAddrCount(ByAddrType<u64>);
impl AddrTypeToExposedAddrCount {
#[inline]
pub(crate) fn sum(&self) -> u64 {
self.0.values().sum()
}
}
impl From<(&ExposedAddrCountAllVecs, Height)> for AddrTypeToExposedAddrCount {
#[inline]
fn from((vecs, starting_height): (&ExposedAddrCountAllVecs, Height)) -> Self {
if let Some(prev_height) = starting_height.decremented() {
let read = |v: &PerBlock<StoredU64>| -> u64 {
v.height.collect_one(prev_height).unwrap().into()
};
Self(ByAddrType {
p2pk65: read(&vecs.by_addr_type.p2pk65),
p2pk33: read(&vecs.by_addr_type.p2pk33),
p2pkh: read(&vecs.by_addr_type.p2pkh),
p2sh: read(&vecs.by_addr_type.p2sh),
p2wpkh: read(&vecs.by_addr_type.p2wpkh),
p2wsh: read(&vecs.by_addr_type.p2wsh),
p2tr: read(&vecs.by_addr_type.p2tr),
p2a: read(&vecs.by_addr_type.p2a),
})
} else {
Default::default()
}
}
}
@@ -0,0 +1,29 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{StoredU64, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{Database, Rw, StorageMode};
use crate::{
indexes,
internal::{PerBlock, WithAddrTypes},
};
/// Exposed address count (`all` + per-type) for a single variant (funded or total).
#[derive(Deref, DerefMut, Traversable)]
pub struct ExposedAddrCountAllVecs<M: StorageMode = Rw>(
#[traversable(flatten)] pub WithAddrTypes<PerBlock<StoredU64, M>>,
);
impl ExposedAddrCountAllVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self(WithAddrTypes::<PerBlock<StoredU64>>::forced_import(
db, name, version, indexes,
)?))
}
}
@@ -0,0 +1,134 @@
//! Exposed address tracking (quantum / pubkey-exposure sense).
//!
//! An address is "exposed" once its public key is in the blockchain. Once
//! exposed, any funds at that address are at cryptographic risk (e.g. from
//! a quantum attacker capable of recovering the private key from the pubkey).
//!
//! When the pubkey gets exposed depends on the address type:
//!
//! - **P2PK33, P2PK65, P2TR**: the pubkey (or P2TR's tweaked output key) is
//! directly in the locking script of the funding output. These addresses are
//! exposed the moment they receive any funds.
//! - **P2PKH, P2SH, P2WPKH, P2WSH**: the locking script contains a hash of
//! the pubkey/script. The pubkey is only revealed when spending. Note that
//! even the spending tx itself exposes the pubkey while the address still
//! holds funds — during the mempool window between broadcast and confirmation,
//! the pubkey is visible while the UTXO being spent is still unspent on-chain.
//! So every spent address of these types has had at least one moment with
//! funds at quantum risk.
//! - **P2A**: anyone-can-spend, no pubkey at all. Excluded from both counters.
//!
//! Formally, with `is_funding_exposed` = `output_type.pubkey_exposed_at_funding()`:
//! - `funded` (count): `(utxo_count > 0) AND (is_funding_exposed OR spent_txo_count >= 1)`
//! - `total` (count): `(is_funding_exposed AND ever received) OR spent_txo_count >= 1`
//! - `supply` (sats): sum of balances of addresses currently in the funded set
//!
//! For P2PK/P2TR types this means `total ≡ total_addr_count` and
//! `funded ≡ funded_addr_count` (every address of those types is exposed by
//! virtue of existing). For P2PKH/P2SH/P2WPKH/P2WSH it's the strict subset of
//! addresses that have been spent from. The aggregate `all` exposed counter
//! sums these, giving "Bitcoin addresses currently with funds at quantum risk".
//!
//! All metrics are tracked as running counters and require no extra fields
//! on the address data — they're maintained via delta detection in
//! `process_received` and `process_sent`.
mod count;
mod supply;
pub use count::{AddrTypeToExposedAddrCount, ExposedAddrCountsVecs};
pub use supply::{AddrTypeToExposedSupply, ExposedAddrSupplyVecs, ExposedSupplyShareVecs};
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, Indexes, Sats, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, Exit, ReadableVec, Rw, StorageMode};
use crate::{indexes, internal::RatioSatsBp16, prices};
/// Top-level container for all exposed address tracking: counts (funded +
/// total), the funded supply, and share of supply.
#[derive(Traversable)]
pub struct ExposedAddrVecs<M: StorageMode = Rw> {
pub count: ExposedAddrCountsVecs<M>,
pub supply: ExposedAddrSupplyVecs<M>,
#[traversable(wrap = "supply", rename = "share")]
pub supply_share: ExposedSupplyShareVecs<M>,
}
impl ExposedAddrVecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self {
count: ExposedAddrCountsVecs::forced_import(db, version, indexes)?,
supply: ExposedAddrSupplyVecs::forced_import(db, version, indexes)?,
supply_share: ExposedSupplyShareVecs::forced_import(db, version, indexes)?,
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.count
.min_stateful_len()
.min(self.supply.min_stateful_len())
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.count
.par_iter_height_mut()
.chain(self.supply.par_iter_height_mut())
}
pub(crate) fn reset_height(&mut self) -> Result<()> {
self.count.reset_height()?;
self.supply.reset_height()?;
self.supply_share.reset_height()?;
Ok(())
}
pub(crate) fn compute_rest(
&mut self,
starting_indexes: &Indexes,
prices: &prices::Vecs,
all_supply_sats: &impl ReadableVec<Height, Sats>,
type_supply_sats: &ByAddrType<&impl ReadableVec<Height, Sats>>,
exit: &Exit,
) -> Result<()> {
self.count.compute_rest(starting_indexes, exit)?;
self.supply
.compute_rest(starting_indexes.height, prices, exit)?;
let max_from = starting_indexes.height;
self.supply_share
.all
.compute_binary::<Sats, Sats, RatioSatsBp16>(
max_from,
&self.supply.all.sats.height,
all_supply_sats,
exit,
)?;
for ((_, share), ((_, exposed), (_, denom))) in self
.supply_share
.by_addr_type
.iter_mut()
.zip(self.supply.by_addr_type.iter().zip(type_supply_sats.iter()))
{
share.compute_binary::<Sats, Sats, RatioSatsBp16>(
max_from,
&exposed.sats.height,
*denom,
exit,
)?;
}
Ok(())
}
}
@@ -0,0 +1,12 @@
//! Exposed address supply (sats) tracking — running sum of balances held by
//! addresses currently in the funded exposed set, per address type plus an
//! aggregated `all`. See the parent [`super`] module for the definition of
//! "exposed" and how it varies by address type.
mod share;
mod state;
mod vecs;
pub use share::ExposedSupplyShareVecs;
pub use state::AddrTypeToExposedSupply;
pub use vecs::ExposedAddrSupplyVecs;
@@ -0,0 +1,36 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{BasisPoints16, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{Database, Rw, StorageMode};
use crate::{
indexes,
internal::{PercentPerBlock, WithAddrTypes},
};
/// Share of exposed supply relative to total supply.
///
/// - `all`: exposed_supply / circulating_supply
/// - Per-type: type's exposed_supply / type's total supply
#[derive(Deref, DerefMut, Traversable)]
pub struct ExposedSupplyShareVecs<M: StorageMode = Rw>(
#[traversable(flatten)] pub WithAddrTypes<PercentPerBlock<BasisPoints16, M>>,
);
impl ExposedSupplyShareVecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self(
WithAddrTypes::<PercentPerBlock<BasisPoints16>>::forced_import(
db,
"exposed_supply_share",
version,
indexes,
)?,
))
}
}
@@ -0,0 +1,42 @@
use brk_cohort::ByAddrType;
use brk_types::{Height, Sats};
use derive_more::{Deref, DerefMut};
use vecdb::ReadableVec;
use crate::internal::ValuePerBlock;
use super::vecs::ExposedAddrSupplyVecs;
/// Runtime running counter for the total balance (sats) held by funded
/// exposed addresses, per address type.
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddrTypeToExposedSupply(ByAddrType<Sats>);
impl AddrTypeToExposedSupply {
#[inline]
pub(crate) fn sum(&self) -> Sats {
self.0.values().copied().sum()
}
}
impl From<(&ExposedAddrSupplyVecs, Height)> for AddrTypeToExposedSupply {
#[inline]
fn from((vecs, starting_height): (&ExposedAddrSupplyVecs, Height)) -> Self {
if let Some(prev_height) = starting_height.decremented() {
let read =
|v: &ValuePerBlock| -> Sats { v.sats.height.collect_one(prev_height).unwrap() };
Self(ByAddrType {
p2pk65: read(&vecs.by_addr_type.p2pk65),
p2pk33: read(&vecs.by_addr_type.p2pk33),
p2pkh: read(&vecs.by_addr_type.p2pkh),
p2sh: read(&vecs.by_addr_type.p2sh),
p2wpkh: read(&vecs.by_addr_type.p2wpkh),
p2wsh: read(&vecs.by_addr_type.p2wsh),
p2tr: read(&vecs.by_addr_type.p2tr),
p2a: read(&vecs.by_addr_type.p2a),
})
} else {
Default::default()
}
}
}
@@ -0,0 +1,34 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::Version;
use derive_more::{Deref, DerefMut};
use vecdb::{Database, Rw, StorageMode};
use crate::{
indexes,
internal::{ValuePerBlock, WithAddrTypes},
};
/// Exposed address supply (sats/btc/cents/usd) — `all` + per-address-type.
/// Tracks the total balance held by addresses currently in the funded
/// exposed set. Sats are pushed stateful per block; cents/usd are derived
/// post-hoc from sats × spot price.
#[derive(Deref, DerefMut, Traversable)]
pub struct ExposedAddrSupplyVecs<M: StorageMode = Rw>(
#[traversable(flatten)] pub WithAddrTypes<ValuePerBlock<M>>,
);
impl ExposedAddrSupplyVecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self(WithAddrTypes::<ValuePerBlock>::forced_import(
db,
"exposed_supply",
version,
indexes,
)?))
}
}
@@ -5,8 +5,8 @@ use brk_error::{Error, Result};
use brk_traversable::Traversable; use brk_traversable::Traversable;
use brk_types::{ use brk_types::{
AnyAddrIndex, Height, OutputType, P2AAddrIndex, P2PK33AddrIndex, P2PK65AddrIndex, AnyAddrIndex, Height, OutputType, P2AAddrIndex, P2PK33AddrIndex, P2PK65AddrIndex,
P2PKHAddrIndex, P2SHAddrIndex, P2TRAddrIndex, P2WPKHAddrIndex, P2WSHAddrIndex, P2PKHAddrIndex, P2SHAddrIndex, P2TRAddrIndex, P2WPKHAddrIndex, P2WSHAddrIndex, TypeIndex,
TypeIndex, Version, Version,
}; };
use rayon::prelude::*; use rayon::prelude::*;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
@@ -2,8 +2,10 @@ mod activity;
mod addr_count; mod addr_count;
mod data; mod data;
mod delta; mod delta;
mod exposed;
mod indexes; mod indexes;
mod new_addr_count; mod new_addr_count;
mod reused;
mod total_addr_count; mod total_addr_count;
mod type_map; mod type_map;
@@ -11,7 +13,9 @@ pub use activity::{AddrActivityVecs, AddrTypeToActivityCounts};
pub use addr_count::{AddrCountsVecs, AddrTypeToAddrCount}; pub use addr_count::{AddrCountsVecs, AddrTypeToAddrCount};
pub use data::AddrsDataVecs; pub use data::AddrsDataVecs;
pub use delta::DeltaVecs; pub use delta::DeltaVecs;
pub use exposed::{AddrTypeToExposedAddrCount, AddrTypeToExposedSupply, ExposedAddrVecs,};
pub use indexes::AnyAddrIndexesVecs; pub use indexes::AnyAddrIndexesVecs;
pub use new_addr_count::NewAddrCountVecs; pub use new_addr_count::NewAddrCountVecs;
pub use reused::{AddrTypeToReusedAddrCount, AddrTypeToReusedAddrEventCount, ReusedAddrVecs};
pub use total_addr_count::TotalAddrCountVecs; pub use total_addr_count::TotalAddrCountVecs;
pub use type_map::{AddrTypeToTypeIndexMap, AddrTypeToVec, HeightToAddrTypeToVec}; pub use type_map::{AddrTypeToTypeIndexMap, AddrTypeToVec, HeightToAddrTypeToVec};
@@ -1,53 +1,35 @@
use brk_cohort::ByAddrType;
use brk_error::Result; use brk_error::Result;
use brk_traversable::Traversable; use brk_traversable::Traversable;
use brk_types::{Height, StoredU64, Version}; use brk_types::{Height, StoredU64, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{Database, Exit, Rw, StorageMode}; use vecdb::{Database, Exit, Rw, StorageMode};
use crate::{ use crate::{
indexes, indexes,
internal::{CachedWindowStarts, PerBlockCumulativeRolling}, internal::{PerBlockCumulativeRolling, WindowStartVec, Windows, WithAddrTypes},
}; };
use super::TotalAddrCountVecs; use super::TotalAddrCountVecs;
/// New address count per block (global + per-type) /// New address count per block (global + per-type).
#[derive(Traversable)] #[derive(Deref, DerefMut, Traversable)]
pub struct NewAddrCountVecs<M: StorageMode = Rw> { pub struct NewAddrCountVecs<M: StorageMode = Rw>(
pub all: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
#[traversable(flatten)] #[traversable(flatten)]
pub by_addr_type: ByAddrType<PerBlockCumulativeRolling<StoredU64, StoredU64, M>>, pub WithAddrTypes<PerBlockCumulativeRolling<StoredU64, StoredU64, M>>,
} );
impl NewAddrCountVecs { impl NewAddrCountVecs {
pub(crate) fn forced_import( pub(crate) fn forced_import(
db: &Database, db: &Database,
version: Version, version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
cached_starts: &CachedWindowStarts, cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> { ) -> Result<Self> {
let all = PerBlockCumulativeRolling::forced_import( Ok(Self(WithAddrTypes::<
db, PerBlockCumulativeRolling<StoredU64, StoredU64>,
"new_addr_count", >::forced_import(
version, db, "new_addr_count", version, indexes, cached_starts
indexes, )?))
cached_starts,
)?;
let by_addr_type = ByAddrType::new_with_name(|name| {
PerBlockCumulativeRolling::forced_import(
db,
&format!("{name}_new_addr_count"),
version,
indexes,
cached_starts,
)
})?;
Ok(Self {
all,
by_addr_type,
})
} }
pub(crate) fn compute( pub(crate) fn compute(
@@ -56,11 +38,12 @@ impl NewAddrCountVecs {
total_addr_count: &TotalAddrCountVecs, total_addr_count: &TotalAddrCountVecs,
exit: &Exit, exit: &Exit,
) -> Result<()> { ) -> Result<()> {
self.all.compute(max_from, exit, |height_vec| { self.0.all.compute(max_from, exit, |height_vec| {
Ok(height_vec.compute_change(max_from, &total_addr_count.all.height, 1, exit)?) Ok(height_vec.compute_change(max_from, &total_addr_count.all.height, 1, exit)?)
})?; })?;
for ((_, new), (_, total)) in self for ((_, new), (_, total)) in self
.0
.by_addr_type .by_addr_type
.iter_mut() .iter_mut()
.zip(total_addr_count.by_addr_type.iter()) .zip(total_addr_count.by_addr_type.iter())
@@ -0,0 +1,78 @@
//! Reused address count tracking — running counters of how many addresses
//! are currently in (or have ever been in) the reused set, per address type
//! plus an aggregated `all`. See the parent [`super`] module for the
//! definition of "reused".
//!
//! Two counters are exposed:
//! - `funded`: addresses currently funded AND with `funded_txo_count > 1`
//! - `total`: addresses that have ever satisfied `funded_txo_count > 1` (monotonic)
mod state;
mod vecs;
pub use state::AddrTypeToReusedAddrCount;
pub use vecs::ReusedAddrCountAllVecs;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Indexes, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, Exit, Rw, StorageMode};
use crate::indexes;
/// Reused address counts: funded (currently with balance) and total (ever reused).
#[derive(Traversable)]
pub struct ReusedAddrCountsVecs<M: StorageMode = Rw> {
pub funded: ReusedAddrCountAllVecs<M>,
pub total: ReusedAddrCountAllVecs<M>,
}
impl ReusedAddrCountsVecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self {
funded: ReusedAddrCountAllVecs::forced_import(
db,
"reused_addr_count",
version,
indexes,
)?,
total: ReusedAddrCountAllVecs::forced_import(
db,
"total_reused_addr_count",
version,
indexes,
)?,
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.funded
.min_stateful_len()
.min(self.total.min_stateful_len())
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.funded
.par_iter_height_mut()
.chain(self.total.par_iter_height_mut())
}
pub(crate) fn reset_height(&mut self) -> Result<()> {
self.funded.reset_height()?;
self.total.reset_height()?;
Ok(())
}
pub(crate) fn compute_rest(&mut self, starting_indexes: &Indexes, exit: &Exit) -> Result<()> {
self.funded.compute_rest(starting_indexes, exit)?;
self.total.compute_rest(starting_indexes, exit)?;
Ok(())
}
}
@@ -0,0 +1,42 @@
use brk_cohort::ByAddrType;
use brk_types::{Height, StoredU64};
use derive_more::{Deref, DerefMut};
use vecdb::ReadableVec;
use crate::internal::PerBlock;
use super::vecs::ReusedAddrCountAllVecs;
/// Runtime counter for reused address counts per address type.
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddrTypeToReusedAddrCount(ByAddrType<u64>);
impl AddrTypeToReusedAddrCount {
#[inline]
pub(crate) fn sum(&self) -> u64 {
self.0.values().sum()
}
}
impl From<(&ReusedAddrCountAllVecs, Height)> for AddrTypeToReusedAddrCount {
#[inline]
fn from((vecs, starting_height): (&ReusedAddrCountAllVecs, Height)) -> Self {
if let Some(prev_height) = starting_height.decremented() {
let read = |v: &PerBlock<StoredU64>| -> u64 {
v.height.collect_one(prev_height).unwrap().into()
};
Self(ByAddrType {
p2pk65: read(&vecs.by_addr_type.p2pk65),
p2pk33: read(&vecs.by_addr_type.p2pk33),
p2pkh: read(&vecs.by_addr_type.p2pkh),
p2sh: read(&vecs.by_addr_type.p2sh),
p2wpkh: read(&vecs.by_addr_type.p2wpkh),
p2wsh: read(&vecs.by_addr_type.p2wsh),
p2tr: read(&vecs.by_addr_type.p2tr),
p2a: read(&vecs.by_addr_type.p2a),
})
} else {
Default::default()
}
}
}
@@ -0,0 +1,29 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{StoredU64, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{Database, Rw, StorageMode};
use crate::{
indexes,
internal::{PerBlock, WithAddrTypes},
};
/// Reused address count (`all` + per-type) for a single variant (funded or total).
#[derive(Deref, DerefMut, Traversable)]
pub struct ReusedAddrCountAllVecs<M: StorageMode = Rw>(
#[traversable(flatten)] pub WithAddrTypes<PerBlock<StoredU64, M>>,
);
impl ReusedAddrCountAllVecs {
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self(WithAddrTypes::<PerBlock<StoredU64>>::forced_import(
db, name, version, indexes,
)?))
}
}
@@ -0,0 +1,11 @@
//! Per-block reused-address event tracking. Holds both the output-side
//! ("an output landed on a previously-used address") and input-side
//! ("an input spent from an address in the reused set") event counters.
//! See [`vecs::ReusedAddrEventsVecs`] for the full description of each
//! metric.
mod state;
mod vecs;
pub use state::AddrTypeToReusedAddrEventCount;
pub use vecs::ReusedAddrEventsVecs;
@@ -0,0 +1,28 @@
use brk_cohort::ByAddrType;
use derive_more::{Deref, DerefMut};
/// Per-block running counter of reused-address events, per address type.
/// Shared runtime container for both output-side events
/// (`output_to_reused_addr_count`, outputs landing on addresses that
/// had already received ≥ 1 prior output) and input-side events
/// (`input_from_reused_addr_count`, inputs spending from addresses
/// with lifetime `funded_txo_count > 1`). Reset at the start of each
/// block (no disk recovery needed since per-block flow is
/// reconstructed deterministically from `process_received` /
/// `process_sent`).
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddrTypeToReusedAddrEventCount(ByAddrType<u64>);
impl AddrTypeToReusedAddrEventCount {
#[inline]
pub(crate) fn sum(&self) -> u64 {
self.0.values().sum()
}
#[inline]
pub(crate) fn reset(&mut self) {
for v in self.0.values_mut() {
*v = 0;
}
}
}
@@ -0,0 +1,261 @@
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{BasisPoints16, Indexes, OutputType, StoredF32, StoredU32, StoredU64, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, AnyVec, Database, Exit, Rw, StorageMode, WritableVec};
use crate::{
indexes, inputs,
internal::{
PerBlockCumulativeRolling, PerBlockRollingAverage, PercentCumulativeRolling,
WindowStartVec, Windows, WithAddrTypes,
},
outputs,
};
use super::state::AddrTypeToReusedAddrEventCount;
/// Per-block reused-address event metrics. Holds three families of
/// signals: output-level (use), input-level (spend), and address-level
/// (active in block).
///
/// `output_to_reused_addr_count`: every output landing on an address that had
/// already received at least one prior output anywhere in its lifetime,
/// i.e. an output-level reuse event. Outputs are not deduplicated per
/// address within a block: an address receiving N outputs in one block
/// that had `before` lifetime outputs contributes
/// `max(0, N - max(0, 1 - before))` events. Only the very first output
/// an address ever sees is excluded. Every subsequent output counts,
/// matching the standard "% of outputs to previously-used addresses"
/// reuse ratio reported by external sources. `output_to_reused_addr_share`
/// uses `outputs::ByTypeVecs::output_count` (all 12 output types) as
/// denominator. `spendable_output_to_reused_addr_share` uses the
/// op_return-excluded 11-type aggregate (`spendable_output_count`).
///
/// `input_from_reused_addr_count`: every input spending from an address
/// whose lifetime `funded_txo_count > 1` at the time of the spend (i.e.
/// the address is in the same reused set tracked by
/// `reused_addr_count`). Every input is checked independently. If a
/// single address has multiple inputs in one block each one counts.
/// This is a *stable-predicate* signal about the sending address, not
/// an output-level repeat event: the first spend from a reused address
/// counts just as much as the tenth. Denominator
/// (`input_from_reused_addr_share`): `inputs::ByTypeVecs::input_count` (11
/// spendable types, where `p2ms`, `unknown`, `empty` count as true
/// negatives).
///
/// `active_reused_addr_count` / `active_reused_addr_share`: block-level
/// *address* signals (single aggregate, not per-type).
/// `active_reused_addr_count` is the count of distinct addresses
/// involved in this block (sent received) that satisfy `is_reused()`
/// after the block's events, populated inline in `process_received`
/// (each receiver, post-receive) and in `process_sent` (each
/// first-encounter sender, deduped against `received_addrs` so
/// addresses that did both aren't double-counted).
/// `active_reused_addr_share` is the per-block ratio
/// `reused / active * 100` as a percentage in `[0, 100]` (or `0.0` for
/// empty blocks). The denominator (distinct active addrs per block)
/// lives on `ActivityCountVecs::active` (`addrs.activity.all.active`),
/// derived from `sending + receiving - bidirectional`. Both fields
/// use `PerBlockRollingAverage` so their lazy 24h/1w/1m/1y series are
/// rolling *averages* of the per-block values. Sums and cumulatives of
/// distinct-address counts would be misleading because the same
/// address can appear in multiple blocks.
#[derive(Traversable)]
pub struct ReusedAddrEventsVecs<M: StorageMode = Rw> {
pub output_to_reused_addr_count:
WithAddrTypes<PerBlockCumulativeRolling<StoredU64, StoredU64, M>>,
pub output_to_reused_addr_share: WithAddrTypes<PercentCumulativeRolling<BasisPoints16, M>>,
pub spendable_output_to_reused_addr_share: PercentCumulativeRolling<BasisPoints16, M>,
pub input_from_reused_addr_count:
WithAddrTypes<PerBlockCumulativeRolling<StoredU64, StoredU64, M>>,
pub input_from_reused_addr_share: WithAddrTypes<PercentCumulativeRolling<BasisPoints16, M>>,
pub active_reused_addr_count: PerBlockRollingAverage<StoredU32, StoredU64, M>,
pub active_reused_addr_share: PerBlockRollingAverage<StoredF32, StoredF32, M>,
}
impl ReusedAddrEventsVecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
let import_count = |name: &str| {
WithAddrTypes::<PerBlockCumulativeRolling<StoredU64, StoredU64>>::forced_import(
db,
name,
version,
indexes,
cached_starts,
)
};
let import_percent = |name: &str| -> Result<WithAddrTypes<
PercentCumulativeRolling<BasisPoints16>,
>> {
Ok(WithAddrTypes {
all: PercentCumulativeRolling::forced_import(db, name, version, indexes)?,
by_addr_type: ByAddrType::new_with_name(|type_name| {
PercentCumulativeRolling::forced_import(
db,
&format!("{type_name}_{name}"),
version,
indexes,
)
})?,
})
};
let output_to_reused_addr_count = import_count("output_to_reused_addr_count")?;
let output_to_reused_addr_share = import_percent("output_to_reused_addr_share")?;
let spendable_output_to_reused_addr_share = PercentCumulativeRolling::forced_import(
db,
"spendable_output_to_reused_addr_share",
version,
indexes,
)?;
let input_from_reused_addr_count = import_count("input_from_reused_addr_count")?;
let input_from_reused_addr_share = import_percent("input_from_reused_addr_share")?;
let active_reused_addr_count = PerBlockRollingAverage::forced_import(
db,
"active_reused_addr_count",
version,
indexes,
cached_starts,
)?;
let active_reused_addr_share = PerBlockRollingAverage::forced_import(
db,
"active_reused_addr_share",
version,
indexes,
cached_starts,
)?;
Ok(Self {
output_to_reused_addr_count,
output_to_reused_addr_share,
spendable_output_to_reused_addr_share,
input_from_reused_addr_count,
input_from_reused_addr_share,
active_reused_addr_count,
active_reused_addr_share,
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.output_to_reused_addr_count
.min_stateful_len()
.min(self.input_from_reused_addr_count.min_stateful_len())
.min(self.active_reused_addr_count.block.len())
.min(self.active_reused_addr_share.block.len())
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.output_to_reused_addr_count
.par_iter_height_mut()
.chain(self.input_from_reused_addr_count.par_iter_height_mut())
.chain([
&mut self.active_reused_addr_count.block as &mut dyn AnyStoredVec,
&mut self.active_reused_addr_share.block as &mut dyn AnyStoredVec,
])
}
pub(crate) fn reset_height(&mut self) -> Result<()> {
self.output_to_reused_addr_count.reset_height()?;
self.input_from_reused_addr_count.reset_height()?;
self.active_reused_addr_count.block.reset()?;
self.active_reused_addr_share.block.reset()?;
Ok(())
}
#[inline(always)]
pub(crate) fn push_height(
&mut self,
uses: &AddrTypeToReusedAddrEventCount,
spends: &AddrTypeToReusedAddrEventCount,
active_addr_count: u32,
active_reused_addr_count: u32,
) {
self.output_to_reused_addr_count
.push_height(uses.sum(), uses.values().copied());
self.input_from_reused_addr_count
.push_height(spends.sum(), spends.values().copied());
self.active_reused_addr_count
.block
.push(StoredU32::from(active_reused_addr_count));
// Stored as a percentage in [0, 100] to match the rest of the
// codebase (Unit.percentage on the website expects 0..100). The
// `active_addr_count` denominator lives on `ActivityCountVecs`
// (`addrs.activity.all.active`), passed in here so we can
// compute the per-block ratio inline.
let share = if active_addr_count > 0 {
100.0 * (active_reused_addr_count as f32 / active_addr_count as f32)
} else {
0.0
};
self.active_reused_addr_share
.block
.push(StoredF32::from(share));
}
pub(crate) fn compute_rest(
&mut self,
starting_indexes: &Indexes,
outputs_by_type: &outputs::ByTypeVecs,
inputs_by_type: &inputs::ByTypeVecs,
exit: &Exit,
) -> Result<()> {
self.output_to_reused_addr_count
.compute_rest(starting_indexes.height, exit)?;
self.input_from_reused_addr_count
.compute_rest(starting_indexes.height, exit)?;
self.active_reused_addr_count
.compute_rest(starting_indexes.height, exit)?;
self.active_reused_addr_share
.compute_rest(starting_indexes.height, exit)?;
self.output_to_reused_addr_share.all.compute_count_ratio(
&self.output_to_reused_addr_count.all,
&outputs_by_type.output_count.all,
starting_indexes.height,
exit,
)?;
self.spendable_output_to_reused_addr_share.compute_count_ratio(
&self.output_to_reused_addr_count.all,
&outputs_by_type.spendable_output_count,
starting_indexes.height,
exit,
)?;
self.input_from_reused_addr_share.all.compute_count_ratio(
&self.input_from_reused_addr_count.all,
&inputs_by_type.input_count.all,
starting_indexes.height,
exit,
)?;
for otype in OutputType::ADDR_TYPES {
self.output_to_reused_addr_share
.by_addr_type
.get_mut_unwrap(otype)
.compute_count_ratio(
self.output_to_reused_addr_count.by_addr_type.get_unwrap(otype),
outputs_by_type.output_count.by_type.get(otype),
starting_indexes.height,
exit,
)?;
self.input_from_reused_addr_share
.by_addr_type
.get_mut_unwrap(otype)
.compute_count_ratio(
self.input_from_reused_addr_count.by_addr_type.get_unwrap(otype),
inputs_by_type.input_count.by_type.get(otype),
starting_indexes.height,
exit,
)?;
}
Ok(())
}
}
@@ -0,0 +1,94 @@
//! Reused address tracking.
//!
//! An address is "reused" if its lifetime `funded_txo_count > 1`, i.e.
//! it has received more than one output across its lifetime. This is
//! the simplest output-multiplicity proxy for address linkability.
//!
//! Two facets are tracked here:
//! - [`count`]: how many distinct addresses are currently reused
//! (funded) and how many have *ever* been reused (total). Per address
//! type plus an aggregated `all`.
//! - [`events`]: per-block address-reuse event counts on both sides.
//! Output-side (`output_to_reused_addr_count`, outputs landing on
//! addresses that had already received ≥ 1 prior output) and
//! input-side (`input_from_reused_addr_count`, inputs spending from
//! addresses with lifetime `funded_txo_count > 1`). Each count is
//! paired with a percent over the matching block-level output/input
//! total.
mod count;
mod events;
pub use count::{AddrTypeToReusedAddrCount, ReusedAddrCountsVecs};
pub use events::{AddrTypeToReusedAddrEventCount, ReusedAddrEventsVecs};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Indexes, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, Exit, Rw, StorageMode};
use crate::{
indexes, inputs,
internal::{WindowStartVec, Windows},
outputs,
};
/// Top-level container for all reused address tracking: counts (funded +
/// total) plus per-block reuse events (output-side + input-side).
#[derive(Traversable)]
pub struct ReusedAddrVecs<M: StorageMode = Rw> {
pub count: ReusedAddrCountsVecs<M>,
pub events: ReusedAddrEventsVecs<M>,
}
impl ReusedAddrVecs {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
cached_starts: &Windows<&WindowStartVec>,
) -> Result<Self> {
Ok(Self {
count: ReusedAddrCountsVecs::forced_import(db, version, indexes)?,
events: ReusedAddrEventsVecs::forced_import(db, version, indexes, cached_starts)?,
})
}
pub(crate) fn min_stateful_len(&self) -> usize {
self.count
.min_stateful_len()
.min(self.events.min_stateful_len())
}
pub(crate) fn par_iter_height_mut(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
self.count
.par_iter_height_mut()
.chain(self.events.par_iter_height_mut())
}
pub(crate) fn reset_height(&mut self) -> Result<()> {
self.count.reset_height()?;
self.events.reset_height()?;
Ok(())
}
pub(crate) fn compute_rest(
&mut self,
starting_indexes: &Indexes,
outputs_by_type: &outputs::ByTypeVecs,
inputs_by_type: &inputs::ByTypeVecs,
exit: &Exit,
) -> Result<()> {
self.count.compute_rest(starting_indexes, exit)?;
self.events.compute_rest(
starting_indexes,
outputs_by_type,
inputs_by_type,
exit,
)?;
Ok(())
}
}
@@ -1,20 +1,21 @@
use brk_cohort::ByAddrType;
use brk_error::Result; use brk_error::Result;
use brk_traversable::Traversable; use brk_traversable::Traversable;
use brk_types::{Height, StoredU64, Version}; use brk_types::{Height, StoredU64, Version};
use derive_more::{Deref, DerefMut};
use vecdb::{Database, Exit, Rw, StorageMode}; use vecdb::{Database, Exit, Rw, StorageMode};
use crate::{indexes, internal::PerBlock}; use crate::{
indexes,
internal::{PerBlock, WithAddrTypes},
};
use super::AddrCountsVecs; use super::AddrCountsVecs;
/// Total address count (global + per-type) with all derived indexes /// Total address count (global + per-type) with all derived indexes.
#[derive(Traversable)] #[derive(Deref, DerefMut, Traversable)]
pub struct TotalAddrCountVecs<M: StorageMode = Rw> { pub struct TotalAddrCountVecs<M: StorageMode = Rw>(
pub all: PerBlock<StoredU64, M>, #[traversable(flatten)] pub WithAddrTypes<PerBlock<StoredU64, M>>,
#[traversable(flatten)] );
pub by_addr_type: ByAddrType<PerBlock<StoredU64, M>>,
}
impl TotalAddrCountVecs { impl TotalAddrCountVecs {
pub(crate) fn forced_import( pub(crate) fn forced_import(
@@ -22,22 +23,12 @@ impl TotalAddrCountVecs {
version: Version, version: Version,
indexes: &indexes::Vecs, indexes: &indexes::Vecs,
) -> Result<Self> { ) -> Result<Self> {
let all = PerBlock::forced_import(db, "total_addr_count", version, indexes)?; Ok(Self(WithAddrTypes::<PerBlock<StoredU64>>::forced_import(
db,
let by_addr_type: ByAddrType<PerBlock<StoredU64>> = "total_addr_count",
ByAddrType::new_with_name(|name| { version,
PerBlock::forced_import( indexes,
db, )?))
&format!("{name}_total_addr_count"),
version,
indexes,
)
})?;
Ok(Self {
all,
by_addr_type,
})
} }
/// Eagerly compute total = addr_count + empty_addr_count. /// Eagerly compute total = addr_count + empty_addr_count.
@@ -48,14 +39,14 @@ impl TotalAddrCountVecs {
empty_addr_count: &AddrCountsVecs, empty_addr_count: &AddrCountsVecs,
exit: &Exit, exit: &Exit,
) -> Result<()> { ) -> Result<()> {
self.all.height.compute_add( self.0.all.height.compute_add(
max_from, max_from,
&addr_count.all.height, &addr_count.all.height,
&empty_addr_count.all.height, &empty_addr_count.all.height,
exit, exit,
)?; )?;
for ((_, total), ((_, addr), (_, empty))) in self.by_addr_type.iter_mut().zip( for ((_, total), ((_, addr), (_, empty))) in self.0.by_addr_type.iter_mut().zip(
addr_count addr_count
.by_addr_type .by_addr_type
.iter() .iter()
+1 -3
View File
@@ -103,9 +103,7 @@ pub(crate) fn load_uncached_addr_data(
// Check if this is a new address (type_index >= first for this height) // Check if this is a new address (type_index >= first for this height)
let first = *first_addr_indexes.get(addr_type).unwrap(); let first = *first_addr_indexes.get(addr_type).unwrap();
if first <= type_index { if first <= type_index {
return Ok(Some(WithAddrDataSource::New( return Ok(Some(WithAddrDataSource::New(FundedAddrData::default())));
FundedAddrData::default(),
)));
} }
// Skip if already in cache // Skip if already in cache
+1 -4
View File
@@ -26,10 +26,7 @@ impl<'a> AddrLookup<'a> {
&mut self, &mut self,
output_type: OutputType, output_type: OutputType,
type_index: TypeIndex, type_index: TypeIndex,
) -> ( ) -> (&mut WithAddrDataSource<FundedAddrData>, TrackingStatus) {
&mut WithAddrDataSource<FundedAddrData>,
TrackingStatus,
) {
use std::collections::hash_map::Entry; use std::collections::hash_map::Entry;
let map = self.funded.get_mut(output_type).unwrap(); let map = self.funded.get_mut(output_type).unwrap();
@@ -1,7 +1,7 @@
use brk_error::Result; use brk_error::Result;
use brk_types::{ use brk_types::{
AnyAddrIndex, EmptyAddrData, EmptyAddrIndex, FundedAddrData, FundedAddrIndex, AnyAddrIndex, EmptyAddrData, EmptyAddrIndex, FundedAddrData, FundedAddrIndex, OutputType,
OutputType, TypeIndex, TypeIndex,
}; };
use vecdb::AnyVec; use vecdb::AnyVec;
@@ -67,14 +67,14 @@ pub(crate) fn process_funded_addrs(
// Pure pushes - no holes remain // Pure pushes - no holes remain
addrs_data.funded.reserve_pushed(pushes_iter.len()); addrs_data.funded.reserve_pushed(pushes_iter.len());
let mut next_index = addrs_data.funded.len(); for (next_index, (addr_type, type_index, data)) in
for (addr_type, type_index, data) in pushes_iter { (addrs_data.funded.len()..).zip(pushes_iter)
{
addrs_data.funded.push(data); addrs_data.funded.push(data);
result.get_mut(addr_type).unwrap().insert( result.get_mut(addr_type).unwrap().insert(
type_index, type_index,
AnyAddrIndex::from(FundedAddrIndex::from(next_index)), AnyAddrIndex::from(FundedAddrIndex::from(next_index)),
); );
next_index += 1;
} }
Ok(result) Ok(result)
@@ -138,14 +138,14 @@ pub(crate) fn process_empty_addrs(
// Pure pushes - no holes remain // Pure pushes - no holes remain
addrs_data.empty.reserve_pushed(pushes_iter.len()); addrs_data.empty.reserve_pushed(pushes_iter.len());
let mut next_index = addrs_data.empty.len(); for (next_index, (addr_type, type_index, data)) in
for (addr_type, type_index, data) in pushes_iter { (addrs_data.empty.len()..).zip(pushes_iter)
{
addrs_data.empty.push(data); addrs_data.empty.push(data);
result.get_mut(addr_type).unwrap().insert( result.get_mut(addr_type).unwrap().insert(
type_index, type_index,
AnyAddrIndex::from(EmptyAddrIndex::from(next_index)), AnyAddrIndex::from(EmptyAddrIndex::from(next_index)),
); );
next_index += 1;
} }
Ok(result) Ok(result)
@@ -3,7 +3,10 @@ use brk_types::{Cents, Sats, TypeIndex};
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use crate::distribution::{ use crate::distribution::{
addr::{AddrTypeToActivityCounts, AddrTypeToVec}, addr::{
AddrTypeToActivityCounts, AddrTypeToExposedAddrCount, AddrTypeToExposedSupply,
AddrTypeToReusedAddrCount, AddrTypeToReusedAddrEventCount, AddrTypeToVec,
},
cohorts::AddrCohorts, cohorts::AddrCohorts,
}; };
@@ -25,8 +28,19 @@ pub(crate) fn process_received(
addr_count: &mut ByAddrType<u64>, addr_count: &mut ByAddrType<u64>,
empty_addr_count: &mut ByAddrType<u64>, empty_addr_count: &mut ByAddrType<u64>,
activity_counts: &mut AddrTypeToActivityCounts, activity_counts: &mut AddrTypeToActivityCounts,
reused_addr_count: &mut AddrTypeToReusedAddrCount,
total_reused_addr_count: &mut AddrTypeToReusedAddrCount,
output_to_reused_addr_count: &mut AddrTypeToReusedAddrEventCount,
active_reused_addr_count: &mut AddrTypeToReusedAddrEventCount,
exposed_addr_count: &mut AddrTypeToExposedAddrCount,
total_exposed_addr_count: &mut AddrTypeToExposedAddrCount,
exposed_supply: &mut AddrTypeToExposedSupply,
) { ) {
let max_type_len = received_data.iter().map(|(_, v)| v.len()).max().unwrap_or(0); let max_type_len = received_data
.iter()
.map(|(_, v)| v.len())
.max()
.unwrap_or(0);
let mut aggregated: FxHashMap<TypeIndex, AggregatedReceive> = let mut aggregated: FxHashMap<TypeIndex, AggregatedReceive> =
FxHashMap::with_capacity_and_hasher(max_type_len, Default::default()); FxHashMap::with_capacity_and_hasher(max_type_len, Default::default());
@@ -39,6 +53,13 @@ pub(crate) fn process_received(
let type_addr_count = addr_count.get_mut(output_type).unwrap(); let type_addr_count = addr_count.get_mut(output_type).unwrap();
let type_empty_count = empty_addr_count.get_mut(output_type).unwrap(); let type_empty_count = empty_addr_count.get_mut(output_type).unwrap();
let type_activity = activity_counts.get_mut_unwrap(output_type); let type_activity = activity_counts.get_mut_unwrap(output_type);
let type_reused_count = reused_addr_count.get_mut(output_type).unwrap();
let type_total_reused_count = total_reused_addr_count.get_mut(output_type).unwrap();
let type_output_to_reused_count = output_to_reused_addr_count.get_mut(output_type).unwrap();
let type_active_reused_count = active_reused_addr_count.get_mut(output_type).unwrap();
let type_exposed_count = exposed_addr_count.get_mut(output_type).unwrap();
let type_total_exposed_count = total_exposed_addr_count.get_mut(output_type).unwrap();
let type_exposed_supply = exposed_supply.get_mut(output_type).unwrap();
// Aggregate receives by address - each address processed exactly once // Aggregate receives by address - each address processed exactly once
for (type_index, value) in vec { for (type_index, value) in vec {
@@ -53,6 +74,13 @@ pub(crate) fn process_received(
// Track receiving activity - each address in receive aggregation // Track receiving activity - each address in receive aggregation
type_activity.receiving += 1; type_activity.receiving += 1;
// Capture state BEFORE the receive mutates funded_txo_count
let was_funded = addr_data.is_funded();
let was_reused = addr_data.is_reused();
let funded_txo_count_before = addr_data.funded_txo_count;
let was_pubkey_exposed = addr_data.is_pubkey_exposed(output_type);
let exposed_contribution_before = addr_data.exposed_supply_contribution(output_type);
match status { match status {
TrackingStatus::New => { TrackingStatus::New => {
*type_addr_count += 1; *type_addr_count += 1;
@@ -130,6 +158,62 @@ pub(crate) fn process_received(
.receive_outputs(addr_data, recv.total_value, price, recv.output_count); .receive_outputs(addr_data, recv.total_value, price, recv.output_count);
} }
} }
// Update reused counts based on the post-receive state
let is_now_reused = addr_data.is_reused();
if is_now_reused && !was_reused {
// Newly crossed the reuse threshold this block
*type_reused_count += 1;
*type_total_reused_count += 1;
} else if is_now_reused && !was_funded {
// Already-reused address reactivating into the funded set
*type_reused_count += 1;
}
// Block-level "active reused address" count: each address
// is processed exactly once here (via aggregation), so we
// count it once iff it is reused after the block's receives.
// The sender-side counterpart in process_sent dedupes
// against `received_addrs` so addresses that did both
// aren't double-counted.
if is_now_reused {
*type_active_reused_count += 1;
}
// Per-block reused-use count: every individual output to this
// address counts iff, at the moment the output arrives, the
// address had already received at least one prior output
// (i.e. it is an output-level "address reuse event"). With
// aggregation, that means we skip the very first output the
// address ever sees and count every subsequent one, so
// `skipped` is `max(0, 1 - before)`.
let skipped = 1u32.saturating_sub(funded_txo_count_before);
let counted = recv.output_count.saturating_sub(skipped);
*type_output_to_reused_count += u64::from(counted);
// Update exposed counts. The address's pubkey-exposure state
// is unchanged by a receive (spent_txo_count unchanged), so we
// can use the captured `was_pubkey_exposed` for both pre and post.
// After the receive the address is always funded, so it's in the
// funded exposed set iff its pubkey is exposed.
//
// Funded exposed enters when the address wasn't funded before but
// is now AND its pubkey is exposed.
// Total exposed (pk_exposed_at_funding types only) increments on
// first-ever receive (status == TrackingStatus::New); for other
// types it's incremented in process_sent on the first spend.
if !was_funded && was_pubkey_exposed {
*type_exposed_count += 1;
}
if output_type.pubkey_exposed_at_funding() && matches!(status, TrackingStatus::New) {
*type_total_exposed_count += 1;
}
// Update exposed supply via post-receive contribution delta.
let exposed_contribution_after = addr_data.exposed_supply_contribution(output_type);
// Receives can only add to balance and membership, so the delta
// is always non-negative.
*type_exposed_supply += exposed_contribution_after - exposed_contribution_before;
} }
} }
} }
@@ -5,7 +5,10 @@ use rustc_hash::FxHashSet;
use vecdb::VecIndex; use vecdb::VecIndex;
use crate::distribution::{ use crate::distribution::{
addr::{AddrTypeToActivityCounts, HeightToAddrTypeToVec}, addr::{
AddrTypeToActivityCounts, AddrTypeToExposedAddrCount, AddrTypeToExposedSupply,
AddrTypeToReusedAddrCount, AddrTypeToReusedAddrEventCount, HeightToAddrTypeToVec,
},
cohorts::AddrCohorts, cohorts::AddrCohorts,
compute::PriceRangeMax, compute::PriceRangeMax,
}; };
@@ -35,6 +38,12 @@ pub(crate) fn process_sent(
addr_count: &mut ByAddrType<u64>, addr_count: &mut ByAddrType<u64>,
empty_addr_count: &mut ByAddrType<u64>, empty_addr_count: &mut ByAddrType<u64>,
activity_counts: &mut AddrTypeToActivityCounts, activity_counts: &mut AddrTypeToActivityCounts,
reused_addr_count: &mut AddrTypeToReusedAddrCount,
input_from_reused_addr_count: &mut AddrTypeToReusedAddrEventCount,
active_reused_addr_count: &mut AddrTypeToReusedAddrEventCount,
exposed_addr_count: &mut AddrTypeToExposedAddrCount,
total_exposed_addr_count: &mut AddrTypeToExposedAddrCount,
exposed_supply: &mut AddrTypeToExposedSupply,
received_addrs: &ByAddrType<FxHashSet<TypeIndex>>, received_addrs: &ByAddrType<FxHashSet<TypeIndex>>,
height_to_price: &[Cents], height_to_price: &[Cents],
height_to_timestamp: &[Timestamp], height_to_timestamp: &[Timestamp],
@@ -57,12 +66,28 @@ pub(crate) fn process_sent(
let type_addr_count = addr_count.get_mut(output_type).unwrap(); let type_addr_count = addr_count.get_mut(output_type).unwrap();
let type_empty_count = empty_addr_count.get_mut(output_type).unwrap(); let type_empty_count = empty_addr_count.get_mut(output_type).unwrap();
let type_activity = activity_counts.get_mut_unwrap(output_type); let type_activity = activity_counts.get_mut_unwrap(output_type);
let type_reused_count = reused_addr_count.get_mut(output_type).unwrap();
let type_input_from_reused_count =
input_from_reused_addr_count.get_mut(output_type).unwrap();
let type_active_reused_count = active_reused_addr_count.get_mut(output_type).unwrap();
let type_exposed_count = exposed_addr_count.get_mut(output_type).unwrap();
let type_total_exposed_count = total_exposed_addr_count.get_mut(output_type).unwrap();
let type_exposed_supply = exposed_supply.get_mut(output_type).unwrap();
let type_received = received_addrs.get(output_type); let type_received = received_addrs.get(output_type);
let type_seen = seen_senders.get_mut_unwrap(output_type); let type_seen = seen_senders.get_mut_unwrap(output_type);
for (type_index, value) in vec { for (type_index, value) in vec {
let addr_data = lookup.get_for_send(output_type, type_index); let addr_data = lookup.get_for_send(output_type, type_index);
// "Input from a reused address" event: the sending
// address is in the reused set (lifetime
// funded_txo_count > 1). Checked once per input. The
// spend itself doesn't touch funded_txo_count so the
// predicate is stable before/after `cohort_state.send`.
if addr_data.is_reused() {
*type_input_from_reused_count += 1;
}
let prev_balance = addr_data.balance(); let prev_balance = addr_data.balance();
let new_balance = prev_balance.checked_sub(value).unwrap(); let new_balance = prev_balance.checked_sub(value).unwrap();
@@ -70,14 +95,29 @@ pub(crate) fn process_sent(
if type_seen.insert(type_index) { if type_seen.insert(type_index) {
type_activity.sending += 1; type_activity.sending += 1;
// Track "both" - addresses that sent AND received this block let also_received = type_received.is_some_and(|s| s.contains(&type_index));
if type_received.is_some_and(|s| s.contains(&type_index)) { // Track "bidirectional": addresses that sent AND
type_activity.both += 1; // received this block.
if also_received {
type_activity.bidirectional += 1;
}
// Block-level "active reused address" count: count
// every distinct sender that's reused, but skip
// those that also received this block (already
// counted in process_received).
if !also_received && addr_data.is_reused() {
*type_active_reused_count += 1;
} }
} }
let will_be_empty = addr_data.has_1_utxos(); let will_be_empty = addr_data.has_1_utxos();
// Capture exposed state BEFORE the spend mutates spent_txo_count.
let was_pubkey_exposed = addr_data.is_pubkey_exposed(output_type);
let exposed_contribution_before =
addr_data.exposed_supply_contribution(output_type);
// Compute buckets once // Compute buckets once
let prev_bucket = AmountBucket::from(prev_balance); let prev_bucket = AmountBucket::from(prev_balance);
let new_bucket = AmountBucket::from(new_balance); let new_bucket = AmountBucket::from(new_balance);
@@ -91,6 +131,28 @@ pub(crate) fn process_sent(
.unwrap(); .unwrap();
cohort_state.send(addr_data, value, current_price, prev_price, peak_price, age)?; cohort_state.send(addr_data, value, current_price, prev_price, peak_price, age)?;
// addr_data.spent_txo_count is now incremented by 1.
// Update exposed supply via post-spend contribution delta.
let exposed_contribution_after = addr_data.exposed_supply_contribution(output_type);
if exposed_contribution_after >= exposed_contribution_before {
*type_exposed_supply +=
exposed_contribution_after - exposed_contribution_before;
} else {
*type_exposed_supply -=
exposed_contribution_before - exposed_contribution_after;
}
// Update exposed counts on first-ever pubkey exposure.
// For non-pk-exposed types this fires on the first spend; for
// pk-exposed types it never fires here (was_pubkey_exposed was
// already true at first receive in process_received).
if !was_pubkey_exposed {
*type_total_exposed_count += 1;
if !will_be_empty {
*type_exposed_count += 1;
}
}
// If crossing a bucket boundary, remove the (now-updated) address from old bucket // If crossing a bucket boundary, remove the (now-updated) address from old bucket
if will_be_empty || crossing_boundary { if will_be_empty || crossing_boundary {
@@ -101,6 +163,17 @@ pub(crate) fn process_sent(
if will_be_empty { if will_be_empty {
*type_addr_count -= 1; *type_addr_count -= 1;
*type_empty_count += 1; *type_empty_count += 1;
// Reused addr leaving the funded reused set
if addr_data.is_reused() {
*type_reused_count -= 1;
}
// Exposed addr leaving the funded exposed set: was in set
// iff its pubkey was exposed pre-spend (since it was funded
// to be in process_sent in the first place), and now leaves
// because it's empty.
if was_pubkey_exposed {
*type_exposed_count -= 1;
}
lookup.move_to_empty(output_type, type_index); lookup.move_to_empty(output_type, type_index);
} else if crossing_boundary { } else if crossing_boundary {
cohorts cohorts
@@ -41,10 +41,7 @@ pub(crate) fn update_tx_counts(
.get_mut(&type_index) .get_mut(&type_index)
{ {
addr_data.tx_count += tx_count; addr_data.tx_count += tx_count;
} else if let Some(addr_data) = empty_cache } else if let Some(addr_data) = empty_cache.get_mut(addr_type).unwrap().get_mut(&type_index)
.get_mut(addr_type)
.unwrap()
.get_mut(&type_index)
{ {
addr_data.tx_count += tx_count; addr_data.tx_count += tx_count;
} }
@@ -90,9 +90,7 @@ pub(crate) fn process_inputs(
}; };
let items: Vec<_> = if input_count < 128 { let items: Vec<_> = if input_count < 128 {
(0..input_count) (0..input_count).map(map_fn).collect::<Result<Vec<_>>>()?
.map(map_fn)
.collect::<Result<Vec<_>>>()?
} else { } else {
(0..input_count) (0..input_count)
.into_par_iter() .into_par_iter()
@@ -109,10 +107,9 @@ pub(crate) fn process_inputs(
Default::default(), Default::default(),
); );
let mut sent_data = HeightToAddrTypeToVec::with_capacity(estimated_unique_heights); let mut sent_data = HeightToAddrTypeToVec::with_capacity(estimated_unique_heights);
let mut addr_data = let mut addr_data = AddrTypeToTypeIndexMap::<WithAddrDataSource<FundedAddrData>>::with_capacity(
AddrTypeToTypeIndexMap::<WithAddrDataSource<FundedAddrData>>::with_capacity( estimated_per_type,
estimated_per_type, );
);
let mut tx_index_vecs = let mut tx_index_vecs =
AddrTypeToTypeIndexMap::<SmallVec<[TxIndex; 4]>>::with_capacity(estimated_per_type); AddrTypeToTypeIndexMap::<SmallVec<[TxIndex; 4]>>::with_capacity(estimated_per_type);
@@ -5,9 +5,7 @@ use rayon::prelude::*;
use smallvec::SmallVec; use smallvec::SmallVec;
use crate::distribution::{ use crate::distribution::{
addr::{ addr::{AddrTypeToTypeIndexMap, AddrTypeToVec, AddrsDataVecs, AnyAddrIndexesVecs},
AddrTypeToTypeIndexMap, AddrTypeToVec, AddrsDataVecs, AnyAddrIndexesVecs,
},
compute::{TxOutData, VecsReaders}, compute::{TxOutData, VecsReaders},
state::Transacted, state::Transacted,
}; };
@@ -79,9 +77,7 @@ pub(crate) fn process_outputs(
}; };
let items: Vec<_> = if output_count < 128 { let items: Vec<_> = if output_count < 128 {
(0..output_count) (0..output_count).map(map_fn).collect::<Result<Vec<_>>>()?
.map(map_fn)
.collect::<Result<Vec<_>>>()?
} else { } else {
(0..output_count) (0..output_count)
.into_par_iter() .into_par_iter()
@@ -93,10 +89,9 @@ pub(crate) fn process_outputs(
let estimated_per_type = (output_count / 8).max(8); let estimated_per_type = (output_count / 8).max(8);
let mut transacted = Transacted::default(); let mut transacted = Transacted::default();
let mut received_data = AddrTypeToVec::with_capacity(estimated_per_type); let mut received_data = AddrTypeToVec::with_capacity(estimated_per_type);
let mut addr_data = let mut addr_data = AddrTypeToTypeIndexMap::<WithAddrDataSource<FundedAddrData>>::with_capacity(
AddrTypeToTypeIndexMap::<WithAddrDataSource<FundedAddrData>>::with_capacity( estimated_per_type,
estimated_per_type, );
);
let mut tx_index_vecs = let mut tx_index_vecs =
AddrTypeToTypeIndexMap::<SmallVec<[TxIndex; 4]>>::with_capacity(estimated_per_type); AddrTypeToTypeIndexMap::<SmallVec<[TxIndex; 4]>>::with_capacity(estimated_per_type);

Some files were not shown because too many files have changed in this diff Show More