Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-16 21:48:10 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: opreturn

Browse Source
This commit is contained in:
nym21
2026-07-16 17:39:05 +02:00
parent 8d43f340a0
commit 29ad37803e
100 changed files with 6084 additions and 1924 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -26,6 +26,7 @@ _*
/playground
/*.txt
/*.csv
/tmp
# Logs
*.log*
Cargo.lock
Generated
+84 -137
View File
@@ -659,6 +659,7 @@ dependencies = [
"byteview",
"fjall",
"rustc-hash",
"tracing",
]
[[package]]
@@ -750,9 +751,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytemuck"
version = "1.25.0"
version = "1.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424"
[[package]]
name = "byteorder"
@@ -768,9 +769,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
[[package]]
name = "bytes"
version = "1.12.0"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593"
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]]
name = "byteview"
@@ -780,9 +781,9 @@ checksum = "1c53ba0f290bfc610084c05582d9c5d421662128fc69f4bf236707af6fd321b9"
[[package]]
name = "cc"
version = "1.2.65"
version = "1.2.67"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96"
checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -1013,18 +1014,18 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
version = "0.5.15"
version = "0.5.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.6"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
@@ -1032,18 +1033,18 @@ dependencies = [
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
version = "0.9.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.12"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
dependencies = [
"crossbeam-utils",
]
@@ -1060,9 +1061,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
[[package]]
name = "crunchy"
@@ -1092,9 +1093,9 @@ dependencies = [
[[package]]
name = "defmt"
version = "1.1.0"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f"
checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1"
dependencies = [
"bitflags 1.3.2",
"defmt-macros",
@@ -1102,12 +1103,11 @@ dependencies = [
[[package]]
name = "defmt-macros"
version = "1.1.0"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b"
checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8"
dependencies = [
"defmt-parser",
"proc-macro-error2",
"proc-macro2",
"quote",
"syn",
@@ -1292,9 +1292,7 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "fjall"
version = "3.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "038acd422d607e0eca09e093f299f9eccf9bd097554343d93746afff81a45113"
version = "3.0.4"
dependencies = [
"byteorder-lite",
"byteview",
@@ -1302,7 +1300,7 @@ dependencies = [
"flume",
"log",
"lsm-tree",
"lz4_flex",
"lz4_flex 0.11.6",
"tempfile",
"xxhash-rust",
]
@@ -1454,18 +1452,6 @@ dependencies = [
"wasi",
]
[[package]]
name = "getrandom"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [
"cfg-if",
"libc",
"r-efi 5.3.0",
"wasip2",
]
[[package]]
name = "getrandom"
version = "0.4.3"
@@ -1474,7 +1460,7 @@ checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
dependencies = [
"cfg-if",
"libc",
"r-efi 6.0.0",
"r-efi",
]
[[package]]
@@ -1576,9 +1562,9 @@ dependencies = [
[[package]]
name = "http-body"
version = "1.0.1"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
dependencies = [
"bytes",
"http",
@@ -1586,9 +1572,9 @@ dependencies = [
[[package]]
name = "http-body-util"
version = "0.1.3"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2"
dependencies = [
"bytes",
"futures-core",
@@ -1876,9 +1862,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "jiff"
version = "0.2.31"
version = "0.2.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634"
checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e"
dependencies = [
"defmt",
"jiff-static",
@@ -1891,9 +1877,9 @@ dependencies = [
[[package]]
name = "jiff-static"
version = "0.2.31"
version = "0.2.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2"
checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc"
dependencies = [
"proc-macro2",
"quote",
@@ -1902,11 +1888,11 @@ dependencies = [
[[package]]
name = "jobserver"
version = "0.1.34"
version = "0.1.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
dependencies = [
"getrandom 0.3.4",
"getrandom 0.4.3",
"libc",
]
@@ -2031,9 +2017,7 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "lsm-tree"
version = "3.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ef86c3c797c10eefcc73407c43ae48c19d4df686131a8334b2895a513e91df4"
version = "3.0.4"
dependencies = [
"byteorder-lite",
"byteview",
@@ -2041,7 +2025,7 @@ dependencies = [
"enum_dispatch",
"interval-heap",
"log",
"lz4_flex",
"lz4_flex 0.11.6",
"quick_cache",
"rustc-hash",
"self_cell",
@@ -2053,13 +2037,19 @@ dependencies = [
[[package]]
name = "lz4_flex"
version = "0.13.1"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
dependencies = [
"twox-hash",
]
[[package]]
name = "lz4_flex"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
[[package]]
name = "matchit"
version = "0.8.4"
@@ -2068,9 +2058,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
[[package]]
name = "memchr"
version = "2.8.2"
version = "2.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
[[package]]
name = "memmap2"
@@ -2114,9 +2104,9 @@ dependencies = [
[[package]]
name = "mio"
version = "1.2.1"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
dependencies = [
"libc",
"wasi",
@@ -2371,28 +2361,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "proc-macro-error-attr2"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5"
dependencies = [
"proc-macro2",
"quote",
]
[[package]]
name = "proc-macro-error2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802"
dependencies = [
"proc-macro-error-attr2",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "proc-macro2"
version = "1.0.106"
@@ -2430,12 +2398,6 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "r-efi"
version = "6.0.0"
@@ -2459,18 +2421,18 @@ dependencies = [
[[package]]
name = "rapidhash"
version = "4.4.2"
version = "4.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37"
checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e"
dependencies = [
"rustversion",
]
[[package]]
name = "rawdb"
version = "0.10.3"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b22c0f9f22e83612f89b67f341029c59d652f59f88d053037ad464ff4f1f886"
checksum = "7df8d0fa0319c87c369464ffe5ee0baa6659ed37d3426a9ae021fb463d95ad52"
dependencies = [
"libc",
"log",
@@ -2543,9 +2505,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.12.4"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2"
dependencies = [
"aho-corasick",
"memchr",
@@ -2555,9 +2517,9 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.4.14"
version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db"
dependencies = [
"aho-corasick",
"memchr",
@@ -2595,15 +2557,15 @@ dependencies = [
[[package]]
name = "rustc-demangle"
version = "0.1.27"
version = "0.1.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb"
[[package]]
name = "rustc-hash"
version = "2.1.2"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
[[package]]
name = "rustc_version"
@@ -2629,9 +2591,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.41"
version = "0.23.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f"
checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
dependencies = [
"log",
"once_cell",
@@ -2664,9 +2626,9 @@ dependencies = [
[[package]]
name = "rustversion"
version = "1.0.22"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
[[package]]
name = "ryu"
@@ -2909,9 +2871,9 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
[[package]]
name = "socket2"
version = "0.6.4"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
dependencies = [
"libc",
"windows-sys 0.61.2",
@@ -2919,9 +2881,9 @@ dependencies = [
[[package]]
name = "spin"
version = "0.9.8"
version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e"
dependencies = [
"lock_api",
]
@@ -3041,9 +3003,9 @@ dependencies = [
[[package]]
name = "thread_local"
version = "1.1.9"
version = "1.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070"
dependencies = [
"cfg-if",
]
@@ -3386,14 +3348,14 @@ checksum = "bfa6c38708f6257f1ec2ca7e5a11f9bbf58a27d7060078b6b333624968183d96"
[[package]]
name = "vecdb"
version = "0.10.3"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b66ff235ce524e97c0d2a8e386fe842b6939016b90ed732844cd91e0337bd5e1"
checksum = "294fcad993a26e2668dca931c610b8461e66837b5f359a0c1a89bdf357cacf4b"
dependencies = [
"itoa",
"libc",
"log",
"lz4_flex",
"lz4_flex 0.13.1",
"parking_lot",
"pco",
"rawdb",
@@ -3409,9 +3371,9 @@ dependencies = [
[[package]]
name = "vecdb_derive"
version = "0.10.3"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d5dadc8894e9260b3ac87613a3cdf2f26c319b1957018b7f87a4e6713c9a14d"
checksum = "87e6ddc694828bd6d372573727e02ea07047f4b65f78aaf859dae2fbf2dbbcfe"
dependencies = [
"quote",
"syn",
@@ -3439,15 +3401,6 @@ version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "wasip2"
version = "1.0.4+wasi-0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
dependencies = [
"wit-bindgen",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.126"
@@ -3692,9 +3645,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winnow"
version = "1.0.3"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1"
checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
[[package]]
name = "wio"
@@ -3705,12 +3658,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "wit-bindgen"
version = "0.57.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
[[package]]
name = "writeable"
version = "0.6.3"
@@ -3759,18 +3706,18 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.52"
version = "0.8.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.52"
version = "0.8.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
dependencies = [
"proc-macro2",
"quote",
@@ -3839,9 +3786,9 @@ dependencies = [
[[package]]
name = "zmij"
version = "1.0.21"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
[[package]]
name = "zstd"
Cargo.toml
+8 -5
View File
@@ -64,15 +64,15 @@ color-eyre = "0.6.5"
corepc-jsonrpc = { package = "jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false }
corepc-types = { version = "0.15.0", features = ["std"], default-features = false }
derive_more = { version = "2.1.1", features = ["deref", "deref_mut"] }
fjall = "3.1.5"
fjall = { version = "=3.0.4", path = "../fjall" }
indexmap = { version = "2.14.0", features = ["serde"] }
jiff = { version = "0.2.31", features = ["perf-inline", "tz-system"], default-features = false }
jiff = { version = "0.2.32", features = ["perf-inline", "tz-system"], default-features = false }
owo-colors = "4.3.0"
parking_lot = "0.12.5"
pco = "1.0.2"
rayon = "1.12.0"
rapidhash = "4.4.2"
rustc-hash = "2.1.2"
rapidhash = "4.5.1"
rustc-hash = "2.1.3"
schemars = { version = "1.2.1", features = ["indexmap2"] }
serde = "1.0.228"
serde_bytes = "0.11.19"
@@ -84,9 +84,12 @@ tower-http = { version = "0.7.0", features = ["catch-panic", "compression-br", "
tower-layer = "0.3"
tracing = { version = "0.1", default-features = false, features = ["std"] }
ureq = { version = "3.3.0", features = ["json"] }
vecdb = { version = "0.10.3", features = ["derive", "serde_json", "pco", "schemars"] }
vecdb = { version = "0.10.4", features = ["derive", "serde_json", "pco", "schemars"] }
# vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] }
[patch.crates-io]
lsm-tree = { path = "../lsm-tree" }
[workspace.metadata.release]
shared-version = true
tag-name = "v{{version}}"
crates/brk_computer/src/distribution/cohorts/utxo/groups.rs
+1 -1
View File
@@ -16,6 +16,7 @@ use vecdb::{
use crate::{
blocks,
distribution::{
DynCohortVecs,
metrics::{
AllCohortMetrics, BasicCohortMetrics, CohortMetricsBase, CoreCohortMetrics,
ExtendedAdjustedCohortMetrics, ExtendedCohortMetrics, ImportConfig,
@@ -23,7 +24,6 @@ use crate::{
TypeCohortMetrics,
},
state::UTXOCohortState,
DynCohortVecs,
},
indexes,
internal::{ValuePerBlockCumulativeRolling, WindowStartVec, Windows},
crates/brk_computer/src/indexes/addr.rs
+1 -6
View File
@@ -250,12 +250,7 @@ impl Vecs {
identity: LazyVecFrom1::init(
"op_return_index",
version,
indexer
.vecs
.scripts
.op_return
.to_tx_index
.read_only_boxed_clone(),
indexer.vecs.op_return.to_tx_index.read_only_boxed_clone(),
|index, _| index,
),
},
crates/brk_computer/src/inputs/by_type/compute.rs
+9 -5
View File
@@ -6,6 +6,8 @@ use vecdb::{AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use super::{Vecs, WithInputTypes};
use crate::internal::{CoinbasePolicy, PerBlockCumulativeRolling, walk_blocks};
const WRITE_INTERVAL: usize = 10_000;
impl Vecs {
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
let starting_lengths = indexer.safe_lengths();
@@ -37,6 +39,7 @@ impl Vecs {
let mut itype_cursor = indexer.vecs.inputs.output_type.cursor();
let mut fi_in_cursor = indexer.vecs.transactions.first_txin_index.cursor();
let mut height = skip;
walk_blocks(
&fi_batch,
@@ -65,7 +68,8 @@ impl Vecs {
);
push_block(&mut self.tx_count, agg.txs_all, &agg.txs_per_type);
if self.input_count.all.block.batch_limit_reached() {
height += 1;
if height.is_multiple_of(WRITE_INTERVAL) {
let _lock = exit.lock();
self.input_count.write()?;
self.tx_count.write()?;
@@ -79,12 +83,12 @@ impl Vecs {
self.input_count.write()?;
self.tx_count.write()?;
}
self.input_count
.compute_rest(starting_lengths.height, exit)?;
self.tx_count.compute_rest(starting_lengths.height, exit)?;
}
self.input_count
.compute_rest(starting_lengths.height, exit)?;
self.tx_count.compute_rest(starting_lengths.height, exit)?;
for (otype, source) in self.input_count.by_type.iter_typed() {
self.input_share.get_mut(otype).compute_count_ratio(
source,
crates/brk_computer/src/inputs/compute.rs
+2 -3
View File
@@ -3,13 +3,12 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes};
use crate::blocks;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
exit: &Exit,
) -> Result<()> {
@@ -18,7 +17,7 @@ impl Vecs {
let starting_lengths = indexer.safe_lengths();
self.spent.compute(indexer, exit)?;
self.count.compute(indexer, indexes, blocks, exit)?;
self.count.compute(indexer, blocks, exit)?;
self.per_sec.compute(&self.count, &starting_lengths, exit)?;
self.by_type.compute(indexer, exit)?;
crates/brk_computer/src/inputs/count/compute.rs
+6 -8
View File
@@ -3,27 +3,25 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes};
use crate::blocks;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let window_starts = blocks.lookback.window_starts();
self.0.compute(
self.sum.compute_count_from_indexes(
starting_height,
&indexes.tx_index.input_count,
&indexer.vecs.transactions.first_tx_index,
&indexes.height.tx_index_count,
&window_starts,
&indexer.vecs.inputs.first_txin_index,
&indexer.vecs.inputs.outpoint,
exit,
0,
)?;
self.compute_rest(starting_height, &window_starts, exit)?;
Ok(())
}
crates/brk_computer/src/internal/per_block/computed/aggregated.rs
+5 -71
View File
@@ -2,10 +2,7 @@ use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::Height;
use schemars::JsonSchema;
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, ImportableVec, PcoVec, ReadableVec, Rw,
StorageMode, VecIndex, VecValue, Version, WritableVec,
};
use vecdb::{Database, EagerVec, Exit, ImportableVec, PcoVec, Rw, StorageMode, Version};
use crate::{
indexes,
@@ -52,82 +49,19 @@ where
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute<A>(
pub(crate) fn compute_rest(
&mut self,
max_from: Height,
source: &impl ReadableVec<A, T>,
first_indexes: &impl ReadableVec<Height, A>,
count_indexes: &impl ReadableVec<Height, brk_types::StoredU64>,
windows: &WindowStarts<'_>,
exit: &Exit,
skip_count: usize,
) -> Result<()>
where
T: From<f64> + Default + Copy + Ord,
f64: From<T>,
A: VecIndex + VecValue + brk_types::CheckedSub<A>,
{
let combined_version = source.version() + first_indexes.version() + count_indexes.version();
let mut index = max_from;
index = {
self.sum
.validate_computed_version_or_reset(combined_version)?;
index.min(Height::from(self.sum.len()))
};
index = {
self.cumulative
.height
.validate_computed_version_or_reset(combined_version)?;
index.min(Height::from(self.cumulative.height.len()))
};
let start = index.to_usize();
self.sum.truncate_if_needed_at(start)?;
self.cumulative.height.truncate_if_needed_at(start)?;
let mut cumulative_val = index.decremented().map_or(T::from(0_usize), |idx| {
self.cumulative
.height
.collect_one_at(idx.to_usize())
.unwrap_or(T::from(0_usize))
});
let fi_len = first_indexes.len();
let first_indexes_batch: Vec<A> = first_indexes.collect_range_at(start, fi_len);
let count_indexes_batch: Vec<brk_types::StoredU64> =
count_indexes.collect_range_at(start, fi_len);
first_indexes_batch
.into_iter()
.zip(count_indexes_batch)
.try_for_each(|(first_index, count_index)| -> Result<()> {
let count = u64::from(count_index) as usize;
let effective_count = count.saturating_sub(skip_count);
let effective_first_index = first_index + skip_count.min(count);
let efi = effective_first_index.to_usize();
let sum_val = source.fold_range_at(
efi,
efi + effective_count,
T::from(0_usize),
|acc, val| acc + val,
);
self.sum.push(sum_val);
cumulative_val += sum_val;
self.cumulative.height.push(cumulative_val);
Ok(())
})?;
let _lock = exit.lock();
self.sum.write()?;
self.cumulative.height.write()?;
drop(_lock);
self.cumulative
.height
.compute_cumulative(max_from, &self.sum, exit)?;
self.rolling.compute(max_from, windows, &self.sum, exit)?;
Ok(())
}
crates/brk_computer/src/internal/per_tx/lazy_distribution.rs
+8 -10
View File
@@ -3,7 +3,7 @@ use brk_indexer::{Indexer, Lengths};
use brk_traversable::Traversable;
use brk_types::TxIndex;
use schemars::JsonSchema;
use vecdb::{Database, Exit, LazyVecFrom2, ReadableVec, Rw, StorageMode, Version};
use vecdb::{Database, Exit, LazyVecFrom1, ReadableVec, Rw, StorageMode, Version};
use crate::{
indexes,
@@ -11,29 +11,27 @@ use crate::{
};
#[derive(Traversable)]
pub struct LazyPerTxDistribution<T, S1, S2, M: StorageMode = Rw>
pub struct LazyPerTxDistribution<T, S, M: StorageMode = Rw>
where
T: ComputedVecValue + PartialOrd + JsonSchema,
S1: ComputedVecValue,
S2: ComputedVecValue,
S: ComputedVecValue,
{
pub tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
pub tx_index: LazyVecFrom1<TxIndex, T, TxIndex, S>,
#[traversable(flatten)]
pub distribution: TxDerivedDistribution<T, M>,
}
impl<T, S1, S2> LazyPerTxDistribution<T, S1, S2>
impl<T, S> LazyPerTxDistribution<T, S>
where
T: NumericValue + JsonSchema,
S1: ComputedVecValue + JsonSchema,
S2: ComputedVecValue + JsonSchema,
S: ComputedVecValue + JsonSchema,
{
pub(crate) fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
tx_index: LazyVecFrom1<TxIndex, T, TxIndex, S>,
) -> Result<Self> {
let distribution = TxDerivedDistribution::forced_import(db, name, version, indexes)?;
Ok(Self {
@@ -52,7 +50,7 @@ where
where
T: Copy + Ord + From<f64> + Default,
f64: From<T>,
LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>: ReadableVec<TxIndex, T>,
LazyVecFrom1<TxIndex, T, TxIndex, S>: ReadableVec<TxIndex, T>,
{
self.distribution
.derive_from(indexer, indexes, starting_lengths, &self.tx_index, exit)
crates/brk_computer/src/internal/per_tx/lazy_distribution_transformed.rs
+7 -9
View File
@@ -1,36 +1,34 @@
use brk_traversable::Traversable;
use brk_types::{TxIndex, Version};
use schemars::JsonSchema;
use vecdb::{LazyVecFrom2, UnaryTransform};
use vecdb::{LazyVecFrom1, UnaryTransform};
use crate::internal::{ComputedVecValue, LazyTxDerivedDistribution, TxDerivedDistribution};
/// Like `LazyPerTxDistribution` but with a lazy-derived distribution
/// (transformed from another type's distribution rather than eagerly computed).
#[derive(Clone, Traversable)]
pub struct LazyPerTxDistributionTransformed<T, S1, S2, DSource>
pub struct LazyPerTxDistributionTransformed<T, S, DSource>
where
T: ComputedVecValue + JsonSchema,
S1: ComputedVecValue,
S2: ComputedVecValue,
S: ComputedVecValue,
DSource: ComputedVecValue,
{
pub tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
pub tx_index: LazyVecFrom1<TxIndex, T, TxIndex, S>,
#[traversable(flatten)]
pub distribution: LazyTxDerivedDistribution<T, DSource>,
}
impl<T, S1, S2, DSource> LazyPerTxDistributionTransformed<T, S1, S2, DSource>
impl<T, S, DSource> LazyPerTxDistributionTransformed<T, S, DSource>
where
T: ComputedVecValue + JsonSchema + 'static,
S1: ComputedVecValue + JsonSchema,
S2: ComputedVecValue + JsonSchema,
S: ComputedVecValue + JsonSchema,
DSource: ComputedVecValue + JsonSchema,
{
pub(crate) fn new<F: UnaryTransform<DSource, T>>(
name: &str,
version: Version,
tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
tx_index: LazyVecFrom1<TxIndex, T, TxIndex, S>,
source_distribution: &TxDerivedDistribution<DSource>,
) -> Self {
let distribution =
crates/brk_computer/src/internal/transform/arithmetic.rs
+4 -4
View File
@@ -146,11 +146,11 @@ impl UnaryTransform<StoredU64, Weight> for VBytesToWeight {
}
}
pub struct VSizeToWeight;
pub struct WeightToVSize;
impl UnaryTransform<VSize, Weight> for VSizeToWeight {
impl UnaryTransform<Weight, VSize> for WeightToVSize {
#[inline(always)]
fn apply(vsize: VSize) -> Weight {
Weight::from(vsize)
fn apply(weight: Weight) -> VSize {
VSize::from(weight)
}
}
crates/brk_computer/src/internal/transform/mod.rs
+1 -1
View File
@@ -8,7 +8,7 @@ mod specialized;
pub use arithmetic::{
BlocksToDaysF32, DifficultyToHashF64, HalveCents, HalveDollars, HalveSats, HalveSatsToBitcoin,
Identity, MaskSats, OneMinusBp16, OneMinusF64, ReturnF32Tenths, ReturnI8, ReturnU16,
ThsToPhsF32, VBytesToWeight, VSizeToWeight,
ThsToPhsF32, VBytesToWeight, WeightToVSize,
};
pub use bps::{
Bp16ToFloat, Bp16ToPercent, Bp32ToFloat, Bp32ToPercent, Bps16ToFloat, Bps16ToPercent,
crates/brk_computer/src/lib.rs
+40 -16
View File
@@ -20,6 +20,7 @@ mod internal;
mod investing;
mod market;
mod mining;
mod op_return;
mod outputs;
mod pools;
pub mod price;
@@ -44,9 +45,10 @@ pub struct Computer<M: StorageMode = Rw> {
pub supply: Box<supply::Vecs<M>>,
pub inputs: Box<inputs::Vecs<M>>,
pub outputs: Box<outputs::Vecs<M>>,
pub op_return: Box<op_return::Vecs<M>>,
}
const VERSION: Version = Version::new(6);
const VERSION: Version = Version::new(7);
impl Computer {
pub fn forced_import(outputs_path: &Path, indexer: &Indexer) -> Result<Self> {
@@ -87,8 +89,9 @@ impl Computer {
let cached_starts = blocks.lookback.cached_window_starts();
let (inputs, outputs, mining, transactions, pools, cointime) =
timed("Imported inputs/outputs/mining/tx/pools/cointime", || {
let (inputs, outputs, mining, transactions, pools, cointime, op_return) = timed(
"Imported inputs/outputs/mining/tx/pools/cointime/op_return",
|| {
thread::scope(|s| -> Result<_> {
let inputs_handle = big_thread().spawn_scoped(s, || -> Result<_> {
Ok(Box::new(inputs::Vecs::forced_import(
@@ -143,15 +146,33 @@ impl Computer {
&cached_starts,
)?);
let op_return_handle = big_thread().spawn_scoped(s, || -> Result<_> {
Ok(Box::new(op_return::Vecs::forced_import(
&computed_path,
VERSION,
&indexes,
)?))
})?;
let inputs = inputs_handle.join().unwrap()?;
let outputs = outputs_handle.join().unwrap()?;
let mining = mining_handle.join().unwrap()?;
let transactions = transactions_handle.join().unwrap()?;
let pools = pools_handle.join().unwrap()?;
let op_return = op_return_handle.join().unwrap()?;
Ok((inputs, outputs, mining, transactions, pools, cointime))
Ok((
inputs,
outputs,
mining,
transactions,
pools,
cointime,
op_return,
))
})
})?;
},
)?;
// Market, indicators, and distribution are independent; import in parallel.
// Supply depends on distribution so it runs after.
@@ -225,6 +246,7 @@ impl Computer {
inputs,
price,
outputs,
op_return,
};
Self::retain_databases(&computed_path)?;
@@ -249,6 +271,7 @@ impl Computer {
supply::DB_NAME,
inputs::DB_NAME,
outputs::DB_NAME,
op_return::DB_NAME,
];
if !computed_path.exists() {
@@ -292,8 +315,7 @@ impl Computer {
let (inputs_result, prices_result) = rayon::join(
|| {
timed("Computed inputs", || {
self.inputs
.compute(indexer, &self.indexes, &self.blocks, exit)
self.inputs.compute(indexer, &self.blocks, exit)
})
},
|| {
@@ -337,18 +359,19 @@ impl Computer {
})
});
let op_return = scope.spawn(|| {
timed("Computed OP_RETURN", || {
self.op_return.compute(indexer, exit)
})
});
timed("Computed outputs", || {
self.outputs.compute(
indexer,
&self.indexes,
&self.inputs,
&self.blocks,
&self.price,
exit,
)
self.outputs
.compute(indexer, &self.inputs, &self.blocks, &self.price, exit)
})?;
tx_mining.join().unwrap()?;
op_return.join().unwrap()?;
market.join().unwrap()?;
Ok(())
})?;
@@ -502,7 +525,8 @@ impl_iter_named!(
distribution,
supply,
inputs,
outputs
outputs,
op_return
);
fn timed<T>(label: &str, f: impl FnOnce() -> T) -> T {
crates/brk_computer/src/op_return/by_kind.rs
+89
View File
@@ -0,0 +1,89 @@
use brk_traversable::Traversable;
use brk_types::OpReturnKind;
macro_rules! define_by_kind {
($($field:ident => $kind:ident),+ $(,)?) => {
#[derive(Traversable)]
pub struct ByKind<T> {
$(pub $field: T),+
}
impl<T> ByKind<T> {
pub fn try_new<E>(
mut create: impl FnMut(OpReturnKind, &'static str) -> Result<T, E>,
) -> Result<Self, E> {
Ok(Self {
$($field: create(OpReturnKind::$kind, stringify!($field))?),+
})
}
pub fn iter(&self) -> impl Iterator<Item = &T> {
[$( &self.$field ),+].into_iter()
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
let Self { $($field),+ } = self;
[$($field),+].into_iter()
}
pub fn iter_typed(&self) -> impl Iterator<Item = (OpReturnKind, &T)> {
[$( (OpReturnKind::$kind, &self.$field) ),+].into_iter()
}
pub fn iter_typed_mut(
&mut self,
) -> impl Iterator<Item = (OpReturnKind, &mut T)> {
let Self { $($field),+ } = self;
[$( (OpReturnKind::$kind, $field) ),+].into_iter()
}
}
};
}
define_by_kind! {
runes => Runes,
veri_block => VeriBlock,
omni => Omni,
stacks => Stacks,
blockstack => Blockstack,
colu => Colu,
open_assets => OpenAssets,
komodo => Komodo,
coin_spark => CoinSpark,
poet => Poet,
docproof => Docproof,
open_timestamps => OpenTimestamps,
factom => Factom,
eternity_wall => EternityWall,
memo => Memo,
bitproof => Bitproof,
ascribe => Ascribe,
stampery => Stampery,
epobc => Epobc,
bare_hash => BareHash,
text => Text,
empty => Empty,
unknown => Unknown,
}
#[cfg(test)]
mod tests {
use std::convert::Infallible;
use brk_types::OpReturnKind;
use super::ByKind;
#[test]
fn covers_every_kind_in_discriminant_order() {
let by_kind =
ByKind::try_new(|kind, _| Ok::<_, Infallible>(kind)).expect("infallible constructor");
let kinds: Vec<_> = by_kind.iter_typed().collect();
assert_eq!(kinds.len(), OpReturnKind::Unknown as usize + 1);
for (index, (kind, value)) in kinds.into_iter().enumerate() {
assert_eq!(kind as usize, index);
assert_eq!(kind, *value);
}
}
}
crates/brk_computer/src/op_return/compute.rs
+242
View File
@@ -0,0 +1,242 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{OpReturnKind, VSize};
use vecdb::{AnyVec, Exit, ReadableVec, VecIndex};
use super::{Vecs, vecs::Totals};
const KIND_COUNT: usize = OpReturnKind::Unknown as usize + 1;
const OLD_STANDARD_MAX_POST_OP_RETURN_BYTES: u64 = 82;
const WRITE_INTERVAL: usize = 10_000;
#[derive(Clone, Copy, Default)]
struct PolicyTotals {
oversized: Totals,
multiple: Totals,
pre_v30_nonstandard: Totals,
}
#[derive(Clone, Copy, Default)]
struct Carrier {
kinds: u32,
output_count: u64,
post_op_return_bytes: u64,
oversized_output_count: u64,
oversized_post_op_return_bytes: u64,
vsize: VSize,
}
impl Carrier {
fn add_output(&mut self, kind: OpReturnKind, post_op_return_bytes: u64) {
self.kinds |= kind_bit(kind);
self.output_count += 1;
self.post_op_return_bytes += post_op_return_bytes;
if post_op_return_bytes > OLD_STANDARD_MAX_POST_OP_RETURN_BYTES {
self.oversized_output_count += 1;
self.oversized_post_op_return_bytes += post_op_return_bytes;
}
}
}
impl Vecs {
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
self.db.sync_bg_tasks()?;
let starting_lengths = indexer.safe_lengths();
let raw = &indexer.vecs.op_return;
let txs = &indexer.vecs.transactions;
let version = raw.first_index.version()
+ raw.to_tx_index.version()
+ raw.kind.version()
+ raw.post_op_return_bytes.version()
+ txs.weight.version();
self.validate_and_truncate(version, starting_lengths.height)?;
let skip = self.min_len();
let end = raw
.first_index
.len()
.min(starting_lengths.height.to_usize());
if skip < end {
self.truncate_if_needed_at(skip)?;
let op_return_len = raw
.to_tx_index
.len()
.min(raw.kind.len())
.min(raw.post_op_return_bytes.len())
.min(starting_lengths.op_return_index.to_usize());
let mut tx_cursor = raw.to_tx_index.cursor();
let mut kind_cursor = raw.kind.cursor();
let mut post_op_return_bytes = raw.post_op_return_bytes.cursor();
let mut first_index_cursor = raw.first_index.cursor();
let mut weight_cursor = txs.weight.cursor();
first_index_cursor.advance(skip);
let mut start = first_index_cursor.next().unwrap().to_usize();
for height in skip..end {
let block_end = if height + 1 < end {
first_index_cursor.next().unwrap().to_usize()
} else {
op_return_len
};
tx_cursor.advance(start - tx_cursor.position());
kind_cursor.advance(start - kind_cursor.position());
post_op_return_bytes.advance(start - post_op_return_bytes.position());
let mut total = Totals::default();
let mut by_kind = [Totals::default(); KIND_COUNT];
let mut policy = PolicyTotals::default();
let mut current_tx = None;
let mut carrier = Carrier::default();
for _ in start..block_end {
let tx_index = tx_cursor.next().unwrap();
let kind = kind_cursor.next().unwrap();
let bytes = u32::from(post_op_return_bytes.next().unwrap()) as u64;
let kind_index = kind as usize;
if current_tx != Some(tx_index) {
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
current_tx = Some(tx_index);
carrier = Carrier::default();
let tx_position = tx_index.to_usize();
weight_cursor.advance(tx_position - weight_cursor.position());
carrier.vsize = VSize::from(weight_cursor.next().unwrap());
}
total.output_count += 1;
total.post_op_return_bytes += bytes;
by_kind[kind_index].output_count += 1;
by_kind[kind_index].post_op_return_bytes += bytes;
carrier.add_output(kind, bytes);
}
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
self.total.push(total);
for (kind, metrics) in self.by_kind.iter_typed_mut() {
metrics.push(by_kind[kind as usize]);
}
self.policy.oversized.push(policy.oversized);
self.policy.multiple.push(policy.multiple);
self.policy
.pre_v30_nonstandard
.push(policy.pre_v30_nonstandard);
if (height + 1).is_multiple_of(WRITE_INTERVAL) {
let _lock = exit.lock();
self.write()?;
}
start = block_end;
}
let _lock = exit.lock();
self.write()?;
}
self.compute_cumulative(starting_lengths.height, exit)?;
let exit = exit.clone();
self.db.run_bg(move |db| {
let _lock = exit.lock();
db.compact_deferred_default()
});
Ok(())
}
}
fn finalize_transaction(
total: &mut Totals,
by_kind: &mut [Totals; KIND_COUNT],
policy: &mut PolicyTotals,
carrier: Carrier,
) {
if carrier.output_count == 0 {
return;
}
add_carrier(total, carrier.vsize);
let mut kinds = carrier.kinds;
while kinds != 0 {
let kind_index = kinds.trailing_zeros() as usize;
add_carrier(&mut by_kind[kind_index], carrier.vsize);
kinds &= kinds - 1;
}
if carrier.oversized_output_count > 0 {
policy.oversized.output_count += carrier.oversized_output_count;
policy.oversized.post_op_return_bytes += carrier.oversized_post_op_return_bytes;
add_carrier(&mut policy.oversized, carrier.vsize);
}
if carrier.output_count > 1 {
policy.multiple.output_count += carrier.output_count;
policy.multiple.post_op_return_bytes += carrier.post_op_return_bytes;
add_carrier(&mut policy.multiple, carrier.vsize);
}
if carrier.oversized_output_count > 0 || carrier.output_count > 1 {
policy.pre_v30_nonstandard.output_count += carrier.output_count;
policy.pre_v30_nonstandard.post_op_return_bytes += carrier.post_op_return_bytes;
add_carrier(&mut policy.pre_v30_nonstandard, carrier.vsize);
}
}
fn add_carrier(metrics: &mut Totals, vsize: VSize) {
metrics.carrier_tx_count += 1;
metrics.carrier_vsize += vsize;
}
const fn kind_bit(kind: OpReturnKind) -> u32 {
1_u32 << kind as u8
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn multiple_kinds_count_one_total_carrier() {
let mut total = Totals::default();
let mut by_kind = [Totals::default(); KIND_COUNT];
let mut policy = PolicyTotals::default();
let mut carrier = Carrier {
vsize: VSize::new(100),
..Carrier::default()
};
carrier.add_output(OpReturnKind::Runes, 15);
carrier.add_output(OpReturnKind::Omni, 15);
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
assert_eq!(total.carrier_tx_count, 1);
assert_eq!(by_kind[OpReturnKind::Runes as usize].carrier_tx_count, 1);
assert_eq!(by_kind[OpReturnKind::Omni as usize].carrier_tx_count, 1);
assert_eq!(policy.multiple.output_count, 2);
assert_eq!(policy.pre_v30_nonstandard.carrier_tx_count, 1);
assert_eq!(policy.oversized.carrier_tx_count, 0);
}
#[test]
fn oversized_output_marks_pre_v30_nonstandard_once() {
let mut total = Totals::default();
let mut by_kind = [Totals::default(); KIND_COUNT];
let mut policy = PolicyTotals::default();
let mut carrier = Carrier {
vsize: VSize::new(120),
..Carrier::default()
};
carrier.add_output(OpReturnKind::Unknown, 83);
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
assert_eq!(policy.oversized.output_count, 1);
assert_eq!(policy.oversized.carrier_vsize, VSize::new(120));
assert_eq!(policy.pre_v30_nonstandard.carrier_tx_count, 1);
assert_eq!(policy.multiple.carrier_tx_count, 0);
}
}
crates/brk_computer/src/op_return/import.rs
+34
View File
@@ -0,0 +1,34 @@
use std::path::Path;
use brk_error::Result;
use brk_types::Version;
use super::{ByKind, Metrics, Policy, Vecs};
use crate::{
indexes,
internal::db_utils::{finalize_db, open_db},
};
impl Vecs {
pub(crate) fn forced_import(
parent_path: &Path,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let db = open_db(parent_path, super::DB_NAME, 1_000_000)?;
let total = Metrics::forced_import(&db, "op_return", version, indexes)?;
let by_kind = ByKind::try_new(|_, name| {
Metrics::forced_import(&db, &format!("op_return_{name}"), version, indexes)
})?;
let policy = Policy::forced_import(&db, version, indexes)?;
let this = Self {
db,
total,
by_kind,
policy,
};
finalize_db(&this.db, &this)?;
Ok(this)
}
}
crates/brk_computer/src/op_return/mod.rs
+9
View File
@@ -0,0 +1,9 @@
mod by_kind;
mod compute;
mod import;
mod vecs;
pub use by_kind::ByKind;
pub use vecs::{Metrics, Policy, Vecs};
pub const DB_NAME: &str = "op_return";
crates/brk_computer/src/op_return/vecs.rs
+274
View File
@@ -0,0 +1,274 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, StoredU64, VSize, Version};
use schemars::JsonSchema;
use vecdb::{
AnyStoredVec, AnyVec, Database, EagerVec, Exit, ImportableVec, PcoVec, Rw, StorageMode,
WritableVec,
};
use super::ByKind;
use crate::{
indexes,
internal::{NumericValue, PerBlock},
};
#[derive(Traversable)]
pub struct Series<T, M: StorageMode = Rw>
where
T: NumericValue + JsonSchema,
{
pub block: M::Stored<EagerVec<PcoVec<Height, T>>>,
pub cumulative: PerBlock<T, M>,
}
impl<T> Series<T>
where
T: NumericValue + JsonSchema,
{
fn forced_import(
db: &Database,
name: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let block = EagerVec::forced_import(db, name, version)?;
let cumulative =
PerBlock::forced_import(db, &format!("{name}_cumulative"), version, indexes)?;
Ok(Self { block, cumulative })
}
fn len(&self) -> usize {
self.block.len()
}
fn validate_and_truncate(&mut self, version: Version, height: Height) -> Result<()> {
self.block.validate_and_truncate(version, height)?;
Ok(())
}
fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> {
self.block.truncate_if_needed_at(len)?;
Ok(())
}
fn push(&mut self, block: T) {
self.block.push(block);
}
fn write(&mut self) -> Result<()> {
self.block.write()?;
Ok(())
}
fn compute_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
self.cumulative
.height
.compute_cumulative(max_from, &self.block, exit)?;
Ok(())
}
}
#[derive(Clone, Copy, Default)]
pub(super) struct Totals {
pub output_count: u64,
pub post_op_return_bytes: u64,
pub carrier_tx_count: u64,
pub carrier_vsize: VSize,
}
#[derive(Traversable)]
pub struct Metrics<M: StorageMode = Rw> {
pub output_count: Series<StoredU64, M>,
pub post_op_return_bytes: Series<StoredU64, M>,
pub carrier_tx_count: Series<StoredU64, M>,
pub carrier_vsize: Series<VSize, M>,
}
impl Metrics {
pub(crate) fn forced_import(
db: &Database,
prefix: &str,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
Ok(Self {
output_count: Series::forced_import(
db,
&format!("{prefix}_output_count"),
version,
indexes,
)?,
post_op_return_bytes: Series::forced_import(
db,
&format!("{prefix}_post_op_return_bytes"),
version,
indexes,
)?,
carrier_tx_count: Series::forced_import(
db,
&format!("{prefix}_carrier_tx_count"),
version,
indexes,
)?,
carrier_vsize: Series::forced_import(
db,
&format!("{prefix}_carrier_vsize"),
version,
indexes,
)?,
})
}
fn len(&self) -> usize {
self.output_count
.len()
.min(self.post_op_return_bytes.len())
.min(self.carrier_tx_count.len())
.min(self.carrier_vsize.len())
}
pub(super) fn push(&mut self, block: Totals) {
self.output_count.push(block.output_count.into());
self.post_op_return_bytes
.push(block.post_op_return_bytes.into());
self.carrier_tx_count.push(block.carrier_tx_count.into());
self.carrier_vsize.push(block.carrier_vsize);
}
fn validate_and_truncate(&mut self, version: Version, height: Height) -> Result<()> {
self.output_count.validate_and_truncate(version, height)?;
self.post_op_return_bytes
.validate_and_truncate(version, height)?;
self.carrier_tx_count
.validate_and_truncate(version, height)?;
self.carrier_vsize.validate_and_truncate(version, height)?;
Ok(())
}
fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> {
self.output_count.truncate_if_needed_at(len)?;
self.post_op_return_bytes.truncate_if_needed_at(len)?;
self.carrier_tx_count.truncate_if_needed_at(len)?;
self.carrier_vsize.truncate_if_needed_at(len)?;
Ok(())
}
fn write(&mut self) -> Result<()> {
self.output_count.write()?;
self.post_op_return_bytes.write()?;
self.carrier_tx_count.write()?;
self.carrier_vsize.write()?;
Ok(())
}
fn compute_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
self.output_count.compute_cumulative(max_from, exit)?;
self.post_op_return_bytes
.compute_cumulative(max_from, exit)?;
self.carrier_tx_count.compute_cumulative(max_from, exit)?;
self.carrier_vsize.compute_cumulative(max_from, exit)?;
Ok(())
}
}
#[derive(Traversable)]
pub struct Policy<M: StorageMode = Rw> {
pub oversized: Metrics<M>,
pub multiple: Metrics<M>,
pub pre_v30_nonstandard: Metrics<M>,
}
impl Policy {
pub(crate) fn forced_import(
db: &Database,
version: Version,
indexes: &indexes::Vecs,
) -> Result<Self> {
let import = |name| {
Metrics::forced_import(db, &format!("op_return_policy_{name}"), version, indexes)
};
Ok(Self {
oversized: import("oversized")?,
multiple: import("multiple")?,
pre_v30_nonstandard: import("pre_v30_nonstandard")?,
})
}
fn iter(&self) -> impl Iterator<Item = &Metrics> {
[&self.oversized, &self.multiple, &self.pre_v30_nonstandard].into_iter()
}
fn iter_mut(&mut self) -> impl Iterator<Item = &mut Metrics> {
[
&mut self.oversized,
&mut self.multiple,
&mut self.pre_v30_nonstandard,
]
.into_iter()
}
}
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
#[traversable(skip)]
pub(crate) db: Database,
pub total: Metrics<M>,
pub by_kind: ByKind<Metrics<M>>,
pub policy: Policy<M>,
}
impl Vecs {
pub(crate) fn min_len(&self) -> usize {
self.by_kind
.iter()
.chain(self.policy.iter())
.map(Metrics::len)
.fold(self.total.len(), usize::min)
}
pub(crate) fn validate_and_truncate(&mut self, version: Version, height: Height) -> Result<()> {
self.total.validate_and_truncate(version, height)?;
for metrics in self.by_kind.iter_mut() {
metrics.validate_and_truncate(version, height)?;
}
for metrics in self.policy.iter_mut() {
metrics.validate_and_truncate(version, height)?;
}
Ok(())
}
pub(crate) fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> {
self.total.truncate_if_needed_at(len)?;
for metrics in self.by_kind.iter_mut() {
metrics.truncate_if_needed_at(len)?;
}
for metrics in self.policy.iter_mut() {
metrics.truncate_if_needed_at(len)?;
}
Ok(())
}
pub(crate) fn write(&mut self) -> Result<()> {
self.total.write()?;
for metrics in self.by_kind.iter_mut() {
metrics.write()?;
}
for metrics in self.policy.iter_mut() {
metrics.write()?;
}
Ok(())
}
pub(crate) fn compute_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
self.total.compute_cumulative(max_from, exit)?;
for metrics in self.by_kind.iter_mut() {
metrics.compute_cumulative(max_from, exit)?;
}
for metrics in self.policy.iter_mut() {
metrics.compute_cumulative(max_from, exit)?;
}
Ok(())
}
}
crates/brk_computer/src/outputs/by_type/compute.rs
+11 -7
View File
@@ -6,6 +6,8 @@ use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use super::{Vecs, WithOutputTypes};
use crate::internal::{CoinbasePolicy, PerBlockCumulativeRolling, walk_blocks};
const WRITE_INTERVAL: usize = 10_000;
impl Vecs {
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
let starting_lengths = indexer.safe_lengths();
@@ -44,6 +46,7 @@ impl Vecs {
let mut otype_cursor = indexer.vecs.outputs.output_type.cursor();
let mut fo_cursor = indexer.vecs.transactions.first_txout_index.cursor();
let mut height = skip;
walk_blocks(
&fi_batch,
@@ -77,7 +80,8 @@ impl Vecs {
.block
.push(StoredU64::from(spendable_total));
if self.output_count.all.block.batch_limit_reached() {
height += 1;
if height.is_multiple_of(WRITE_INTERVAL) {
let _lock = exit.lock();
self.output_count.write()?;
self.spendable_output_count.block.write()?;
@@ -93,14 +97,14 @@ impl Vecs {
self.spendable_output_count.block.write()?;
self.tx_count.write()?;
}
self.output_count
.compute_rest(starting_lengths.height, exit)?;
self.spendable_output_count
.compute_rest(starting_lengths.height, exit)?;
self.tx_count.compute_rest(starting_lengths.height, exit)?;
}
self.output_count
.compute_rest(starting_lengths.height, exit)?;
self.spendable_output_count
.compute_rest(starting_lengths.height, exit)?;
self.tx_count.compute_rest(starting_lengths.height, exit)?;
for (otype, source) in self.output_count.by_type.iter_typed() {
self.output_share.get_mut(otype).compute_count_ratio(
source,
crates/brk_computer/src/outputs/compute.rs
+2 -3
View File
@@ -3,14 +3,13 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes, inputs, price};
use crate::{blocks, inputs, price};
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
inputs: &inputs::Vecs,
blocks: &blocks::Vecs,
prices: &price::Vecs,
@@ -20,7 +19,7 @@ impl Vecs {
let starting_lengths = indexer.safe_lengths();
self.count.compute(indexer, indexes, blocks, exit)?;
self.count.compute(indexer, blocks, exit)?;
self.per_sec.compute(&self.count, &starting_lengths, exit)?;
self.value.compute(indexer, prices, exit)?;
self.by_type.compute(indexer, exit)?;
crates/brk_computer/src/outputs/count/compute.rs
+7 -8
View File
@@ -3,27 +3,26 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::{blocks, indexes};
use crate::blocks;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
blocks: &blocks::Vecs,
exit: &Exit,
) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let window_starts = blocks.lookback.window_starts();
self.total.compute(
self.total.sum.compute_count_from_indexes(
starting_height,
&indexes.tx_index.output_count,
&indexer.vecs.transactions.first_tx_index,
&indexes.height.tx_index_count,
&window_starts,
&indexer.vecs.outputs.first_txout_index,
&indexer.vecs.outputs.value,
exit,
0,
)?;
self.total
.compute_rest(starting_height, &window_starts, exit)?;
Ok(())
}
}
crates/brk_computer/src/price/compute.rs
+1 -1
View File
@@ -3,7 +3,7 @@ use std::ops::Range;
use brk_error::Result;
use brk_indexer::{Indexer, Lengths};
use brk_oracle::{
bin_to_cents, cents_to_bin, Config, Oracle, PaymentFilter, START_HEIGHT_FAST, START_HEIGHT_SLOW,
Config, Oracle, PaymentFilter, START_HEIGHT_FAST, START_HEIGHT_SLOW, bin_to_cents, cents_to_bin,
};
use brk_types::{Cents, OutputType, Sats, TxIndex, TxOutIndex};
use tracing::info;
crates/brk_computer/src/transactions/compute.rs
+6 -1
View File
@@ -19,7 +19,7 @@ impl Vecs {
self.db.sync_bg_tasks()?;
let (r1, (r2, r3)) = rayon::join(
|| self.count.compute(indexer, &blocks.lookback, exit),
|| self.count.compute(indexer, indexes, &blocks.lookback, exit),
|| {
rayon::join(
|| self.versions.compute(indexer, exit),
@@ -34,6 +34,11 @@ impl Vecs {
self.fees
.compute(indexer, indexes, &inputs.spent, &self.size, exit)?;
self.patterns
.compute(indexer, indexes, &inputs.spent, exit)?;
self.policy.compute(indexer, indexes, &self.fees, exit)?;
self.volume
.compute(indexer, indexes, prices, &self.count, &self.fees, exit)?;
crates/brk_computer/src/transactions/count/compute.rs
+5 -4
View File
@@ -3,12 +3,13 @@ use brk_indexer::Indexer;
use vecdb::Exit;
use super::Vecs;
use crate::blocks;
use crate::{blocks, indexes};
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
lookback: &blocks::LookbackVecs,
exit: &Exit,
) -> Result<()> {
@@ -17,10 +18,10 @@ impl Vecs {
let window_starts = lookback.window_starts();
self.total
.compute(starting_height, &window_starts, exit, |height| {
Ok(height.compute_count_from_indexes(
Ok(height.compute_transform(
starting_height,
&indexer.vecs.transactions.first_tx_index,
&indexer.vecs.transactions.txid,
&indexes.height.tx_index_count,
|(height, count, ..)| (height, count),
exit,
)?)
})?;
crates/brk_computer/src/transactions/fees/compute.rs
+361 -96
View File
@@ -1,6 +1,10 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{FeeRate, OutPoint, Sats, TxInIndex, TxIndex, VSize};
use brk_types::{
ChunkInput, CpfpClusterTxIndex, FeeRate, OutPoint, Sats, StoredBool, StoredU64, TxInIndex,
TxIndex, VSize, linearize,
};
use smallvec::SmallVec;
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec, unlikely};
use super::super::size;
@@ -70,7 +74,11 @@ impl Vecs {
let dep_version = self.input_value.version()
+ self.output_value.version()
+ size_vecs.vsize.tx_index.version();
+ size_vecs.vsize.tx_index.version()
+ indexer.vecs.inputs.outpoint.version()
+ indexer.vecs.transactions.first_tx_index.version()
+ indexer.vecs.transactions.first_txin_index.version()
+ indexes.height.tx_index_count.version();
self.fee
.tx_index
@@ -80,92 +88,135 @@ impl Vecs {
self.effective_fee_rate
.tx_index
.validate_computed_version_or_reset(dep_version)?;
self.is_cpfp_parent
.validate_computed_version_or_reset(dep_version)?;
self.is_cpfp_child
.validate_computed_version_or_reset(dep_version)?;
self.count
.cpfp_parent
.validate_computed_version_or_reset(dep_version)?;
self.count
.cpfp_child
.validate_computed_version_or_reset(dep_version)?;
let target = self
.input_value
.len()
.min(self.output_value.len())
.min(size_vecs.vsize.tx_index.len());
let min = self
.min(size_vecs.vsize.tx_index.len())
.min(starting_lengths.tx_index.to_usize());
let tx_len = self
.fee
.tx_index
.len()
.min(self.fee_rate.len())
.min(self.effective_fee_rate.tx_index.len())
.min(self.is_cpfp_parent.len())
.min(self.is_cpfp_child.len())
.min(starting_lengths.tx_index.to_usize());
if min >= target {
return Ok(());
}
self.fee
.tx_index
.truncate_if_needed(starting_lengths.tx_index)?;
self.fee_rate
.truncate_if_needed(starting_lengths.tx_index)?;
self.effective_fee_rate
.tx_index
.truncate_if_needed(starting_lengths.tx_index)?;
let start_tx = self.fee.tx_index.len();
let max_height = indexer.vecs.transactions.first_tx_index.len();
let start_height = if start_tx == 0 {
0
let max_height = indexer
.vecs
.transactions
.first_tx_index
.len()
.min(starting_lengths.height.to_usize())
.min(indexes.height.tx_index_count.len());
let next_height = if tx_len >= target {
max_height
} else {
indexes
.tx_heights
.get_shared(TxIndex::from(start_tx))
.get_shared(TxIndex::from(tx_len))
.unwrap()
.to_usize()
};
let count_len = self
.count
.cpfp_parent
.len()
.min(self.count.cpfp_child.len())
.min(max_height);
let start_height = count_len.min(next_height);
if start_height >= max_height {
return Ok(());
}
let start_tx = indexer
.vecs
.transactions
.first_tx_index
.collect_one_at(start_height)
.unwrap()
.to_usize();
self.fee
.tx_index
.truncate_if_needed(TxIndex::from(start_tx))?;
self.fee_rate.truncate_if_needed(TxIndex::from(start_tx))?;
self.effective_fee_rate
.tx_index
.truncate_if_needed(TxIndex::from(start_tx))?;
self.is_cpfp_parent
.truncate_if_needed(TxIndex::from(start_tx))?;
self.is_cpfp_child
.truncate_if_needed(TxIndex::from(start_tx))?;
self.count.cpfp_parent.truncate_if_needed_at(start_height)?;
self.count.cpfp_child.truncate_if_needed_at(start_height)?;
let mut tx_count = indexes.height.tx_index_count.cursor();
let mut next_block_input = indexer.vecs.inputs.first_txin_index.cursor();
tx_count.advance(start_height);
next_block_input.advance(start_height + 1);
let mut input_values = Vec::new();
let mut output_values = Vec::new();
let mut vsizes = Vec::new();
let mut txin_starts = Vec::new();
let mut outpoints = Vec::new();
let mut fees = Vec::new();
let mut cluster = Cluster::default();
let mut first_tx = start_tx;
for h in start_height..max_height {
let first_tx: usize = indexer
.vecs
.transactions
.first_tx_index
.collect_one_at(h)
.unwrap()
.to_usize();
let n = *indexes.height.tx_index_count.collect_one_at(h).unwrap() as usize;
let n = u64::from(tx_count.next().unwrap()) as usize;
if first_tx + n > target {
break;
}
// Batch read all per-tx data for this block
let input_values = self.input_value.collect_range_at(first_tx, first_tx + n);
let output_values = self.output_value.collect_range_at(first_tx, first_tx + n);
let vsizes: Vec<VSize> = size_vecs
self.input_value
.collect_range_into_at(first_tx, first_tx + n, &mut input_values);
self.output_value
.collect_range_into_at(first_tx, first_tx + n, &mut output_values);
size_vecs
.vsize
.tx_index
.collect_range_at(first_tx, first_tx + n);
let txin_starts: Vec<TxInIndex> = indexer
.collect_range_into_at(first_tx, first_tx + n, &mut vsizes);
indexer
.vecs
.transactions
.first_txin_index
.collect_range_at(first_tx, first_tx + n);
.collect_range_into_at(first_tx, first_tx + n, &mut txin_starts);
let input_begin = txin_starts[0].to_usize();
let input_end = if h + 1 < max_height {
next_block_input.next().unwrap().to_usize()
} else {
indexer
.vecs
.inputs
.first_txin_index
.collect_one_at(h + 1)
.unwrap()
.to_usize()
} else {
indexer.vecs.inputs.outpoint.len()
.outpoint
.len()
.min(starting_lengths.txin_index.to_usize())
};
let outpoints: Vec<OutPoint> = indexer
.vecs
.inputs
.outpoint
.collect_range_at(input_begin, input_end);
indexer.vecs.inputs.outpoint.collect_range_into_at(
input_begin,
input_end,
&mut outpoints,
);
// Compute fee + fee_rate per tx
let mut fees = Vec::with_capacity(n);
fees.clear();
fees.reserve(n);
for j in 0..n {
let fee = if unlikely(input_values[j].is_max()) {
Sats::ZERO
@@ -178,36 +229,56 @@ impl Vecs {
}
// Effective fee rate via same-block CPFP clustering
let effective = cluster_fee_rates(
cluster_fee_rates(
&txin_starts,
&outpoints,
input_begin,
first_tx,
&fees,
&vsizes,
&mut cluster,
);
for rate in effective {
self.effective_fee_rate.tx_index.push(rate);
let mut parent_count = 0;
let mut child_count = 0;
for ((&effective, &fee), &vsize) in cluster.rates.iter().zip(&fees).zip(&vsizes) {
let (is_parent, is_child) = cpfp_roles(effective, FeeRate::from((fee, vsize)));
parent_count += is_parent as u64;
child_count += is_child as u64;
self.effective_fee_rate.tx_index.push(effective);
self.is_cpfp_parent.push(StoredBool::from(is_parent));
self.is_cpfp_child.push(StoredBool::from(is_child));
}
self.count.cpfp_parent.push(StoredU64::from(parent_count));
self.count.cpfp_child.push(StoredU64::from(child_count));
if h % 1_000 == 0 {
let _lock = exit.lock();
self.fee.tx_index.write()?;
self.fee_rate.write()?;
self.effective_fee_rate.tx_index.write()?;
self.is_cpfp_parent.write()?;
self.is_cpfp_child.write()?;
self.count.cpfp_parent.write()?;
self.count.cpfp_child.write()?;
}
first_tx += n;
}
let _lock = exit.lock();
self.fee.tx_index.write()?;
self.fee_rate.write()?;
self.effective_fee_rate.tx_index.write()?;
self.is_cpfp_parent.write()?;
self.is_cpfp_child.write()?;
self.count.cpfp_parent.write()?;
self.count.cpfp_child.write()?;
Ok(())
}
}
/// Clusters same-block parent-child txs and computes effective fee rate per cluster.
/// Computes SFL chunk rates for each same-block dependency component.
fn cluster_fee_rates(
txin_starts: &[TxInIndex],
outpoints: &[OutPoint],
@@ -215,57 +286,251 @@ fn cluster_fee_rates(
first_tx: usize,
fees: &[Sats],
vsizes: &[VSize],
) -> Vec<FeeRate> {
cluster: &mut Cluster,
) {
let n = fees.len();
let mut parent: Vec<usize> = (0..n).collect();
cluster.rates.clear();
cluster.rates.extend(
fees.iter()
.zip(vsizes)
.map(|(&fee, &vsize)| FeeRate::from((fee, vsize))),
);
cluster.parents.clear();
cluster.parents.resize_with(n, SmallVec::new);
cluster.roots.clear();
cluster.roots.extend(0..n);
cluster.members.clear();
cluster.local_index.clear();
cluster.local_index.resize(n, usize::MAX);
for j in 1..n {
let start = txin_starts[j].to_usize() - outpoint_base;
let end = if j + 1 < txin_starts.len() {
txin_starts[j + 1].to_usize() - outpoint_base
} else {
outpoints.len()
};
for child in 0..n {
let mut parents: SmallVec<[usize; 2]> =
same_block_parents(child, txin_starts, outpoints, outpoint_base, first_tx, n).collect();
parents.sort_unstable();
parents.dedup();
for &parent in &parents {
union(&mut cluster.roots, child, parent);
}
cluster.parents[child] = parents;
}
for op in &outpoints[start..end] {
if op.is_coinbase() {
continue;
}
let parent_tx = op.tx_index().to_usize();
if parent_tx >= first_tx && parent_tx < first_tx + n {
union(&mut parent, j, parent_tx - first_tx);
}
for tx in 0..n {
cluster.members.push((root(&mut cluster.roots, tx), tx));
}
cluster.members.sort_unstable();
let mut start = 0;
while start < n {
let component_root = cluster.members[start].0;
let end = cluster.members[start..]
.partition_point(|&(candidate, _)| candidate == component_root)
+ start;
if end - start > 1 {
linearize_component(
&cluster.members[start..end],
&cluster.parents,
fees,
vsizes,
&mut cluster.rates,
&mut cluster.local_index,
&mut cluster.local_parents,
);
}
start = end;
}
}
#[allow(clippy::too_many_arguments)]
fn linearize_component(
members: &[(usize, usize)],
parents: &[SmallVec<[usize; 2]>],
fees: &[Sats],
vsizes: &[VSize],
rates: &mut [FeeRate],
local_index: &mut [usize],
local_parents: &mut Vec<SmallVec<[CpfpClusterTxIndex; 2]>>,
) {
for (local, &(_, tx)) in members.iter().enumerate() {
local_index[tx] = local;
}
local_parents.clear();
local_parents.extend(members.iter().map(|&(_, tx)| {
parents[tx]
.iter()
.map(|&parent| CpfpClusterTxIndex::from(local_index[parent] as u32))
.collect()
}));
let inputs: Vec<ChunkInput<'_>> = members
.iter()
.enumerate()
.map(|(local, &(_, tx))| ChunkInput {
fee: fees[tx],
vsize: vsizes[tx],
parents: local_parents[local].as_slice(),
})
.collect();
for chunk in linearize(&inputs) {
for local in chunk.txs {
rates[members[u32::from(local) as usize].1] = chunk.feerate;
}
}
let mut cluster_fee = vec![Sats::ZERO; n];
let mut cluster_vsize = vec![VSize::from(0u64); n];
for j in 0..n {
let root = find(&mut parent, j);
cluster_fee[root] += fees[j];
cluster_vsize[root] += vsizes[j];
}
(0..n)
.map(|j| {
let root = find(&mut parent, j);
FeeRate::from((cluster_fee[root], cluster_vsize[root]))
})
.collect()
}
fn find(parent: &mut [usize], mut i: usize) -> usize {
while parent[i] != i {
parent[i] = parent[parent[i]];
i = parent[i];
fn union(roots: &mut [usize], left: usize, right: usize) {
let left = root(roots, left);
let right = root(roots, right);
if left != right {
roots[right] = left;
}
i
}
fn union(parent: &mut [usize], a: usize, b: usize) {
let ra = find(parent, a);
let rb = find(parent, b);
if ra != rb {
parent[ra] = rb;
fn root(roots: &mut [usize], node: usize) -> usize {
let mut root = node;
while roots[root] != root {
root = roots[root];
}
let mut current = node;
while roots[current] != current {
let next = roots[current];
roots[current] = root;
current = next;
}
root
}
fn same_block_parents<'a>(
tx: usize,
txin_starts: &'a [TxInIndex],
outpoints: &'a [OutPoint],
outpoint_base: usize,
first_tx: usize,
tx_count: usize,
) -> impl Iterator<Item = usize> + 'a {
let start = txin_starts[tx].to_usize() - outpoint_base;
let end = txin_starts
.get(tx + 1)
.map_or(outpoints.len(), |index| index.to_usize() - outpoint_base);
outpoints[start..end].iter().filter_map(move |outpoint| {
let parent = outpoint.tx_index().to_usize();
(parent >= first_tx && parent < first_tx + tx_count).then_some(parent - first_tx)
})
}
fn cpfp_roles(effective: FeeRate, raw: FeeRate) -> (bool, bool) {
(effective > raw, effective < raw)
}
#[derive(Default)]
struct Cluster {
rates: Vec<FeeRate>,
parents: Vec<SmallVec<[usize; 2]>>,
roots: Vec<usize>,
members: Vec<(usize, usize)>,
local_index: Vec<usize>,
local_parents: Vec<SmallVec<[CpfpClusterTxIndex; 2]>>,
}
#[cfg(test)]
mod tests {
use brk_types::{FeeRate, OutPoint, Sats, TxInIndex, TxIndex, VSize, Vout};
use super::{Cluster, cluster_fee_rates, cpfp_roles};
#[test]
fn marks_actual_cpfp_roles() {
let mut cluster = Cluster::default();
cluster_fee_rates(
&[TxInIndex::from(0usize), TxInIndex::from(1usize)],
&[
OutPoint::COINBASE,
OutPoint::new(TxIndex::from(10usize), Vout::ZERO),
],
0,
10,
&[Sats::new(100), Sats::new(200)],
&[VSize::new(100), VSize::new(100)],
&mut cluster,
);
assert_eq!(cluster.rates, [FeeRate::new(1.5), FeeRate::new(1.5)]);
assert_eq!(
[
cpfp_roles(cluster.rates[0], FeeRate::new(1.0)),
cpfp_roles(cluster.rates[1], FeeRate::new(2.0)),
],
[(true, false), (false, true)]
);
}
#[test]
fn keeps_independent_transaction_rates_separate() {
let mut cluster = Cluster::default();
cluster_fee_rates(
&[TxInIndex::from(0usize), TxInIndex::from(1usize)],
&[OutPoint::COINBASE, OutPoint::COINBASE],
0,
10,
&[Sats::new(100), Sats::new(300)],
&[VSize::new(100), VSize::new(100)],
&mut cluster,
);
assert_eq!(cluster.rates, [FeeRate::new(1.0), FeeRate::new(3.0)]);
assert_eq!(
[
cpfp_roles(cluster.rates[0], FeeRate::new(1.0)),
cpfp_roles(cluster.rates[1], FeeRate::new(3.0)),
],
[(false, false), (false, false)]
);
}
#[test]
fn linearizes_shared_parent_branches_independently_of_sibling_order() {
let txin_starts = [
TxInIndex::from(0usize),
TxInIndex::from(1usize),
TxInIndex::from(2usize),
];
let outpoints = [
OutPoint::COINBASE,
OutPoint::new(TxIndex::from(10usize), Vout::ZERO),
OutPoint::new(TxIndex::from(10usize), Vout::ZERO),
];
let vsizes = [VSize::new(100); 3];
let mut cluster = Cluster::default();
cluster_fee_rates(
&txin_starts,
&outpoints,
0,
10,
&[Sats::ZERO, Sats::ZERO, Sats::new(3_000)],
&vsizes,
&mut cluster,
);
assert_eq!(
cluster.rates,
[FeeRate::new(15.0), FeeRate::new(0.0), FeeRate::new(15.0)]
);
cluster_fee_rates(
&txin_starts,
&outpoints,
0,
10,
&[Sats::ZERO, Sats::new(3_000), Sats::ZERO],
&vsizes,
&mut cluster,
);
assert_eq!(
cluster.rates,
[FeeRate::new(15.0), FeeRate::new(15.0), FeeRate::new(0.0)]
);
}
}
crates/brk_computer/src/transactions/fees/import.rs
+7 -1
View File
@@ -2,7 +2,7 @@ use brk_error::Result;
use brk_types::Version;
use vecdb::{Database, EagerVec, ImportableVec};
use super::Vecs;
use super::{CountVecs, Vecs};
use crate::{indexes, internal::PerTxDistribution};
/// Bump this when fee/feerate aggregation logic changes (e.g., skip coinbase, skip zero-fee).
@@ -16,6 +16,10 @@ impl Vecs {
) -> Result<Self> {
let v = version + VERSION;
Ok(Self {
count: CountVecs {
cpfp_parent: EagerVec::forced_import(db, "cpfp_parent_count", version)?,
cpfp_child: EagerVec::forced_import(db, "cpfp_child_count", version)?,
},
input_value: EagerVec::forced_import(db, "input_value", version)?,
output_value: EagerVec::forced_import(db, "output_value", version)?,
fee: PerTxDistribution::forced_import(db, "fee", v, indexes)?,
@@ -26,6 +30,8 @@ impl Vecs {
v,
indexes,
)?,
is_cpfp_parent: EagerVec::forced_import(db, "is_cpfp_parent", version)?,
is_cpfp_child: EagerVec::forced_import(db, "is_cpfp_child", version)?,
})
}
}
crates/brk_computer/src/transactions/fees/mod.rs
+1 -1
View File
@@ -2,4 +2,4 @@ mod compute;
mod import;
mod vecs;
pub use vecs::Vecs;
pub use vecs::{CountVecs, Vecs};
crates/brk_computer/src/transactions/fees/vecs.rs
+10 -1
View File
@@ -1,14 +1,23 @@
use brk_traversable::Traversable;
use brk_types::{FeeRate, Sats, TxIndex};
use brk_types::{FeeRate, Height, Sats, StoredBool, StoredU64, TxIndex};
use vecdb::{EagerVec, PcoVec, Rw, StorageMode};
use crate::internal::PerTxDistribution;
#[derive(Traversable)]
pub struct CountVecs<M: StorageMode = Rw> {
pub cpfp_parent: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
pub cpfp_child: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
}
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub count: CountVecs<M>,
pub input_value: M::Stored<EagerVec<PcoVec<TxIndex, Sats>>>,
pub output_value: M::Stored<EagerVec<PcoVec<TxIndex, Sats>>>,
pub fee: PerTxDistribution<Sats, M>,
pub fee_rate: M::Stored<EagerVec<PcoVec<TxIndex, FeeRate>>>,
pub effective_fee_rate: PerTxDistribution<FeeRate, M>,
pub is_cpfp_parent: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
pub is_cpfp_child: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
}
crates/brk_computer/src/transactions/import.rs
+7 -1
View File
@@ -12,7 +12,9 @@ use crate::{
},
};
use super::{CountVecs, FeesVecs, SizeVecs, Vecs, VersionsVecs, VolumeVecs};
use super::{
CountVecs, FeesVecs, PatternsVecs, PolicyVecs, SizeVecs, Vecs, VersionsVecs, VolumeVecs,
};
impl Vecs {
pub(crate) fn forced_import(
@@ -28,6 +30,8 @@ impl Vecs {
let count = CountVecs::forced_import(&db, version, indexes, cached_starts)?;
let size = SizeVecs::forced_import(&db, version, indexer, indexes)?;
let fees = FeesVecs::forced_import(&db, version, indexes)?;
let patterns = PatternsVecs::forced_import(&db, version)?;
let policy = PolicyVecs::forced_import(&db, version)?;
let versions = VersionsVecs::forced_import(&db, version, indexes, cached_starts)?;
let volume = VolumeVecs::forced_import(&db, version, indexes, cached_starts)?;
@@ -36,6 +40,8 @@ impl Vecs {
count,
size,
fees,
patterns,
policy,
versions,
volume,
};
crates/brk_computer/src/transactions/mod.rs
+6
View File
@@ -1,5 +1,7 @@
pub mod count;
pub mod fees;
pub mod patterns;
pub mod policy;
pub mod size;
pub mod versions;
pub mod volume;
@@ -12,6 +14,8 @@ use vecdb::{Database, Rw, StorageMode};
pub use count::Vecs as CountVecs;
pub use fees::Vecs as FeesVecs;
pub use patterns::Vecs as PatternsVecs;
pub use policy::Vecs as PolicyVecs;
pub use size::Vecs as SizeVecs;
pub use versions::Vecs as VersionsVecs;
pub use volume::Vecs as VolumeVecs;
@@ -26,6 +30,8 @@ pub struct Vecs<M: StorageMode = Rw> {
pub count: CountVecs<M>,
pub size: SizeVecs<M>,
pub fees: FeesVecs<M>,
pub patterns: PatternsVecs<M>,
pub policy: PolicyVecs<M>,
pub versions: VersionsVecs<M>,
pub volume: VolumeVecs<M>,
}
crates/brk_computer/src/transactions/patterns/coinjoin.rs
+127
View File
@@ -0,0 +1,127 @@
use brk_types::{OutputType, Sats, TypeIndex};
use rustc_hash::FxHashSet;
type Address = (OutputType, TypeIndex);
#[derive(Default)]
pub(super) struct Candidate {
values: FxHashSet<Sats>,
input_addresses: FxHashSet<Address>,
output_addresses: FxHashSet<Address>,
zero_values: usize,
address_reuse: bool,
}
impl Candidate {
pub(super) fn clear(&mut self) {
self.values.clear();
self.input_addresses.clear();
self.output_addresses.clear();
self.zero_values = 0;
self.address_reuse = false;
}
pub(super) fn add_input(
&mut self,
value: Sats,
output_type: OutputType,
type_index: TypeIndex,
) {
self.add_value(value);
if has_script_address(output_type) {
self.address_reuse |= !self.input_addresses.insert((output_type, type_index));
}
}
pub(super) fn add_output(
&mut self,
value: Sats,
output_type: OutputType,
type_index: TypeIndex,
) {
self.add_value(value);
if has_script_address(output_type) {
let address = (output_type, type_index);
self.address_reuse |=
self.input_addresses.contains(&address) || !self.output_addresses.insert(address);
}
}
pub(super) fn is_match(&self, input_count: usize, output_count: usize) -> bool {
!self.address_reuse
&& self.values.len() + self.zero_values <= (input_count + output_count) / 2
}
fn add_value(&mut self, value: Sats) {
if value.is_zero() {
self.zero_values += 1;
} else {
self.values.insert(value);
}
}
}
fn has_script_address(output_type: OutputType) -> bool {
matches!(
output_type,
OutputType::P2PK65
| OutputType::P2PK33
| OutputType::P2PKH
| OutputType::P2SH
| OutputType::P2WPKH
| OutputType::P2WSH
| OutputType::P2TR
| OutputType::P2A
)
}
#[cfg(test)]
mod tests {
use brk_types::{OutputType, Sats, TypeIndex};
use super::Candidate;
#[test]
fn repeated_values_match_without_address_reuse() {
let mut candidate = Candidate::default();
for index in 0usize..5 {
candidate.add_input(
Sats::new(10_000),
OutputType::P2WPKH,
TypeIndex::from(index),
);
candidate.add_output(
Sats::new(9_000),
OutputType::P2WPKH,
TypeIndex::from(index + 10),
);
}
assert!(candidate.is_match(5, 5));
}
#[test]
fn reused_output_address_does_not_match() {
let mut candidate = Candidate::default();
for _ in 0..5 {
candidate.add_output(
Sats::new(9_000),
OutputType::P2WPKH,
TypeIndex::from(1usize),
);
}
assert!(!candidate.is_match(5, 5));
}
#[test]
fn reused_input_address_does_not_match() {
let mut candidate = Candidate::default();
for _ in 0..5 {
candidate.add_input(
Sats::new(10_000),
OutputType::P2PK33,
TypeIndex::from(1usize),
);
}
assert!(!candidate.is_match(5, 5));
}
}
crates/brk_computer/src/transactions/patterns/compute.rs
+251
View File
@@ -0,0 +1,251 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{StoredBool, StoredU64, TxIndex};
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use super::{Vecs, coinjoin::Candidate};
use crate::{indexes, inputs};
const WRITE_INTERVAL: usize = 10_000;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
spent: &inputs::SpentVecs,
exit: &Exit,
) -> Result<()> {
let features = &indexer.vecs.transaction_features;
let version = indexes.tx_index.input_count.version()
+ indexes.tx_index.output_count.version()
+ indexer.vecs.transactions.first_tx_index.version()
+ indexer.vecs.transactions.first_txin_index.version()
+ indexer.vecs.transactions.first_txout_index.version()
+ spent.value.version()
+ indexer.vecs.inputs.output_type.version()
+ indexer.vecs.inputs.type_index.version()
+ indexer.vecs.outputs.value.version()
+ indexer.vecs.outputs.output_type.version()
+ indexer.vecs.outputs.type_index.version()
+ features.has_op_return.version()
+ features.has_inscription.version()
+ indexes.height.tx_index_count.version();
for vec in [
&mut self.is_coinjoin,
&mut self.is_consolidation,
&mut self.is_batch_payout,
] {
vec.validate_computed_version_or_reset(version)?;
}
for vec in [
&mut self.count.coinjoin,
&mut self.count.consolidation,
&mut self.count.batch_payout,
] {
vec.validate_computed_version_or_reset(version)?;
}
let lengths = indexer.safe_lengths();
let target_tx = lengths.tx_index.to_usize();
let target_height = lengths.height.to_usize();
let tx_len = self
.is_coinjoin
.len()
.min(self.is_consolidation.len())
.min(self.is_batch_payout.len())
.min(target_tx);
let count_len = self
.count
.coinjoin
.len()
.min(self.count.consolidation.len())
.min(self.count.batch_payout.len())
.min(target_height);
let start_height = count_len.min(next_height(indexes, tx_len, target_tx, target_height));
if start_height >= target_height {
return Ok(());
}
let first_tx = &indexer.vecs.transactions.first_tx_index;
let start_tx = first_tx.collect_one_at(start_height).unwrap().to_usize();
self.is_coinjoin.truncate_if_needed_at(start_tx)?;
self.is_consolidation.truncate_if_needed_at(start_tx)?;
self.is_batch_payout.truncate_if_needed_at(start_tx)?;
self.count.coinjoin.truncate_if_needed_at(start_height)?;
self.count
.consolidation
.truncate_if_needed_at(start_height)?;
self.count
.batch_payout
.truncate_if_needed_at(start_height)?;
let first_txin = indexer
.vecs
.transactions
.first_txin_index
.collect_one_at(start_tx)
.unwrap()
.to_usize();
let first_txout = indexer
.vecs
.transactions
.first_txout_index
.collect_one_at(start_tx)
.unwrap()
.to_usize();
let mut input_count = indexes.tx_index.input_count.cursor();
let mut output_count = indexes.tx_index.output_count.cursor();
let mut input_value = spent.value.cursor();
let mut input_type = indexer.vecs.inputs.output_type.cursor();
let mut input_type_index = indexer.vecs.inputs.type_index.cursor();
let mut output_value = indexer.vecs.outputs.value.cursor();
let mut output_type = indexer.vecs.outputs.output_type.cursor();
let mut output_type_index = indexer.vecs.outputs.type_index.cursor();
let mut has_op_return = features.has_op_return.cursor();
let mut has_inscription = features.has_inscription.cursor();
let mut tx_count = indexes.height.tx_index_count.cursor();
input_count.advance(start_tx);
output_count.advance(start_tx);
input_value.advance(first_txin);
input_type.advance(first_txin);
input_type_index.advance(first_txin);
output_value.advance(first_txout);
output_type.advance(first_txout);
output_type_index.advance(first_txout);
has_op_return.advance(start_tx);
has_inscription.advance(start_tx);
tx_count.advance(start_height);
let mut candidate = Candidate::default();
let mut block_start = start_tx;
for height in start_height..target_height {
let block_end = block_start + u64::from(tx_count.next().unwrap()) as usize;
let mut coinjoin_count = 0;
let mut consolidation_count = 0;
let mut batch_payout_count = 0;
for tx_index in block_start..block_end {
let inputs = usize::from(input_count.next().unwrap());
let outputs = usize::from(output_count.next().unwrap());
let op_return = has_op_return.next().unwrap().is_true();
let inscription = has_inscription.next().unwrap().is_true();
let token_related = op_return || inscription;
let consolidation = is_consolidation(inputs, outputs);
let batch_payout = is_batch_payout(inputs, outputs, tx_index == block_start);
let coinjoin_candidate = is_coinjoin_candidate(inputs, outputs, token_related);
let coinjoin = if coinjoin_candidate {
candidate.clear();
for _ in 0..inputs {
candidate.add_input(
input_value.next().unwrap(),
input_type.next().unwrap(),
input_type_index.next().unwrap(),
);
}
for _ in 0..outputs {
candidate.add_output(
output_value.next().unwrap(),
output_type.next().unwrap(),
output_type_index.next().unwrap(),
);
}
candidate.is_match(inputs, outputs)
} else {
input_value.advance(inputs);
input_type.advance(inputs);
input_type_index.advance(inputs);
output_value.advance(outputs);
output_type.advance(outputs);
output_type_index.advance(outputs);
false
};
coinjoin_count += coinjoin as u64;
consolidation_count += consolidation as u64;
batch_payout_count += batch_payout as u64;
self.is_coinjoin.push(StoredBool::from(coinjoin));
self.is_consolidation.push(StoredBool::from(consolidation));
self.is_batch_payout.push(StoredBool::from(batch_payout));
}
self.count.coinjoin.push(StoredU64::from(coinjoin_count));
self.count
.consolidation
.push(StoredU64::from(consolidation_count));
self.count
.batch_payout
.push(StoredU64::from(batch_payout_count));
if (height + 1).is_multiple_of(WRITE_INTERVAL) {
let _lock = exit.lock();
self.write()?;
}
block_start = block_end;
}
let _lock = exit.lock();
self.write()
}
fn write(&mut self) -> Result<()> {
self.is_coinjoin.write()?;
self.is_consolidation.write()?;
self.is_batch_payout.write()?;
self.count.coinjoin.write()?;
self.count.consolidation.write()?;
self.count.batch_payout.write()?;
Ok(())
}
}
fn next_height(
indexes: &indexes::Vecs,
tx_len: usize,
target_tx: usize,
target_height: usize,
) -> usize {
if tx_len >= target_tx {
target_height
} else {
indexes
.tx_heights
.get_shared(TxIndex::from(tx_len))
.unwrap()
.to_usize()
}
}
fn is_consolidation(inputs: usize, outputs: usize) -> bool {
inputs >= outputs * 5
}
fn is_batch_payout(inputs: usize, outputs: usize, is_coinbase: bool) -> bool {
!is_coinbase && outputs >= inputs * 5
}
fn is_coinjoin_candidate(inputs: usize, outputs: usize, token_related: bool) -> bool {
inputs >= 5 && outputs >= 5 && inputs < outputs * 5 && outputs < inputs * 5 && !token_related
}
#[cfg(test)]
mod tests {
use super::{is_batch_payout, is_coinjoin_candidate, is_consolidation};
#[test]
fn ratio_boundaries_match_filter_semantics() {
assert!(is_consolidation(25, 5));
assert!(is_batch_payout(5, 25, false));
assert!(!is_batch_payout(1, 5, true));
assert!(!is_coinjoin_candidate(25, 5, false));
assert!(!is_coinjoin_candidate(5, 25, false));
assert!(is_coinjoin_candidate(5, 5, false));
assert!(!is_coinjoin_candidate(5, 5, true));
}
}
crates/brk_computer/src/transactions/patterns/import.rs
+20
View File
@@ -0,0 +1,20 @@
use brk_error::Result;
use brk_types::Version;
use vecdb::{Database, EagerVec, ImportableVec};
use super::{CountVecs, Vecs};
impl Vecs {
pub(crate) fn forced_import(db: &Database, version: Version) -> Result<Self> {
Ok(Self {
count: CountVecs {
coinjoin: EagerVec::forced_import(db, "coinjoin_count", version)?,
consolidation: EagerVec::forced_import(db, "consolidation_count", version)?,
batch_payout: EagerVec::forced_import(db, "batch_payout_count", version)?,
},
is_coinjoin: EagerVec::forced_import(db, "is_coinjoin", version)?,
is_consolidation: EagerVec::forced_import(db, "is_consolidation", version)?,
is_batch_payout: EagerVec::forced_import(db, "is_batch_payout", version)?,
})
}
}
crates/brk_computer/src/transactions/patterns/mod.rs
+6
View File
@@ -0,0 +1,6 @@
mod coinjoin;
mod compute;
mod import;
mod vecs;
pub use vecs::{CountVecs, Vecs};
crates/brk_computer/src/transactions/patterns/vecs.rs
+18
View File
@@ -0,0 +1,18 @@
use brk_traversable::Traversable;
use brk_types::{Height, StoredBool, StoredU64, TxIndex};
use vecdb::{EagerVec, PcoVec, Rw, StorageMode};
#[derive(Traversable)]
pub struct CountVecs<M: StorageMode = Rw> {
pub coinjoin: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
pub consolidation: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
pub batch_payout: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
}
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub count: CountVecs<M>,
pub is_coinjoin: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
pub is_consolidation: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
pub is_batch_payout: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
}
crates/brk_computer/src/transactions/policy/compute.rs
+127
View File
@@ -0,0 +1,127 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{Sats, StoredBool, StoredU64, TxIndex};
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use super::Vecs;
use crate::{indexes, transactions::fees};
const FIRST_EPHEMERAL_DUST_HEIGHT: usize = 905_000;
const WRITE_INTERVAL: usize = 10_000;
impl Vecs {
pub(crate) fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
fees: &fees::Vecs,
exit: &Exit,
) -> Result<()> {
let features = &indexer.vecs.transaction_features;
let version = features.is_unconditionally_nonstandard.version()
+ features.has_dust_output.version()
+ fees.fee.tx_index.version()
+ indexer.vecs.transactions.first_tx_index.version()
+ indexes.height.tx_index_count.version();
self.is_nonstandard
.validate_computed_version_or_reset(version)?;
self.count.validate_computed_version_or_reset(version)?;
let lengths = indexer.safe_lengths();
let target_tx = lengths.tx_index.to_usize().min(fees.fee.tx_index.len());
let target_height = lengths.height.to_usize();
let tx_len = self.is_nonstandard.len().min(target_tx);
let count_len = self.count.len().min(target_height);
let next_height = if tx_len >= target_tx {
target_height
} else {
indexes
.tx_heights
.get_shared(TxIndex::from(tx_len))
.unwrap()
.to_usize()
};
let start_height = count_len.min(next_height);
if start_height >= target_height {
return Ok(());
}
let first_tx = &indexer.vecs.transactions.first_tx_index;
let start_tx = first_tx.collect_one_at(start_height).unwrap().to_usize();
self.is_nonstandard.truncate_if_needed_at(start_tx)?;
self.count.truncate_if_needed_at(start_height)?;
let mut unconditional = features.is_unconditionally_nonstandard.cursor();
let mut has_dust = features.has_dust_output.cursor();
let mut fee = fees.fee.tx_index.cursor();
let mut tx_count = indexes.height.tx_index_count.cursor();
unconditional.advance(start_tx);
has_dust.advance(start_tx);
fee.advance(start_tx);
tx_count.advance(start_height);
let mut block_start = start_tx;
for height in start_height..target_height {
let block_end =
(block_start + u64::from(tx_count.next().unwrap()) as usize).min(target_tx);
let mut count = 0;
for _ in block_start..block_end {
let raw = unconditional.next().unwrap().is_true();
let dust = has_dust.next().unwrap().is_true();
let nonstandard = if raw {
fee.advance(1);
true
} else if dust {
dust_is_nonstandard(height, fee.next().unwrap())
} else {
fee.advance(1);
false
};
count += nonstandard as u64;
self.is_nonstandard.push(StoredBool::from(nonstandard));
}
self.count.push(StoredU64::from(count));
if (height + 1).is_multiple_of(WRITE_INTERVAL) {
let _lock = exit.lock();
self.is_nonstandard.write()?;
self.count.write()?;
}
block_start = block_end;
}
let _lock = exit.lock();
self.is_nonstandard.write()?;
self.count.write()?;
Ok(())
}
}
fn dust_is_nonstandard(height: usize, fee: Sats) -> bool {
height < FIRST_EPHEMERAL_DUST_HEIGHT || fee != Sats::ZERO
}
#[cfg(test)]
mod tests {
use brk_types::Sats;
use super::{FIRST_EPHEMERAL_DUST_HEIGHT, dust_is_nonstandard};
#[test]
fn zero_fee_ephemeral_dust_starts_at_activation() {
assert!(dust_is_nonstandard(
FIRST_EPHEMERAL_DUST_HEIGHT - 1,
Sats::ZERO
));
assert!(!dust_is_nonstandard(
FIRST_EPHEMERAL_DUST_HEIGHT,
Sats::ZERO
));
assert!(dust_is_nonstandard(
FIRST_EPHEMERAL_DUST_HEIGHT,
Sats::new(1)
));
}
}
crates/brk_computer/src/transactions/policy/import.rs
+14
View File
@@ -0,0 +1,14 @@
use brk_error::Result;
use brk_types::Version;
use vecdb::{Database, EagerVec, ImportableVec};
use super::Vecs;
impl Vecs {
pub(crate) fn forced_import(db: &Database, version: Version) -> Result<Self> {
Ok(Self {
count: EagerVec::forced_import(db, "nonstandard_count", version)?,
is_nonstandard: EagerVec::forced_import(db, "is_nonstandard", version)?,
})
}
}
crates/brk_computer/src/transactions/policy/mod.rs
+5
View File
@@ -0,0 +1,5 @@
mod compute;
mod import;
mod vecs;
pub use vecs::Vecs;
crates/brk_computer/src/transactions/policy/vecs.rs
+9
View File
@@ -0,0 +1,9 @@
use brk_traversable::Traversable;
use brk_types::{Height, StoredBool, StoredU64, TxIndex};
use vecdb::{EagerVec, PcoVec, Rw, StorageMode};
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub count: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
pub is_nonstandard: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
}
crates/brk_computer/src/transactions/size/compute.rs
+1 -1
View File
@@ -14,7 +14,7 @@ impl Vecs {
) -> Result<()> {
let starting_lengths = indexer.safe_lengths();
self.vsize
self.weight
.derive_from(indexer, indexes, &starting_lengths, exit)?;
Ok(())
crates/brk_computer/src/transactions/size/import.rs
+16 -22
View File
@@ -1,12 +1,12 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{TxIndex, VSize, Version, Weight};
use vecdb::{Database, LazyVecFrom2, ReadableCloneableVec};
use brk_types::{Version, Weight};
use vecdb::{Database, LazyVecFrom1, ReadableCloneableVec};
use super::Vecs;
use crate::{
indexes,
internal::{LazyPerTxDistribution, LazyPerTxDistributionTransformed, VSizeToWeight},
internal::{Identity, LazyPerTxDistribution, LazyPerTxDistributionTransformed, WeightToVSize},
};
impl Vecs {
@@ -16,37 +16,31 @@ impl Vecs {
indexer: &Indexer,
indexes: &indexes::Vecs,
) -> Result<Self> {
let tx_index_to_vsize = LazyVecFrom2::init(
"tx_vsize",
let tx_index_to_weight = LazyVecFrom1::transformed::<Identity<Weight>>(
"tx_weight",
version,
indexer.vecs.transactions.base_size.read_only_boxed_clone(),
indexer.vecs.transactions.total_size.read_only_boxed_clone(),
|_index: TxIndex, base_size, total_size| {
VSize::from(Weight::from_sizes(*base_size, *total_size))
},
indexer.vecs.transactions.weight.read_only_boxed_clone(),
);
let vsize = LazyPerTxDistribution::forced_import(
let weight = LazyPerTxDistribution::forced_import(
db,
"tx_vsize",
"tx_weight",
version,
indexes,
tx_index_to_vsize,
tx_index_to_weight,
)?;
let tx_index_to_weight = LazyVecFrom2::init(
"tx_weight",
let tx_index_to_vsize = LazyVecFrom1::transformed::<WeightToVSize>(
"tx_vsize",
version,
indexer.vecs.transactions.base_size.read_only_boxed_clone(),
indexer.vecs.transactions.total_size.read_only_boxed_clone(),
|_index: TxIndex, base_size, total_size| Weight::from_sizes(*base_size, *total_size),
indexer.vecs.transactions.weight.read_only_boxed_clone(),
);
let weight = LazyPerTxDistributionTransformed::new::<VSizeToWeight>(
"tx_weight",
let vsize = LazyPerTxDistributionTransformed::new::<WeightToVSize>(
"tx_vsize",
version,
tx_index_to_weight,
&vsize.distribution,
tx_index_to_vsize,
&weight.distribution,
);
Ok(Self { vsize, weight })
crates/brk_computer/src/transactions/size/vecs.rs
+3 -3
View File
@@ -1,11 +1,11 @@
use brk_traversable::Traversable;
use brk_types::{StoredU32, VSize, Weight};
use brk_types::{VSize, Weight};
use vecdb::{Rw, StorageMode};
use crate::internal::{LazyPerTxDistribution, LazyPerTxDistributionTransformed};
#[derive(Traversable)]
pub struct Vecs<M: StorageMode = Rw> {
pub vsize: LazyPerTxDistribution<VSize, StoredU32, StoredU32, M>,
pub weight: LazyPerTxDistributionTransformed<Weight, StoredU32, StoredU32, VSize>,
pub vsize: LazyPerTxDistributionTransformed<VSize, Weight, Weight>,
pub weight: LazyPerTxDistribution<Weight, Weight, M>,
}
crates/brk_computer/src/transactions/versions/compute.rs
+18 -77
View File
@@ -1,90 +1,31 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::{StoredU64, TxVersion};
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
use vecdb::Exit;
use super::Vecs;
impl Vecs {
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
let starting_height = indexer.safe_lengths().height;
let lengths = indexer.safe_lengths();
let starting_height = lengths.height;
let counts = &indexer.vecs.transaction_features.count;
let dep_version = indexer.vecs.transactions.tx_version.version()
+ indexer.vecs.transactions.first_tx_index.version()
+ indexer.vecs.transactions.txid.version();
for vec in [&mut self.v1.block, &mut self.v2.block, &mut self.v3.block] {
vec.validate_and_truncate(dep_version, starting_height)?;
for (metrics, source) in [
(&mut self.v1, &counts.v1),
(&mut self.v2, &counts.v2),
(&mut self.v3, &counts.v3),
(&mut self.other, &counts.other_version),
] {
metrics.compute(starting_height, exit, |block| {
Ok(block.compute_transform(
starting_height,
source,
|(height, count, ..)| (height, count),
exit,
)?)
})?;
}
let skip = self
.v1
.block
.len()
.min(self.v2.block.len())
.min(self.v3.block.len());
let first_tx_index = &indexer.vecs.transactions.first_tx_index;
let end = first_tx_index.len();
if skip >= end {
return Ok(());
}
// Truncate all 3 to skip, then push (no per-element bounds checks).
self.v1.block.truncate_if_needed_at(skip)?;
self.v2.block.truncate_if_needed_at(skip)?;
self.v3.block.truncate_if_needed_at(skip)?;
// Single cursor over tx_version — scanned once for all 3 version counts.
let mut cursor = indexer.vecs.transactions.tx_version.cursor();
let fi_batch = first_tx_index.collect_range_at(skip, end);
let txid_len = indexer.vecs.transactions.txid.len();
for (j, first_index) in fi_batch.iter().enumerate() {
let next_first = fi_batch
.get(j + 1)
.map(|fi| fi.to_usize())
.unwrap_or(txid_len);
let mut c1: usize = 0;
let mut c2: usize = 0;
let mut c3: usize = 0;
let fi = first_index.to_usize();
cursor.advance(fi - cursor.position());
for _ in fi..next_first {
match cursor.next().unwrap() {
TxVersion::ONE => c1 += 1,
TxVersion::TWO => c2 += 1,
TxVersion::THREE => c3 += 1,
_ => {}
}
}
self.v1.block.push(StoredU64::from(c1 as u64));
self.v2.block.push(StoredU64::from(c2 as u64));
self.v3.block.push(StoredU64::from(c3 as u64));
if self.v1.block.batch_limit_reached() {
let _lock = exit.lock();
self.v1.block.write()?;
self.v2.block.write()?;
self.v3.block.write()?;
}
}
{
let _lock = exit.lock();
self.v1.block.write()?;
self.v2.block.write()?;
self.v3.block.write()?;
}
// Derive cumulative + sums from base
self.v1.compute_rest(starting_height, exit)?;
self.v2.compute_rest(starting_height, exit)?;
self.v3.compute_rest(starting_height, exit)?;
Ok(())
}
}
crates/brk_computer/src/transactions/versions/import.rs
+7
View File
@@ -37,6 +37,13 @@ impl Vecs {
indexes,
cached_starts,
)?,
other: PerBlockCumulativeRolling::forced_import(
db,
"tx_other_version",
version,
indexes,
cached_starts,
)?,
})
}
}
crates/brk_computer/src/transactions/versions/vecs.rs
+1
View File
@@ -9,4 +9,5 @@ pub struct Vecs<M: StorageMode = Rw> {
pub v1: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
pub v2: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
pub v3: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
pub other: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
}
crates/brk_indexer/src/constants.rs
+1 -1
View File
@@ -4,7 +4,7 @@ use brk_types::{TxIndex, Txid, TxidPrefix, Version};
// One version for all data sources
// Increment on **change _OR_ addition**
pub const VERSION: Version = Version::new(26);
pub const VERSION: Version = Version::new(27);
pub const SNAPSHOT_BLOCK_RANGE: usize = 1_000;
/// Known duplicate Bitcoin transactions (BIP30)
crates/brk_indexer/src/lengths.rs
+3 -4
View File
@@ -93,8 +93,7 @@ impl Lengths {
.p2ms
.first_index
.checked_push(height, self.p2ms_output_index)?;
vecs.scripts
.op_return
vecs.op_return
.first_index
.checked_push(height, self.op_return_index)?;
vecs.addrs
@@ -177,8 +176,8 @@ impl Lengths {
height,
)?,
op_return_index: next_index(
&vecs.scripts.op_return.first_index,
&vecs.scripts.op_return.to_tx_index,
&vecs.op_return.first_index,
&vecs.op_return.to_tx_index,
height,
)?,
p2pk33_addr_index: next_index(
crates/brk_indexer/src/lib.rs
+18 -19
View File
@@ -37,6 +37,7 @@ pub struct Indexer<M: StorageMode = Rw> {
path: PathBuf,
pub vecs: Vecs<M>,
pub stores: Stores,
buffers: BlockBuffers,
safe_lengths: SafeLengths,
}
@@ -53,7 +54,12 @@ impl<M: StorageMode> Indexer<M> {
/// actually queryable.
pub fn tip_blockhash(&self) -> BlockHash {
match self.safe_lengths().height.decremented() {
Some(h) => self.vecs.blocks.blockhash.collect_one(h).unwrap_or_default(),
Some(h) => self
.vecs
.blocks
.blockhash
.collect_one(h)
.unwrap_or_default(),
None => BlockHash::default(),
}
}
@@ -102,6 +108,7 @@ impl Indexer {
path: indexed_path.clone(),
vecs,
stores,
buffers: BlockBuffers::default(),
safe_lengths,
})
};
@@ -128,6 +135,7 @@ impl Indexer {
/// record that gets replayed on every recovery), this cleanly recreates.
fn full_reset(&mut self) -> Result<()> {
info!("Full reset...");
self.buffers.reset();
self.safe_lengths.reset();
self.vecs.reset()?;
let stores_path = self.path.join("stores");
@@ -196,6 +204,8 @@ impl Indexer {
debug!("Rollback vecs done.");
drop(lock);
self.buffers.continue_from(prev_hash);
let mut lengths = starting_lengths;
let is_export_height =
@@ -227,10 +237,10 @@ impl Indexer {
};
let mut readers = Readers::new(&self.vecs);
let mut buffers = BlockBuffers::default();
let vecs = &mut self.vecs;
let stores = &mut self.stores;
let buffers = &mut self.buffers;
for block in reader.after(prev_hash)?.iter() {
let block = match block {
@@ -277,38 +287,26 @@ impl Indexer {
processor.push_block_size_and_weight(&txs)?;
let (txins_result, txouts_result) = rayon::join(
|| processor.process_inputs(&txs, &mut buffers.txid_prefix_map),
|| processor.process_outputs(),
|| processor.process_inputs(&txs, &mut buffers.inputs),
|| processor.process_outputs(&mut buffers.addresses),
);
let txins = txins_result?;
let txouts = txouts_result?;
let tx_count = block.txdata.len();
let input_count = txins.len();
let output_count = txouts.len();
BlockProcessor::collect_same_block_spent_outpoints(
&txins,
&mut buffers.same_block_spent,
);
processor.check_txid_collisions(&txs)?;
let sigops = processor.compute_sigops(&txins, &txouts);
processor.finalize_and_store_metadata(
processor.analyze_and_finalize_transactions(
txs,
txouts,
txins,
sigops,
&buffers.same_block_spent,
&mut buffers.already_added_addrs,
&mut buffers.same_block_output_info,
&mut buffers.addresses,
)?;
processor
.lengths
.add_block(tx_count, input_count, output_count);
buffers.finish_block(*block.hash());
if is_export_height(height) {
drop(readers);
@@ -389,6 +387,7 @@ impl ReadOnlyClone for Indexer {
path: self.path.clone(),
vecs: self.vecs.read_only_clone(),
stores: self.stores.clone(),
buffers: BlockBuffers::default(),
safe_lengths: self.safe_lengths.clone(),
}
}
crates/brk_indexer/src/processor/metadata.rs → crates/brk_indexer/src/processor/block/mod.rs
+12 -11
View File
@@ -1,21 +1,22 @@
use brk_error::{Error, Result};
use brk_types::{BlockHashPrefix, Timestamp};
use tracing::error;
use vecdb::WritableVec;
use vecdb::{WritableVec, unlikely};
use super::{BlockProcessor, ComputedTx};
use super::{BlockProcessor, transaction::ComputedTx};
impl BlockProcessor<'_> {
pub fn process_block_metadata(&mut self) -> Result<()> {
pub(crate) fn process_block_metadata(&mut self) -> Result<()> {
let height = self.height;
let blockhash = self.block.hash();
let blockhash_prefix = BlockHashPrefix::from(blockhash);
if self
.stores
.blockhash_prefix_to_height
.get(&blockhash_prefix)?
.is_some_and(|prev_height| *prev_height != height)
if unlikely(self.check_collisions)
&& self
.stores
.blockhash_prefix_to_height
.get(&blockhash_prefix)?
.is_some_and(|prev_height| *prev_height != height)
{
error!("BlockHash: {blockhash}");
return Err(Error::Internal("BlockHash prefix collision"));
@@ -51,13 +52,13 @@ impl BlockProcessor<'_> {
/// Push block total_size and weight, reusing per-tx sizes already computed in ComputedTx.
/// This avoids redundant tx serialization (base_size + total_size were already computed).
pub fn push_block_size_and_weight(&mut self, txs: &[ComputedTx]) -> Result<()> {
pub(crate) fn push_block_size_and_weight(&mut self, txs: &[ComputedTx]) -> Result<()> {
let overhead = bitcoin::block::Header::SIZE + bitcoin::VarInt::from(txs.len()).size();
let mut total_size = overhead;
let mut weight = overhead * 4;
let mut weight = bitcoin::Weight::from_non_witness_data_size(overhead as u64);
let mut sw_txs = 0u32;
let mut sw_size = 0usize;
let mut sw_weight = 0usize;
let mut sw_weight = bitcoin::Weight::ZERO;
for (i, tx) in txs.iter().enumerate() {
total_size += tx.total_size as usize;
crates/brk_indexer/src/processor/buffer.rs
+29
View File
@@ -0,0 +1,29 @@
use brk_types::BlockHash;
use super::{txin::InputResolver, txout::BlockAddresses};
/// Reusable buffers cleared and refilled each block to avoid allocation churn.
#[derive(Default)]
pub(crate) struct BlockBuffers {
pub(crate) inputs: InputResolver,
pub(crate) addresses: BlockAddresses,
tip: Option<BlockHash>,
}
impl BlockBuffers {
pub(crate) fn continue_from(&mut self, parent: Option<BlockHash>) {
if self.tip != parent {
self.addresses.clear_cache();
}
self.tip = parent;
}
pub(crate) fn finish_block(&mut self, blockhash: BlockHash) {
self.tip = Some(blockhash);
}
pub(crate) fn reset(&mut self) {
self.addresses.clear_cache();
self.tip = None;
}
}
crates/brk_indexer/src/processor/mod.rs
+13 -80
View File
@@ -1,89 +1,22 @@
mod metadata;
mod sigops;
mod tx;
mod block;
mod buffer;
mod transaction;
mod txin;
mod txout;
mod types;
pub use types::*;
pub(crate) use buffer::BlockBuffers;
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_types::{AddrHash, Block, Height, OutPoint, SigOps, TxInIndex, TypeIndex};
use rustc_hash::{FxHashMap, FxHashSet};
use brk_types::{Block, Height};
use crate::{Lengths, Readers, Stores, Vecs};
/// Processes a single block, extracting and storing all indexed data.
pub struct BlockProcessor<'a> {
pub block: &'a Block,
pub height: Height,
pub check_collisions: bool,
pub lengths: &'a mut Lengths,
pub vecs: &'a mut Vecs,
pub stores: &'a mut Stores,
pub readers: &'a Readers,
}
impl BlockProcessor<'_> {
/// Finalizes outputs/inputs in parallel with storing tx metadata.
#[allow(clippy::too_many_arguments)]
pub fn finalize_and_store_metadata(
&mut self,
txs: Vec<ComputedTx>,
txouts: Vec<ProcessedOutput>,
txins: Vec<(TxInIndex, InputSource)>,
sigops: Vec<SigOps>,
same_block_spent_outpoints: &FxHashSet<OutPoint>,
already_added: &mut ByAddrType<FxHashMap<AddrHash, TypeIndex>>,
same_block_info: &mut FxHashMap<OutPoint, SameBlockOutputInfo>,
) -> Result<()> {
let lengths = &mut *self.lengths;
// Split transactions vecs: finalize needs first_txout_index/first_txin_index, metadata needs the rest
let (first_txout_index, first_txin_index, mut tx_metadata) =
self.vecs.transactions.split_for_finalize();
let outputs = &mut self.vecs.outputs;
let inputs = &mut self.vecs.inputs;
let addrs = &mut self.vecs.addrs;
let scripts = &mut self.vecs.scripts;
let addr_hash_stores = &mut self.stores.addr_type_to_addr_hash_to_addr_index;
let addr_tx_index_stores = &mut self.stores.addr_type_to_addr_index_and_tx_index;
let addr_outpoint_stores = &mut self.stores.addr_type_to_addr_index_and_unspent_outpoint;
let txid_prefix_store = &mut self.stores.txid_prefix_to_tx_index;
let (finalize_result, metadata_result) = rayon::join(
|| -> Result<()> {
txout::finalize_outputs(
lengths,
first_txout_index,
outputs,
addrs,
scripts,
addr_hash_stores,
addr_tx_index_stores,
addr_outpoint_stores,
txouts,
same_block_spent_outpoints,
already_added,
same_block_info,
)?;
txin::finalize_inputs(
first_txin_index,
inputs,
addr_tx_index_stores,
addr_outpoint_stores,
txins,
same_block_info,
)
},
|| tx::store_tx_metadata(txs, sigops, txid_prefix_store, &mut tx_metadata),
);
finalize_result?;
metadata_result?;
Ok(())
}
pub(crate) struct BlockProcessor<'a> {
pub(crate) block: &'a Block,
pub(crate) height: Height,
pub(crate) check_collisions: bool,
pub(crate) lengths: &'a mut Lengths,
pub(crate) vecs: &'a mut Vecs,
pub(crate) stores: &'a mut Stores,
pub(crate) readers: &'a Readers,
}
crates/brk_indexer/src/processor/sigops.rs
-182
View File
@@ -1,182 +0,0 @@
use bitcoin::{Script, script::Instruction};
use brk_types::{OutputType, SigOps, TxInIndex};
use rayon::prelude::*;
use super::{BlockProcessor, InputSource, ProcessedOutput};
impl BlockProcessor<'_> {
/// BIP-141 sigop cost per tx in the block. Mirrors
/// `bitcoin::Transaction::total_sigop_cost` but dispatches on each
/// input's prevout `OutputType` and each output's `OutputType`
/// (already resolved by `process_inputs`/`process_outputs`) instead
/// of round-tripping through bitcoin's closure API with
/// synthetic-prevout `ScriptBuf` allocations. The legacy-sigop walk
/// is short-circuited by `OutputType` for every script with a
/// canonical shape (~99% of outputs and ~95% of inputs on mainnet);
/// only `OpReturn`/`Unknown` outputs and non-segwit/non-P2SH inputs
/// fall back to a real script walk.
pub fn compute_sigops(
&self,
txins: &[(TxInIndex, InputSource)],
txouts: &[ProcessedOutput<'_>],
) -> Vec<SigOps> {
let txdata = &self.block.txdata;
let base_tx_index = u32::from(self.lengths.tx_index);
let mut tx_input_offsets = Vec::with_capacity(txdata.len());
let mut tx_output_offsets = Vec::with_capacity(txdata.len());
let mut input_offset = 0usize;
let mut output_offset = 0usize;
for tx in txdata {
tx_input_offsets.push(input_offset);
input_offset += tx.input.len();
tx_output_offsets.push(output_offset);
output_offset += tx.output.len();
}
txdata
.par_iter()
.enumerate()
.map(|(i, tx)| {
if tx.is_coinbase() {
return SigOps::ZERO;
}
let in_start = tx_input_offsets[i];
let tx_inputs = &txins[in_start..in_start + tx.input.len()];
let out_start = tx_output_offsets[i];
let tx_outputs = &txouts[out_start..out_start + tx.output.len()];
let mut legacy: usize = 0;
let mut redeem: usize = 0;
let mut witness: usize = 0;
for (input, (_, source)) in tx.input.iter().zip(tx_inputs.iter()) {
let prev_kind = match source {
InputSource::PreviousBlock { output_type, .. } => *output_type,
InputSource::SameBlock { outpoint, .. } => {
let local = (u32::from(outpoint.tx_index()) - base_tx_index) as usize;
let vout = u32::from(outpoint.vout()) as usize;
txouts[tx_output_offsets[local] + vout].output_type
}
};
// Single match per input: legacy script_sig sigops AND the
// redeem/witness contribution. Consensus enforces a
// push-only or empty script_sig in the four cases below
// (BIP-16 for P2SH from block 173805 onwards; BIP-141 /
// BIP-341 for segwit/taproot from activation), so legacy
// sigops are guaranteed 0 there. Everything else falls
// through to a real `count_sigops_legacy` walk.
match prev_kind {
OutputType::P2SH => {
// Faithful to bitcoin's count_p2sh_sigops + the
// nested-segwit branch of count_witness_sigops in
// a single script walk: redeem sigops use
// last_pushdata (no push-only check), wrapped
// witness sigops require both push-only and
// last_pushdata.
let (last_push, is_push_only) =
last_push_and_push_only(&input.script_sig);
let Some(redeem_bytes) = last_push else {
continue;
};
let rs = Script::from_bytes(redeem_bytes);
redeem = redeem.saturating_add(rs.count_sigops());
if !is_push_only {
continue;
}
if rs.is_p2wpkh() {
witness = witness.saturating_add(1);
} else if rs.is_p2wsh()
&& let Some(last) = input.witness.last()
{
witness =
witness.saturating_add(Script::from_bytes(last).count_sigops());
}
}
OutputType::P2WPKH => {
witness = witness.saturating_add(1);
}
OutputType::P2WSH => {
if let Some(last) = input.witness.last() {
witness =
witness.saturating_add(Script::from_bytes(last).count_sigops());
}
}
OutputType::P2TR => {}
_ => {
legacy = legacy.saturating_add(input.script_sig.count_sigops_legacy());
}
}
}
for processed in tx_outputs {
legacy = legacy.saturating_add(legacy_sigops_for_output(
processed.output_type,
&processed.txout.script_pubkey,
));
}
SigOps::from(
legacy
.saturating_mul(4)
.saturating_add(redeem.saturating_mul(4))
.saturating_add(witness),
)
})
.collect()
}
}
/// Legacy sigop count of a script_pubkey, dispatched on `OutputType`.
/// Every variant except `OpReturn` and `Unknown` has a canonical shape
/// recognised by `OutputType::from`'s exact byte-pattern matchers, so
/// the legacy sigop count is fixed: P2PKH and P2PK both end in a
/// single OP_CHECKSIG (1), P2MS contains one OP_CHECKMULTISIG counted
/// as 20 in legacy mode, and P2SH/P2WPKH/P2WSH/P2TR/P2A/Empty contain
/// no CHECKSIG-class opcodes outside their pushdata. `OpReturn`
/// payloads can include 0xac/0xae bytes outside a push, and `Unknown`
/// can be anything, so both fall back to a real script walk.
#[inline]
fn legacy_sigops_for_output(output_type: OutputType, script_pubkey: &Script) -> usize {
match output_type {
OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65 => 1,
OutputType::P2MS => 20,
OutputType::P2SH
| OutputType::P2WPKH
| OutputType::P2WSH
| OutputType::P2TR
| OutputType::P2A
| OutputType::Empty => 0,
OutputType::OpReturn | OutputType::Unknown => script_pubkey.count_sigops_legacy(),
}
}
/// Single-pass equivalent of bitcoin's private `last_pushdata()` plus the
/// public `is_push_only()`: returns the bytes of the script's last
/// `Instruction::PushBytes` (only when it is the *last* instruction)
/// alongside whether every instruction was a push (per Core,
/// `OP_RESERVED` and `OP_PUSHNUM_1..16` count as pushes too).
fn last_push_and_push_only(script: &Script) -> (Option<&[u8]>, bool) {
let mut last: Option<&[u8]> = None;
let mut push_only = true;
for inst in script.instructions() {
match inst {
Ok(Instruction::PushBytes(b)) => {
last = Some(b.as_bytes());
}
Ok(Instruction::Op(op)) => {
last = None;
if op.to_u8() > 0x60 {
push_only = false;
}
}
Err(_) => {
last = None;
push_only = false;
break;
}
}
}
(last, push_only)
}
crates/brk_indexer/src/processor/transaction/analysis/features/mod.rs
+19
View File
@@ -0,0 +1,19 @@
mod sighash;
use crate::TxFeatureFlags;
pub(super) fn scan_ecdsa_signature(bytes: &[u8], flags: &mut TxFeatureFlags) {
sighash::scan_ecdsa_signature(bytes, flags);
}
pub(super) fn scan_taproot_signature(bytes: &[u8], flags: &mut TxFeatureFlags) {
sighash::scan_taproot_signature(bytes, flags);
}
pub(super) fn record_validated_ecdsa_sighash(bytes: &[u8], flags: &mut TxFeatureFlags) {
sighash::record_validated_ecdsa_sighash(bytes, flags);
}
pub(super) fn record_validated_taproot_sighash(bytes: &[u8], flags: &mut TxFeatureFlags) {
sighash::record_validated_taproot_sighash(bytes, flags);
}
crates/brk_indexer/src/processor/transaction/analysis/features/sighash.rs
+131
View File
@@ -0,0 +1,131 @@
use bitcoin::{
secp256k1::{ecdsa::Signature as EcdsaSignature, schnorr::Signature as SchnorrSignature},
sighash::{EcdsaSighashType, TapSighashType},
};
use crate::TxFeatureFlags;
const MIN_ECDSA_SIGNATURE_BYTES: usize = 9;
const MAX_ECDSA_SIGNATURE_BYTES: usize = 73;
pub(super) fn scan_ecdsa_signature(bytes: &[u8], flags: &mut TxFeatureFlags) {
if !is_ecdsa_signature_candidate(bytes) {
return;
}
let (sighash_byte, der) = bytes.split_last().unwrap();
let Ok(sighash_type) = EcdsaSighashType::from_standard(u32::from(*sighash_byte)) else {
return;
};
let candidate_flags = ecdsa_feature_flags(sighash_type);
if flags.contains_all(candidate_flags) {
return;
}
if EcdsaSignature::from_der(der).is_err() {
return;
}
flags.insert(candidate_flags);
}
pub(super) fn scan_taproot_signature(bytes: &[u8], flags: &mut TxFeatureFlags) {
let (sighash_type, signature) = match bytes.len() {
64 => (TapSighashType::Default, bytes),
65 => {
let Ok(sighash_type) = TapSighashType::from_consensus_u8(bytes[64]) else {
return;
};
(sighash_type, &bytes[..64])
}
_ => return,
};
let candidate_flags = taproot_feature_flags(sighash_type);
if flags.contains_all(candidate_flags) {
return;
}
if SchnorrSignature::from_slice(signature).is_err() {
return;
}
flags.insert(candidate_flags);
}
pub(super) fn record_validated_ecdsa_sighash(bytes: &[u8], flags: &mut TxFeatureFlags) {
let Ok(sighash_type) = EcdsaSighashType::from_standard(u32::from(*bytes.last().unwrap()))
else {
return;
};
flags.insert(ecdsa_feature_flags(sighash_type));
}
pub(super) fn record_validated_taproot_sighash(bytes: &[u8], flags: &mut TxFeatureFlags) {
let sighash_type = match bytes.len() {
64 => TapSighashType::Default,
65 => {
let Ok(sighash_type) = TapSighashType::from_consensus_u8(bytes[64]) else {
return;
};
sighash_type
}
_ => return,
};
flags.insert(taproot_feature_flags(sighash_type));
}
fn ecdsa_feature_flags(sighash_type: EcdsaSighashType) -> u32 {
let mut flags = match sighash_type {
EcdsaSighashType::All | EcdsaSighashType::AllPlusAnyoneCanPay => {
TxFeatureFlags::SIGHASH_ALL
}
EcdsaSighashType::None | EcdsaSighashType::NonePlusAnyoneCanPay => {
TxFeatureFlags::SIGHASH_NONE
}
EcdsaSighashType::Single | EcdsaSighashType::SinglePlusAnyoneCanPay => {
TxFeatureFlags::SIGHASH_SINGLE
}
};
if sighash_type.to_u32() & 0x80 != 0 {
flags |= TxFeatureFlags::SIGHASH_ANYONE_CAN_PAY;
}
flags
}
#[inline]
fn is_ecdsa_signature_candidate(bytes: &[u8]) -> bool {
(MIN_ECDSA_SIGNATURE_BYTES..=MAX_ECDSA_SIGNATURE_BYTES).contains(&bytes.len())
&& bytes.first() == Some(&0x30)
}
fn taproot_feature_flags(sighash_type: TapSighashType) -> u32 {
let mut flags = match sighash_type {
TapSighashType::Default => TxFeatureFlags::SIGHASH_DEFAULT,
TapSighashType::All | TapSighashType::AllPlusAnyoneCanPay => TxFeatureFlags::SIGHASH_ALL,
TapSighashType::None | TapSighashType::NonePlusAnyoneCanPay => TxFeatureFlags::SIGHASH_NONE,
TapSighashType::Single | TapSighashType::SinglePlusAnyoneCanPay => {
TxFeatureFlags::SIGHASH_SINGLE
}
};
if matches!(
sighash_type,
TapSighashType::AllPlusAnyoneCanPay
| TapSighashType::NonePlusAnyoneCanPay
| TapSighashType::SinglePlusAnyoneCanPay
) {
flags |= TxFeatureFlags::SIGHASH_ANYONE_CAN_PAY;
}
flags
}
#[cfg(test)]
mod tests {
use super::is_ecdsa_signature_candidate;
#[test]
fn rejects_impossible_ecdsa_signature_encodings() {
assert!(!is_ecdsa_signature_candidate(&[0x30; 8]));
assert!(is_ecdsa_signature_candidate(&[0x30; 9]));
assert!(is_ecdsa_signature_candidate(&[0x30; 73]));
assert!(!is_ecdsa_signature_candidate(&[0x30; 74]));
assert!(!is_ecdsa_signature_candidate(&[0x02; 33]));
}
}
crates/brk_indexer/src/processor/transaction/analysis/input/mod.rs
+111
View File
@@ -0,0 +1,111 @@
mod redeem;
mod script_sig;
mod witness;
use bitcoin::{TxIn, taproot::LeafVersion};
use brk_types::OutputType;
use crate::TxFeatureFlags;
pub(super) struct Facts<'a> {
pub(super) script_sig: ScriptSigFacts<'a>,
redeem: redeem::Facts,
pub(super) witness: WitnessFacts<'a>,
}
impl Facts<'_> {
#[inline]
pub(super) fn redeem_sigops(&self) -> Option<usize> {
self.redeem.sigops()
}
#[inline]
pub(super) fn redeem_is_p2wpkh(&self) -> bool {
self.redeem.is_p2wpkh()
}
#[inline]
pub(super) fn redeem_is_p2wsh(&self) -> bool {
self.redeem.is_p2wsh()
}
#[inline]
pub(super) fn redeem_is_witness_program(&self) -> bool {
self.redeem.is_witness_program()
}
}
pub(super) struct ScriptSigFacts<'a> {
pub(super) accurate_sigops: usize,
pub(super) last_push: Option<&'a [u8]>,
pub(super) legacy_sigops: usize,
pub(super) push_only: bool,
}
impl ScriptSigFacts<'_> {
const EMPTY: Self = Self {
accurate_sigops: 0,
last_push: None,
legacy_sigops: 0,
push_only: true,
};
}
pub(super) struct WitnessFacts<'a> {
pub(super) has_annex: bool,
pub(super) last: Option<&'a [u8]>,
pub(super) leaf_version: Option<LeafVersion>,
pub(super) max_argument_bytes: usize,
pub(super) stack_items: usize,
}
pub(super) fn analyze<'a>(
input: &'a TxIn,
output_type: OutputType,
flags: &mut TxFeatureFlags,
) -> Facts<'a> {
flags.insert_type(output_type);
let script_sig = if input.script_sig.is_empty() {
ScriptSigFacts::EMPTY
} else {
script_sig::analyze(
&input.script_sig,
output_type,
input.witness.is_empty(),
flags,
)
};
let redeem = redeem::Facts::analyze(script_sig.last_push, output_type);
let witness = witness::analyze(
&input.witness,
redeem.effective_output_type(output_type, input.witness.len()),
flags,
);
Facts {
script_sig,
redeem,
witness,
}
}
#[cfg(test)]
mod tests {
use bitcoin::TxIn;
use brk_types::OutputType;
use super::analyze;
use crate::TxFeatureFlags;
#[test]
fn empty_script_sig_has_empty_facts() {
let input = TxIn::default();
let facts = analyze(&input, OutputType::Unknown, &mut TxFeatureFlags::default());
assert_eq!(facts.script_sig.accurate_sigops, 0);
assert_eq!(facts.script_sig.last_push, None);
assert_eq!(facts.script_sig.legacy_sigops, 0);
assert!(facts.script_sig.push_only);
}
}
crates/brk_indexer/src/processor/transaction/analysis/input/redeem.rs
+113
View File
@@ -0,0 +1,113 @@
use bitcoin::{Script, WitnessVersion};
use brk_types::OutputType;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Kind {
None,
P2WPKH,
P2WSH,
Witness,
Other,
}
#[derive(Clone, Copy)]
pub(super) struct Facts {
kind: Kind,
sigops: usize,
}
impl Facts {
pub(super) fn analyze(last_push: Option<&[u8]>, output_type: OutputType) -> Self {
if output_type != OutputType::P2SH {
return Self::NONE;
}
let Some(bytes) = last_push else {
return Self::NONE;
};
let script = Script::from_bytes(bytes);
let kind = match script.witness_version() {
Some(WitnessVersion::V0) if script.len() == 22 => Kind::P2WPKH,
Some(WitnessVersion::V0) if script.len() == 34 => Kind::P2WSH,
Some(_) => Kind::Witness,
None => Kind::Other,
};
Self {
kind,
sigops: script.count_sigops(),
}
}
pub(super) fn effective_output_type(
self,
output_type: OutputType,
witness_items: usize,
) -> OutputType {
match self.kind {
Kind::P2WPKH if witness_items == 2 => OutputType::P2WPKH,
_ => output_type,
}
}
pub(super) fn is_present(self) -> bool {
self.kind != Kind::None
}
pub(super) fn is_p2wpkh(self) -> bool {
self.kind == Kind::P2WPKH
}
pub(super) fn is_p2wsh(self) -> bool {
self.kind == Kind::P2WSH
}
pub(super) fn is_witness_program(self) -> bool {
matches!(self.kind, Kind::P2WPKH | Kind::P2WSH | Kind::Witness)
}
pub(super) fn sigops(self) -> Option<usize> {
self.is_present().then_some(self.sigops)
}
const NONE: Self = Self {
kind: Kind::None,
sigops: 0,
};
}
#[cfg(test)]
mod tests {
use bitcoin::ScriptBuf;
use brk_types::OutputType;
use super::{Facts, Kind};
fn classify(hex: &str) -> Facts {
let script = ScriptBuf::from_hex(hex).unwrap();
Facts::analyze(Some(script.as_bytes()), OutputType::P2SH)
}
#[test]
fn classifies_redeem_scripts_once() {
assert_eq!(
classify("00140000000000000000000000000000000000000000").kind,
Kind::P2WPKH
);
assert_eq!(
classify("00200000000000000000000000000000000000000000000000000000000000000000").kind,
Kind::P2WSH
);
assert_eq!(
classify("51200000000000000000000000000000000000000000000000000000000000000000").kind,
Kind::Witness
);
assert_eq!(classify("ac").kind, Kind::Other);
assert_eq!(classify("ac").sigops(), Some(1));
assert_eq!(Facts::analyze(None, OutputType::P2SH).kind, Kind::None);
assert_eq!(
Facts::analyze(Some(&[0xac]), OutputType::P2PKH).kind,
Kind::None
);
}
}
crates/brk_indexer/src/processor/transaction/analysis/input/script_sig.rs
+240
View File
@@ -0,0 +1,240 @@
use bitcoin::{
Script,
opcodes::all::{OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY, OP_CHECKSIG, OP_CHECKSIGVERIFY},
script::Instruction,
};
use brk_types::OutputType;
use super::{super::features, ScriptSigFacts};
use crate::TxFeatureFlags;
pub(super) fn analyze<'a>(
script: &'a Script,
output_type: OutputType,
scan_signatures: bool,
flags: &mut TxFeatureFlags,
) -> ScriptSigFacts<'a> {
if scan_signatures && let Some((signature, last_push)) = direct_push_spend(script, output_type)
{
features::record_validated_ecdsa_sighash(signature, flags);
return ScriptSigFacts {
accurate_sigops: 0,
last_push: Some(last_push),
legacy_sigops: 0,
push_only: true,
};
}
let mut accurate_sigops = 0;
let mut first_push = None;
let mut last_push = None;
let mut legacy_sigops = 0;
let mut only_push_bytes = true;
let mut push_count = 0;
let mut push_only = true;
let mut pushnum = None;
let validated_shape = scan_signatures
&& matches!(
output_type,
OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65
);
for instruction in script.instructions() {
match instruction {
Ok(Instruction::PushBytes(bytes)) => {
let bytes = bytes.as_bytes();
first_push.get_or_insert(bytes);
last_push = Some(bytes);
push_count += 1;
pushnum = None;
if scan_signatures && !validated_shape {
features::scan_ecdsa_signature(bytes, flags);
}
}
Ok(Instruction::Op(opcode)) => {
only_push_bytes = false;
last_push = None;
if opcode.to_u8() > 0x60 {
push_only = false;
}
match opcode {
OP_CHECKSIG | OP_CHECKSIGVERIFY => {
accurate_sigops += 1;
legacy_sigops += 1;
}
OP_CHECKMULTISIG | OP_CHECKMULTISIGVERIFY => {
accurate_sigops += pushnum.unwrap_or(20);
legacy_sigops += 20;
}
_ => pushnum = decode_pushnum(opcode.to_u8()),
}
}
Err(_) => {
only_push_bytes = false;
last_push = None;
push_only = false;
break;
}
}
}
if validated_shape {
let expected_pushes = match output_type {
OutputType::P2PKH => 2,
OutputType::P2PK33 | OutputType::P2PK65 => 1,
_ => unreachable!(),
};
if only_push_bytes && push_count == expected_pushes {
features::record_validated_ecdsa_sighash(first_push.unwrap(), flags);
} else {
scan_ecdsa_signatures(script, flags);
}
}
ScriptSigFacts {
accurate_sigops,
last_push,
legacy_sigops,
push_only,
}
}
fn direct_push_spend(script: &Script, output_type: OutputType) -> Option<(&[u8], &[u8])> {
let bytes = script.as_bytes();
let signature_len = usize::from(*bytes.first()?);
if !(1..=75).contains(&signature_len) {
return None;
}
let signature_end = 1 + signature_len;
let signature = bytes.get(1..signature_end)?;
match output_type {
OutputType::P2PK33 | OutputType::P2PK65 => {
(signature_end == bytes.len()).then_some((signature, signature))
}
OutputType::P2PKH => {
let public_key_len = usize::from(*bytes.get(signature_end)?);
if !matches!(public_key_len, 33 | 65) {
return None;
}
let public_key_start = signature_end + 1;
let public_key_end = public_key_start + public_key_len;
(public_key_end == bytes.len())
.then(|| (signature, &bytes[public_key_start..public_key_end]))
}
_ => None,
}
}
fn scan_ecdsa_signatures(script: &Script, flags: &mut TxFeatureFlags) {
for instruction in script.instructions() {
if let Ok(Instruction::PushBytes(bytes)) = instruction {
features::scan_ecdsa_signature(bytes.as_bytes(), flags);
}
}
}
#[inline]
fn decode_pushnum(opcode: u8) -> Option<usize> {
(0x51..=0x60)
.contains(&opcode)
.then(|| usize::from(opcode - 0x50))
}
#[cfg(test)]
mod tests {
use bitcoin::ScriptBuf;
use brk_types::OutputType;
use super::analyze;
use crate::TxFeatureFlags;
fn analyze_script(script: &ScriptBuf) -> super::ScriptSigFacts<'_> {
analyze(
script,
OutputType::Unknown,
false,
&mut TxFeatureFlags::default(),
)
}
#[test]
fn analyzes_last_push_and_push_only_together() {
let pushed = ScriptBuf::from_hex("03616263").unwrap();
let facts = analyze_script(&pushed);
assert_eq!(facts.last_push, Some(b"abc".as_slice()));
assert!(facts.push_only);
let followed_by_checksig = ScriptBuf::from_hex("03616263ac").unwrap();
let facts = analyze_script(&followed_by_checksig);
assert_eq!(facts.last_push, None);
assert!(!facts.push_only);
let push_number = ScriptBuf::from_hex("51").unwrap();
let facts = analyze_script(&push_number);
assert_eq!(facts.last_push, None);
assert!(facts.push_only);
}
#[test]
fn counts_legacy_and_accurate_sigops_together() {
let script = ScriptBuf::from_hex("52aeac").unwrap();
let facts = analyze_script(&script);
assert_eq!(facts.legacy_sigops, 21);
assert_eq!(facts.accurate_sigops, 3);
}
#[test]
fn records_validated_p2pkh_and_p2pk_sighashes() {
let signature = [0x30, 0x06, 0x02, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01];
let compressed_key = [0x02; 33];
let uncompressed_key = [0x04; 65];
let mut scripts = Vec::new();
for public_key in [compressed_key.as_slice(), uncompressed_key.as_slice()] {
let mut bytes = Vec::with_capacity(signature.len() + public_key.len() + 2);
bytes.push(signature.len() as u8);
bytes.extend(signature);
bytes.push(public_key.len() as u8);
bytes.extend(public_key);
scripts.push((ScriptBuf::from_bytes(bytes), OutputType::P2PKH, public_key));
}
let mut p2pk = Vec::with_capacity(signature.len() + 1);
p2pk.push(signature.len() as u8);
p2pk.extend(signature);
scripts.push((
ScriptBuf::from_bytes(p2pk),
OutputType::P2PK33,
signature.as_slice(),
));
for (script, output_type, last_push) in scripts {
let mut flags = TxFeatureFlags::default();
let facts = analyze(&script, output_type, true, &mut flags);
assert_eq!(facts.accurate_sigops, 0);
assert_eq!(facts.last_push, Some(last_push));
assert_eq!(facts.legacy_sigops, 0);
assert!(facts.push_only);
assert!(flags.is_set(TxFeatureFlags::SIGHASH_ALL));
}
}
#[test]
fn falls_back_for_noncanonical_p2pkh_shape() {
let script = ScriptBuf::from_hex(
"000930060201010201010121020000000000000000000000000000000000000000000000000000000000000000",
)
.unwrap();
let mut flags = TxFeatureFlags::default();
analyze(&script, OutputType::P2PKH, true, &mut flags);
assert!(flags.is_set(TxFeatureFlags::SIGHASH_ALL));
}
}
crates/brk_indexer/src/processor/transaction/analysis/input/witness.rs
+202
View File
@@ -0,0 +1,202 @@
use bitcoin::{
Script, Witness,
opcodes::all::OP_IF,
script::Instruction,
taproot::{LeafVersion, TAPROOT_ANNEX_PREFIX, TAPROOT_CONTROL_BASE_SIZE, TAPROOT_LEAF_MASK},
};
use brk_types::OutputType;
use super::{super::features, WitnessFacts};
use crate::TxFeatureFlags;
pub(super) fn analyze<'a>(
witness: &'a Witness,
output_type: OutputType,
flags: &mut TxFeatureFlags,
) -> WitnessFacts<'a> {
match output_type {
OutputType::P2TR => analyze_taproot(witness, flags),
OutputType::P2WPKH => analyze_p2wpkh(witness, flags),
_ => analyze_ecdsa(witness, flags),
}
}
fn analyze_p2wpkh<'a>(witness: &'a Witness, flags: &mut TxFeatureFlags) -> WitnessFacts<'a> {
let mut stack = witness.iter();
let signature = stack.next().unwrap();
let public_key = stack.next().unwrap();
debug_assert!(stack.next().is_none());
features::record_validated_ecdsa_sighash(signature, flags);
WitnessFacts {
has_annex: false,
last: Some(public_key),
leaf_version: None,
max_argument_bytes: signature.len(),
stack_items: 2,
}
}
fn analyze_ecdsa<'a>(witness: &'a Witness, flags: &mut TxFeatureFlags) -> WitnessFacts<'a> {
let stack_items = witness.len();
let mut last = None;
let mut max_argument_bytes = 0;
for (index, item) in witness.iter().enumerate() {
features::scan_ecdsa_signature(item, flags);
if index + 1 < stack_items {
max_argument_bytes = max_argument_bytes.max(item.len());
}
last = Some(item);
}
WitnessFacts {
has_annex: false,
last,
leaf_version: None,
max_argument_bytes,
stack_items,
}
}
fn analyze_taproot<'a>(witness: &'a Witness, flags: &mut TxFeatureFlags) -> WitnessFacts<'a> {
let len = witness.len();
if len == 1 {
let signature = witness.last().unwrap();
features::record_validated_taproot_sighash(signature, flags);
return WitnessFacts {
has_annex: false,
last: Some(signature),
leaf_version: None,
max_argument_bytes: signature.len(),
stack_items: 1,
};
}
let last = witness.last();
let has_annex = len > 1 && last.is_some_and(|item| item.first() == Some(&TAPROOT_ANNEX_PREFIX));
if has_annex {
flags.insert(TxFeatureFlags::ANNEX);
}
let stack_items = len - usize::from(has_annex);
let argument_count = if stack_items == 1 {
1
} else {
stack_items.saturating_sub(2)
};
let script_index = (stack_items >= 2).then(|| stack_items - 2);
let control_index = (stack_items >= 2).then(|| stack_items - 1);
let is_key_path = stack_items == 1;
let mut leaf_version = None;
let mut max_argument_bytes = 0;
for (index, item) in witness.iter().enumerate() {
if index < argument_count {
if is_key_path {
features::record_validated_taproot_sighash(item, flags);
} else {
features::scan_taproot_signature(item, flags);
}
max_argument_bytes = max_argument_bytes.max(item.len());
} else if Some(index) == script_index {
if has_inscription_envelope(Script::from_bytes(item)) {
flags.insert(TxFeatureFlags::INSCRIPTION);
}
} else if Some(index) == control_index && item.len() >= TAPROOT_CONTROL_BASE_SIZE {
leaf_version = LeafVersion::from_consensus(item[0] & TAPROOT_LEAF_MASK).ok();
}
}
WitnessFacts {
has_annex,
last,
leaf_version,
max_argument_bytes,
stack_items,
}
}
fn has_inscription_envelope(script: &Script) -> bool {
let mut state = 0;
for instruction in script.instructions() {
state = match (state, instruction) {
(0, Ok(Instruction::PushBytes(bytes))) if bytes.is_empty() => 1,
(1, Ok(Instruction::Op(OP_IF))) => 2,
(2, Ok(Instruction::PushBytes(bytes))) if bytes.as_bytes() == b"ord" => return true,
(_, Ok(Instruction::PushBytes(bytes))) if bytes.is_empty() => 1,
_ => 0,
};
}
false
}
#[cfg(test)]
mod tests {
use bitcoin::{ScriptBuf, TxIn, Witness, taproot::TAPROOT_ANNEX_PREFIX};
use brk_types::OutputType;
use super::{analyze, has_inscription_envelope};
use crate::TxFeatureFlags;
#[test]
fn recognizes_only_ord_envelopes() {
let inscription = ScriptBuf::from_hex("0063036f726468").unwrap();
let generic_envelope = ScriptBuf::from_hex("006303666f6f68").unwrap();
assert!(has_inscription_envelope(&inscription));
assert!(!has_inscription_envelope(&generic_envelope));
}
#[test]
fn reads_tapscript_before_control_block_and_annex() {
let script = ScriptBuf::from_hex("0063036f726468").unwrap();
let control_block = [0xc0; 33];
let annex = [TAPROOT_ANNEX_PREFIX, 0x01];
let input = TxIn {
witness: Witness::from_slice(&[
[0x01].as_slice(),
script.as_bytes(),
control_block.as_slice(),
annex.as_slice(),
]),
..TxIn::default()
};
let mut flags = TxFeatureFlags::default();
let facts = analyze(&input.witness, OutputType::P2TR, &mut flags);
assert!(facts.has_annex);
assert!(flags.is_set(TxFeatureFlags::ANNEX));
assert!(flags.is_set(TxFeatureFlags::INSCRIPTION));
}
#[test]
fn reads_fixed_shape_p2wpkh_once() {
let signature = [0x01; 71];
let public_key = [0x02; 33];
let witness = Witness::from_slice(&[signature.as_slice(), public_key.as_slice()]);
let mut flags = TxFeatureFlags::default();
let facts = analyze(&witness, OutputType::P2WPKH, &mut flags);
assert_eq!(facts.last, Some(public_key.as_slice()));
assert_eq!(facts.max_argument_bytes, signature.len());
assert_eq!(facts.stack_items, 2);
assert!(flags.is_set(TxFeatureFlags::SIGHASH_ALL));
}
#[test]
fn reads_taproot_key_path_once() {
let signature = [0x00; 64];
let witness = Witness::from_slice(&[signature]);
let mut flags = TxFeatureFlags::default();
let facts = analyze(&witness, OutputType::P2TR, &mut flags);
assert_eq!(facts.last, Some(signature.as_slice()));
assert_eq!(facts.max_argument_bytes, signature.len());
assert_eq!(facts.stack_items, 1);
assert!(flags.is_set(TxFeatureFlags::SIGHASH_DEFAULT));
}
}
crates/brk_indexer/src/processor/transaction/analysis/mod.rs
+98
View File
@@ -0,0 +1,98 @@
mod features;
mod input;
mod output;
mod policy;
mod sigops;
use brk_types::{OutputType, SigOps};
use rayon::prelude::*;
use vecdb::{likely, unlikely};
use super::ComputedTx;
use crate::TxFeatureFlags;
use crate::processor::{BlockProcessor, txin::InputSource, txout::ProcessedOutput};
pub(super) struct TransactionAnalysis {
pub(super) total_sigop_cost: SigOps,
pub(super) explicitly_rbf: bool,
pub(super) features: TxFeatureFlags,
}
impl BlockProcessor<'_> {
#[inline]
pub(crate) fn tracks_executed_legacy_sigops(&self) -> bool {
policy::tracks_executed_legacy_sigops(self.height)
}
pub(super) fn analyze_transactions(
&self,
txs: &[ComputedTx<'_>],
txins: &[InputSource],
txouts: &[ProcessedOutput],
) -> Vec<TransactionAnalysis> {
let block_first_tx_index = self.lengths.tx_index;
let track_executed_legacy = self.tracks_executed_legacy_sigops();
txs.par_iter()
.map(|tx| {
let tx_inputs = tx.inputs(txins);
let tx_outputs = tx.outputs(txouts);
let is_coinbase = tx.is_coinbase(block_first_tx_index);
let mut sigops = sigops::Accumulator::new(track_executed_legacy);
let mut flags = TxFeatureFlags::default();
let mut explicitly_rbf = false;
let mut output_scanner = output::Scanner::default();
let mut policy = policy::Accumulator::new(self.height);
if unlikely(is_coinbase) {
let input = &tx.tx.input[0];
explicitly_rbf = input.sequence.is_rbf();
sigops.scan_coinbase_input(input);
} else {
for (input, source) in tx.tx.input.iter().zip(tx_inputs) {
explicitly_rbf |= input.sequence.is_rbf();
let (output_type, legacy_sigops) = resolved_output_facts(source, txouts);
let facts = input::analyze(input, output_type, &mut flags);
sigops.scan_input(output_type, legacy_sigops, &facts);
policy.scan_input(input, output_type, &facts);
}
}
for (txout, output) in tx.tx.output.iter().zip(tx_outputs) {
output_scanner.scan(txout, output, &mut flags);
sigops.scan_output(txout, output);
if likely(!is_coinbase) {
policy.scan_output(txout, output);
}
}
let sigops = sigops.finish();
if likely(!is_coinbase) {
policy.finish(tx, sigops, &mut flags);
}
TransactionAnalysis {
total_sigop_cost: sigops.total,
explicitly_rbf,
features: flags,
}
})
.collect()
}
}
fn resolved_output_facts(source: &InputSource, txouts: &[ProcessedOutput]) -> (OutputType, SigOps) {
match source {
InputSource::PreviousBlock {
output_type,
legacy_sigops,
..
} => (*output_type, *legacy_sigops),
InputSource::SameBlock { txout_offset, .. } => {
let output = &txouts[*txout_offset];
(output.output_type, output.legacy_sigops)
}
InputSource::Coinbase => unreachable!(),
}
}
crates/brk_indexer/src/processor/transaction/analysis/output.rs
+79
View File
@@ -0,0 +1,79 @@
use crate::TxFeatureFlags;
use crate::processor::txout::ProcessedOutput;
use bitcoin::{PublicKey, TxOut, script::Instruction};
use brk_types::OutputType;
#[derive(Default)]
pub(super) struct Scanner {
p2wsh_count: usize,
payload_size: usize,
}
impl Scanner {
pub(super) fn scan(
&mut self,
txout: &TxOut,
output: &ProcessedOutput,
flags: &mut TxFeatureFlags,
) {
flags.insert_type(output.output_type);
match output.output_type {
OutputType::P2PK33 | OutputType::P2PK65 => {
if txout.script_pubkey.p2pk_public_key().is_none() {
flags.insert(TxFeatureFlags::FAKE_PUBKEY);
}
self.p2wsh_count = 0;
}
OutputType::P2MS => {
if has_fake_multisig_key(&txout.script_pubkey) {
flags.insert(TxFeatureFlags::FAKE_PUBKEY);
}
self.p2wsh_count = 0;
}
OutputType::P2WSH => {
let program = &txout.script_pubkey.as_bytes()[2..];
if self.p2wsh_count == 0 {
self.payload_size = u16::from_be_bytes([program[0], program[1]]) as usize;
}
self.p2wsh_count += 1;
let output_count = (self.payload_size + 33) / 32;
if self.p2wsh_count == output_count {
let padding = output_count * 32 - self.payload_size - 2;
if program[32 - padding..].iter().all(|byte| *byte == 0) {
flags.insert(TxFeatureFlags::FAKE_SCRIPTHASH);
}
}
}
_ => self.p2wsh_count = 0,
}
}
}
fn has_fake_multisig_key(script: &bitcoin::Script) -> bool {
script.instructions().any(|instruction| {
let Ok(Instruction::PushBytes(bytes)) = instruction else {
return false;
};
let bytes = bytes.as_bytes();
is_burn_key(bytes) || PublicKey::from_slice(bytes).is_err()
})
}
fn is_burn_key(bytes: &[u8]) -> bool {
bytes.len() == 33
&& matches!(
bytes,
[0x02, rest @ ..]
if rest.iter().all(|byte| *byte == 0x02)
|| rest.iter().all(|byte| *byte == 0x22)
)
|| bytes.len() == 33
&& matches!(
bytes,
[0x03, rest @ ..]
if rest.iter().all(|byte| *byte == 0x03)
|| rest.iter().all(|byte| *byte == 0x33)
)
}
crates/brk_indexer/src/processor/transaction/analysis/policy.rs
+384
View File
@@ -0,0 +1,384 @@
use bitcoin::{
Amount, Script, Transaction, TxIn, TxOut, WitnessVersion, policy::MAX_STANDARD_TX_SIGOPS_COST,
taproot::LeafVersion,
};
use brk_types::{Height, OutputType, SigOps};
use super::super::ComputedTx;
use super::{input, sigops::ComputedSigOps};
use crate::TxFeatureFlags;
use crate::processor::txout::ProcessedOutput;
// Deterministic policy snapshots, not consensus activation heights.
const LAST_V2_POLICY_HEIGHT: u32 = 863_500;
const FIRST_V29_POLICY_HEIGHT: u32 = 892_500;
const FIRST_V30_POLICY_HEIGHT: u32 = 921_000;
// rust-bitcoin 0.32 still exposes Core's previous 82-byte policy value.
const MIN_STANDARD_TX_NONWITNESS_SIZE: u32 = 65;
const MAX_EXECUTED_LEGACY_SIGOP_COST: u32 = 10_000;
const MAX_SCRIPT_SIG_SIZE: usize = 1_650;
const MAX_P2SH_SIGOPS: usize = 15;
const MAX_V29_OP_RETURN_SCRIPT_BYTES: usize = 83;
const MAX_V30_OP_RETURN_SCRIPT_BYTES: usize = 100_000;
const MAX_P2WSH_SCRIPT_BYTES: usize = 3_600;
const MAX_P2WSH_STACK_ITEMS: usize = 100;
const MAX_WITNESS_STACK_ITEM_BYTES: usize = 80;
const MAX_STANDARD_BARE_MULTISIG_SIGOP_COST: u32 =
3 * bitcoin::constants::WITNESS_SCALE_FACTOR as u32;
#[derive(Default)]
pub(super) struct Accumulator {
height: u32,
nonstandard: bool,
dust_output_count: usize,
op_return_count: usize,
op_return_script_bytes: usize,
}
impl Accumulator {
pub(super) fn new(height: Height) -> Self {
Self {
height: height.into(),
..Self::default()
}
}
pub(super) fn scan_input(
&mut self,
input: &TxIn,
output_type: OutputType,
facts: &input::Facts<'_>,
) {
if input.script_sig.len() > MAX_SCRIPT_SIG_SIZE || !facts.script_sig.push_only {
self.nonstandard = true;
return;
}
self.nonstandard |= match output_type {
OutputType::P2SH => facts
.redeem_sigops()
.is_some_and(|count| count > MAX_P2SH_SIGOPS),
OutputType::P2A => self.height <= LAST_V2_POLICY_HEIGHT,
OutputType::P2TR => facts.witness.has_annex,
OutputType::OpReturn | OutputType::Empty | OutputType::Unknown => true,
_ => false,
} || has_nonstandard_witness(output_type, facts);
}
pub(super) fn scan_output(&mut self, txout: &TxOut, output: &ProcessedOutput) {
let script = &txout.script_pubkey;
self.nonstandard |= match output.output_type {
OutputType::Empty => true,
OutputType::Unknown => !is_standard_unknown_witness(script),
OutputType::P2MS => has_too_many_bare_multisig_keys(output.legacy_sigops),
OutputType::OpReturn => {
self.op_return_count += 1;
self.op_return_script_bytes += script.len();
false
}
_ => false,
};
self.dust_output_count += is_dust(txout.value, output.output_type, script) as usize;
}
pub(super) fn finish(
mut self,
tx: &ComputedTx<'_>,
sigops: ComputedSigOps,
flags: &mut TxFeatureFlags,
) {
self.nonstandard |= if self.height < FIRST_V30_POLICY_HEIGHT {
self.op_return_count > 1 || self.op_return_script_bytes > MAX_V29_OP_RETURN_SCRIPT_BYTES
} else {
self.op_return_script_bytes > MAX_V30_OP_RETURN_SCRIPT_BYTES
};
self.nonstandard |=
has_unconditionally_nonstandard_dust(self.height, self.dust_output_count);
self.nonstandard |= has_nonstandard_header(tx, sigops, self.height);
if self.nonstandard {
flags.insert(TxFeatureFlags::UNCONDITIONALLY_NONSTANDARD);
}
if self.dust_output_count > 0 {
flags.insert(TxFeatureFlags::DUST_OUTPUT);
}
}
}
pub(super) fn tracks_executed_legacy_sigops(height: Height) -> bool {
u32::from(height) >= FIRST_V30_POLICY_HEIGHT
}
fn has_nonstandard_header(tx: &ComputedTx, sigops: ComputedSigOps, height: u32) -> bool {
has_nonstandard_version(tx.tx.version.0, height)
|| tx.weight() > Transaction::MAX_STANDARD_WEIGHT
|| tx.base_size < MIN_STANDARD_TX_NONWITNESS_SIZE
|| u32::from(sigops.total) > MAX_STANDARD_TX_SIGOPS_COST
|| height >= FIRST_V30_POLICY_HEIGHT
&& u32::from(sigops.executed_legacy) > MAX_EXECUTED_LEGACY_SIGOP_COST
}
fn has_unconditionally_nonstandard_dust(height: u32, dust_output_count: usize) -> bool {
if height < FIRST_V29_POLICY_HEIGHT {
dust_output_count > 0
} else {
dust_output_count > 1
}
}
fn is_dust(value: Amount, output_type: OutputType, script: &Script) -> bool {
let threshold = match output_type {
OutputType::P2PK65 => 672,
OutputType::P2PK33 => 576,
OutputType::P2PKH => 546,
OutputType::P2MS | OutputType::Unknown => {
return value < script.minimal_non_dust();
}
OutputType::P2SH => 540,
OutputType::OpReturn => return false,
OutputType::P2WPKH => 294,
OutputType::P2WSH | OutputType::P2TR => 330,
OutputType::P2A => 240,
OutputType::Empty => 471,
};
value.to_sat() < threshold
}
fn has_nonstandard_witness(output_type: OutputType, facts: &input::Facts<'_>) -> bool {
if facts.witness.stack_items == 0 {
return false;
}
match output_type {
OutputType::P2A => true,
OutputType::P2WPKH => false,
OutputType::P2WSH => has_nonstandard_p2wsh_witness(&facts.witness),
OutputType::P2TR => {
facts.witness.has_annex || has_nonstandard_taproot_witness(&facts.witness)
}
OutputType::P2SH => {
if facts.redeem_sigops().is_none() {
return true;
}
if facts.redeem_is_p2wsh() {
has_nonstandard_p2wsh_witness(&facts.witness)
} else {
!facts.redeem_is_witness_program()
}
}
_ => true,
}
}
fn has_nonstandard_p2wsh_witness(witness: &input::WitnessFacts<'_>) -> bool {
let stack_items = witness.stack_items - 1;
witness.last.unwrap().len() > MAX_P2WSH_SCRIPT_BYTES
|| stack_items > MAX_P2WSH_STACK_ITEMS
|| witness.max_argument_bytes > MAX_WITNESS_STACK_ITEM_BYTES
}
fn has_nonstandard_taproot_witness(witness: &input::WitnessFacts<'_>) -> bool {
if witness.stack_items < 2 {
return false;
}
witness.leaf_version.is_none()
|| witness.leaf_version == Some(LeafVersion::TapScript)
&& witness.max_argument_bytes > MAX_WITNESS_STACK_ITEM_BYTES
}
fn has_nonstandard_version(version: i32, height: u32) -> bool {
let max = if height <= LAST_V2_POLICY_HEIGHT {
2
} else {
3
};
!(1..=max).contains(&version)
}
fn is_standard_unknown_witness(script: &Script) -> bool {
script
.witness_version()
.is_some_and(|version| version != WitnessVersion::V0)
}
fn has_too_many_bare_multisig_keys(sigops: SigOps) -> bool {
u32::from(sigops) > MAX_STANDARD_BARE_MULTISIG_SIGOP_COST
}
#[cfg(test)]
mod tests {
use bitcoin::{Amount, ScriptBuf, TxIn, Witness};
use brk_types::{
AddrBytes, Height, OutputType, P2ABytes, P2PK33Bytes, P2PK65Bytes, P2PKHBytes, P2SHBytes,
P2TRBytes, P2WPKHBytes, P2WSHBytes, SigOps,
};
use super::super::input;
use super::{
FIRST_V29_POLICY_HEIGHT, FIRST_V30_POLICY_HEIGHT, LAST_V2_POLICY_HEIGHT,
has_nonstandard_p2wsh_witness, has_nonstandard_taproot_witness, has_nonstandard_version,
has_nonstandard_witness, has_too_many_bare_multisig_keys,
has_unconditionally_nonstandard_dust, is_dust, is_standard_unknown_witness,
tracks_executed_legacy_sigops,
};
#[test]
fn tracks_executed_sigops_only_when_policy_uses_them() {
assert!(!tracks_executed_legacy_sigops(Height::from(
FIRST_V30_POLICY_HEIGHT - 1
)));
assert!(tracks_executed_legacy_sigops(Height::from(
FIRST_V30_POLICY_HEIGHT
)));
}
#[test]
fn accepts_only_policy_transaction_versions() {
assert!(has_nonstandard_version(0, LAST_V2_POLICY_HEIGHT));
assert!(has_nonstandard_version(-1, LAST_V2_POLICY_HEIGHT));
assert!(!has_nonstandard_version(1, LAST_V2_POLICY_HEIGHT));
assert!(!has_nonstandard_version(2, LAST_V2_POLICY_HEIGHT));
assert!(has_nonstandard_version(3, LAST_V2_POLICY_HEIGHT));
assert!(!has_nonstandard_version(3, LAST_V2_POLICY_HEIGHT + 1));
}
#[test]
fn applies_ephemeral_dust_policy_from_v29() {
assert!(!has_unconditionally_nonstandard_dust(
FIRST_V29_POLICY_HEIGHT - 1,
0
));
assert!(has_unconditionally_nonstandard_dust(
FIRST_V29_POLICY_HEIGHT - 1,
1
));
assert!(!has_unconditionally_nonstandard_dust(
FIRST_V29_POLICY_HEIGHT,
1
));
assert!(has_unconditionally_nonstandard_dust(
FIRST_V29_POLICY_HEIGHT,
2
));
}
#[test]
fn uses_exact_dust_thresholds_for_fixed_scripts() {
let scripts = [
AddrBytes::from(P2PK65Bytes::from(&[0; 65][..])).to_script_pubkey(),
AddrBytes::from(P2PK33Bytes::from(&[0; 33][..])).to_script_pubkey(),
AddrBytes::from(P2PKHBytes::from(&[0; 20][..])).to_script_pubkey(),
AddrBytes::from(P2SHBytes::from(&[0; 20][..])).to_script_pubkey(),
AddrBytes::from(P2WPKHBytes::from(&[0; 20][..])).to_script_pubkey(),
AddrBytes::from(P2WSHBytes::from(&[0; 32][..])).to_script_pubkey(),
AddrBytes::from(P2TRBytes::from(&[0; 32][..])).to_script_pubkey(),
AddrBytes::from(P2ABytes::from(&[0; 2][..])).to_script_pubkey(),
];
let thresholds = [672, 576, 546, 540, 294, 330, 330, 240];
for ((output_type, script), threshold) in OutputType::ADDR_TYPES
.into_iter()
.zip(&scripts)
.zip(thresholds)
{
assert_eq!(script.minimal_non_dust().to_sat(), threshold);
assert!(is_dust(
Amount::from_sat(threshold - 1),
output_type,
script
));
assert!(!is_dust(Amount::from_sat(threshold), output_type, script));
}
let empty = ScriptBuf::new();
assert_eq!(empty.minimal_non_dust().to_sat(), 471);
assert!(is_dust(Amount::from_sat(470), OutputType::Empty, &empty));
assert!(!is_dust(Amount::from_sat(471), OutputType::Empty, &empty));
}
#[test]
fn keeps_exact_dust_calculation_for_unknown_scripts() {
let script = ScriptBuf::from_bytes(vec![0x61; 1_000]);
let threshold = script.minimal_non_dust().to_sat();
assert!(is_dust(
Amount::from_sat(threshold - 1),
OutputType::Unknown,
&script
));
assert!(!is_dust(
Amount::from_sat(threshold),
OutputType::Unknown,
&script
));
}
#[test]
fn recognizes_future_witness_programs() {
assert!(is_standard_unknown_witness(
&ScriptBuf::from_hex(
"52200000000000000000000000000000000000000000000000000000000000000000"
)
.unwrap()
));
assert!(!is_standard_unknown_witness(
&ScriptBuf::from_hex(
"00200000000000000000000000000000000000000000000000000000000000000000"
)
.unwrap()
));
}
#[test]
fn limits_standard_bare_multisig_to_three_keys() {
assert!(!has_too_many_bare_multisig_keys(SigOps::new(12)));
assert!(has_too_many_bare_multisig_keys(SigOps::new(16)));
}
#[test]
fn rejects_p2a_witness_stuffing() {
let input = TxIn {
witness: Witness::from_slice(&[b"stuffing"]),
..TxIn::default()
};
let facts = input::analyze(
&input,
OutputType::P2A,
&mut crate::TxFeatureFlags::default(),
);
assert!(has_nonstandard_witness(OutputType::P2A, &facts,));
}
#[test]
fn enforces_witness_stack_item_limits() {
let oversized = [0_u8; 81];
let input = TxIn {
witness: Witness::from_slice(&[oversized.as_slice(), [0x51].as_slice()]),
..TxIn::default()
};
let facts = input::analyze(
&input,
OutputType::P2WSH,
&mut crate::TxFeatureFlags::default(),
);
assert!(has_nonstandard_p2wsh_witness(&facts.witness));
let control_block = [0xc0_u8; 33];
let input = TxIn {
witness: Witness::from_slice(&[
oversized.as_slice(),
[0x51].as_slice(),
control_block.as_slice(),
]),
..TxIn::default()
};
let facts = input::analyze(
&input,
OutputType::P2TR,
&mut crate::TxFeatureFlags::default(),
);
assert!(has_nonstandard_taproot_witness(&facts.witness));
}
}
crates/brk_indexer/src/processor/transaction/analysis/sigops.rs
+192
View File
@@ -0,0 +1,192 @@
use bitcoin::{Script, TxIn, TxOut, constants::WITNESS_SCALE_FACTOR};
use brk_types::{OutputType, SigOps};
use crate::processor::txout::ProcessedOutput;
use super::input;
#[derive(Default)]
pub(super) struct Accumulator {
legacy: usize,
redeem: usize,
witness: usize,
executed_legacy: usize,
track_executed_legacy: bool,
}
impl Accumulator {
pub(super) fn new(track_executed_legacy: bool) -> Self {
Self {
track_executed_legacy,
..Self::default()
}
}
pub(super) fn scan_input(
&mut self,
prev_kind: OutputType,
prev_legacy_sigops: SigOps,
facts: &input::Facts<'_>,
) {
if !facts.script_sig.push_only {
self.legacy = self.legacy.saturating_add(facts.script_sig.legacy_sigops);
}
if self.track_executed_legacy {
if !facts.script_sig.push_only {
self.executed_legacy = self.executed_legacy.saturating_add(
facts
.script_sig
.accurate_sigops
.saturating_mul(WITNESS_SCALE_FACTOR),
);
}
if prev_kind != OutputType::P2SH {
self.executed_legacy = self
.executed_legacy
.saturating_add(u32::from(prev_legacy_sigops) as usize);
}
}
match prev_kind {
OutputType::P2SH => {
let Some(redeem_sigops) = facts.redeem_sigops() else {
return;
};
self.redeem = self.redeem.saturating_add(redeem_sigops);
if self.track_executed_legacy {
self.executed_legacy = self
.executed_legacy
.saturating_add(redeem_sigops.saturating_mul(WITNESS_SCALE_FACTOR));
}
if !facts.script_sig.push_only {
return;
}
if facts.redeem_is_p2wpkh() {
self.witness = self.witness.saturating_add(1);
} else if facts.redeem_is_p2wsh()
&& let Some(last) = facts.witness.last
{
self.witness = self
.witness
.saturating_add(Script::from_bytes(last).count_sigops());
}
}
OutputType::P2WPKH => self.witness = self.witness.saturating_add(1),
OutputType::P2WSH => {
if let Some(last) = facts.witness.last {
self.witness = self
.witness
.saturating_add(Script::from_bytes(last).count_sigops());
}
}
OutputType::P2TR => {}
_ => {}
}
}
pub(super) fn scan_coinbase_input(&mut self, input: &TxIn) {
self.legacy = self
.legacy
.saturating_add(input.script_sig.count_sigops_legacy());
}
pub(super) fn scan_output(&mut self, txout: &TxOut, output: &ProcessedOutput) {
self.legacy = self
.legacy
.saturating_add(legacy_sigops_for_output(output, &txout.script_pubkey));
}
pub(super) fn finish(self) -> ComputedSigOps {
ComputedSigOps {
total: SigOps::from(
self.legacy
.saturating_mul(WITNESS_SCALE_FACTOR)
.saturating_add(self.redeem.saturating_mul(WITNESS_SCALE_FACTOR))
.saturating_add(self.witness),
),
executed_legacy: SigOps::from(self.executed_legacy),
}
}
}
#[derive(Clone, Copy, Default)]
pub(super) struct ComputedSigOps {
pub(super) total: SigOps,
pub(super) executed_legacy: SigOps,
}
/// Legacy sigop count of a script_pubkey, dispatched on `OutputType`.
/// Every variant except `OpReturn` and `Unknown` has a canonical shape
/// recognised by `OutputType::from`'s exact byte-pattern matchers, so
/// the legacy sigop count is fixed: P2PKH and P2PK both end in a
/// single OP_CHECKSIG (1), P2MS contains one OP_CHECKMULTISIG counted
/// as 20 in legacy mode, and P2SH/P2WPKH/P2WSH/P2TR/P2A/Empty contain
/// no CHECKSIG-class opcodes outside their pushdata. `OpReturn`
/// payloads can include 0xac/0xae bytes outside a push, and `Unknown`
/// can be anything, so both fall back to a real script walk.
#[inline]
fn legacy_sigops_for_output(output: &ProcessedOutput, script_pubkey: &Script) -> usize {
match output.output_type {
OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65 => 1,
OutputType::P2MS => 20,
OutputType::P2SH
| OutputType::P2WPKH
| OutputType::P2WSH
| OutputType::P2TR
| OutputType::P2A
| OutputType::Empty => 0,
OutputType::OpReturn => output.op_return_legacy_sigops(),
OutputType::Unknown => script_pubkey.count_sigops_legacy(),
}
}
#[cfg(test)]
mod tests {
use bitcoin::{ScriptBuf, TxIn};
use brk_types::{OutputType, SigOps};
use super::{Accumulator, input};
use crate::TxFeatureFlags;
#[test]
fn push_only_script_sig_keeps_prevout_executed_sigops() {
let input = TxIn {
script_sig: ScriptBuf::from_hex("00").unwrap(),
..TxIn::default()
};
let facts = input::analyze(&input, OutputType::P2PKH, &mut TxFeatureFlags::default());
let mut accumulator = Accumulator::new(true);
accumulator.scan_input(OutputType::P2PKH, SigOps::new(4), &facts);
assert_eq!(u32::from(accumulator.finish().executed_legacy), 4);
}
#[test]
fn counts_coinbase_legacy_script_sigops() {
let input = TxIn {
script_sig: ScriptBuf::from_hex("ac").unwrap(),
..TxIn::default()
};
let mut accumulator = Accumulator::new(false);
accumulator.scan_coinbase_input(&input);
assert_eq!(u32::from(accumulator.finish().total), 4);
}
#[test]
fn counts_legacy_script_sigops_for_known_prevout_types() {
let input = TxIn {
script_sig: ScriptBuf::from_hex("ac").unwrap(),
..TxIn::default()
};
let facts = input::analyze(&input, OutputType::P2TR, &mut TxFeatureFlags::default());
let mut accumulator = Accumulator::new(false);
accumulator.scan_input(OutputType::P2TR, SigOps::ZERO, &facts);
assert_eq!(u32::from(accumulator.finish().total), 4);
}
}
crates/brk_indexer/src/processor/transaction/computed.rs
+82
View File
@@ -0,0 +1,82 @@
use bitcoin::Transaction;
use brk_types::{TxIndex, Txid, TxidPrefix, Vout};
pub(crate) struct ComputedTx<'a> {
pub(crate) tx_index: TxIndex,
pub(crate) tx: &'a Transaction,
pub(crate) txid: Txid,
pub(crate) insert_txid_prefix: bool,
pub(crate) base_size: u32,
pub(crate) total_size: u32,
input_offset: usize,
output_offset: usize,
}
impl ComputedTx<'_> {
pub(super) fn new(
tx_index: TxIndex,
tx: &Transaction,
txid: Txid,
insert_txid_prefix: bool,
base_size: u32,
total_size: u32,
) -> ComputedTx<'_> {
ComputedTx {
tx_index,
tx,
txid,
insert_txid_prefix,
base_size,
total_size,
input_offset: 0,
output_offset: 0,
}
}
pub(super) fn set_block_offsets(txs: &mut [Self]) {
let mut input_offset = 0;
let mut output_offset = 0;
for tx in txs {
tx.input_offset = input_offset;
tx.output_offset = output_offset;
input_offset += tx.tx.input.len();
output_offset += tx.tx.output.len();
}
}
#[inline]
pub(super) fn inputs<'a, T>(&self, inputs: &'a [T]) -> &'a [T] {
&inputs[self.input_offset..self.input_offset + self.tx.input.len()]
}
#[inline]
pub(super) fn outputs<'a, T>(&self, outputs: &'a [T]) -> &'a [T] {
&outputs[self.output_offset..self.output_offset + self.tx.output.len()]
}
#[inline]
pub(crate) fn txout_offset(&self, vout: Vout) -> usize {
self.output_offset + usize::from(vout)
}
#[inline]
pub(crate) fn is_coinbase(&self, block_first_tx_index: TxIndex) -> bool {
self.tx_index == block_first_tx_index
}
#[inline]
pub(crate) fn txid_prefix(&self) -> TxidPrefix {
TxidPrefix::from(&self.txid)
}
#[inline]
pub(crate) fn is_segwit(&self) -> bool {
self.base_size != self.total_size
}
#[inline]
pub(crate) fn weight(&self) -> bitcoin::Weight {
brk_types::Weight::from_sizes(self.base_size, self.total_size).into()
}
}
crates/brk_indexer/src/processor/transaction/mod.rs
+216
View File
@@ -0,0 +1,216 @@
mod analysis;
mod computed;
use brk_error::{Error, Result};
use brk_store::Store;
use brk_types::{Height, StoredBool, TxIndex, TxVersion, Txid, TxidPrefix};
use rayon::prelude::*;
use tracing::error;
use vecdb::{AnyVec, WritableVec, likely, unlikely};
use crate::constants::DUPLICATE_TXIDS;
use crate::{TransactionCounts, TransactionFeaturesVecs, TxMetadataVecs};
pub(super) use computed::ComputedTx;
use self::analysis::TransactionAnalysis;
use super::{
BlockProcessor,
txin::{self, InputSource},
txout::{self, BlockAddresses, ProcessedOutput},
};
impl<'a> BlockProcessor<'a> {
pub(crate) fn compute_txids(&self) -> Result<Vec<ComputedTx<'a>>> {
let will_check_collisions = self.check_collisions;
let base_tx_index = self.lengths.tx_index;
let mut txs = self
.block
.txdata
.par_iter()
.enumerate()
.map(|(index, tx)| {
let (btc_txid, base_size, total_size) = self.block.compute_tx_id_and_sizes(index);
let txid = Txid::from(btc_txid);
let tx_index = base_tx_index + TxIndex::from(index);
let insert_txid_prefix = if likely(!will_check_collisions) {
true
} else {
let txid_prefix = TxidPrefix::from(&txid);
let prev_tx_index = self
.stores
.txid_prefix_to_tx_index
.get(&txid_prefix)?
.map(|value| *value);
if let Some(prev_tx_index) = prev_tx_index {
self.validate_txid_collision(tx_index, prev_tx_index)?;
false
} else {
true
}
};
Ok(ComputedTx::new(
tx_index,
tx,
txid,
insert_txid_prefix,
base_size,
total_size,
))
})
.collect::<Result<Vec<_>>>()?;
ComputedTx::set_block_offsets(&mut txs);
Ok(txs)
}
fn validate_txid_collision(&self, tx_index: TxIndex, prev_tx_index: TxIndex) -> Result<()> {
if tx_index == prev_tx_index {
return Ok(());
}
let len = self.vecs.transactions.txid.len();
let prev_txid = self
.vecs
.transactions
.txid
.get_pushed_or_read(prev_tx_index, &self.readers.txid)
.ok_or(Error::Internal("Missing txid for tx_index"))
.inspect_err(|_| {
error!(?tx_index, len, "Missing txid for tx_index");
})?;
if unlikely(!DUPLICATE_TXIDS.contains(&prev_txid)) {
error!(
height = ?self.height,
?tx_index,
?prev_txid,
?prev_tx_index,
"Unexpected TXID collision"
);
return Err(Error::Internal("Unexpected TXID collision"));
}
Ok(())
}
/// Analyzes transactions, then finalizes outputs/inputs in parallel with metadata storage.
#[allow(clippy::too_many_arguments)]
pub(crate) fn analyze_and_finalize_transactions(
&mut self,
txs: Vec<ComputedTx>,
mut txouts: Vec<ProcessedOutput>,
txins: Vec<InputSource>,
addresses: &mut BlockAddresses,
) -> Result<()> {
let transaction_analyses = self.analyze_transactions(&txs, &txins, &txouts);
let lengths = &mut *self.lengths;
let base_tx_index = lengths.tx_index;
let base_txin_index = lengths.txin_index;
let transactions = &self.block.txdata;
// Split transactions vecs: finalize needs first_txout_index/first_txin_index, metadata needs the rest
let (first_txout_index, first_txin_index, mut tx_metadata) =
self.vecs.transactions.split_for_finalize();
let outputs = &mut self.vecs.outputs;
let inputs = &mut self.vecs.inputs;
let addrs = &mut self.vecs.addrs;
let scripts = &mut self.vecs.scripts;
let op_return = &mut self.vecs.op_return;
let transaction_features = &mut self.vecs.transaction_features;
let height = self.height;
let addr_hash_stores = &mut self.stores.addr_type_to_addr_hash_to_addr_index;
let addr_tx_index_stores = &mut self.stores.addr_type_to_addr_index_and_tx_index;
let addr_outpoint_stores = &mut self.stores.addr_type_to_addr_index_and_unspent_outpoint;
let txid_prefix_store = &mut self.stores.txid_prefix_to_tx_index;
let (finalize_result, metadata_result) = rayon::join(
|| -> Result<()> {
txout::finalize_outputs(
transactions,
base_tx_index,
lengths,
first_txout_index,
outputs,
addrs,
scripts,
op_return,
addr_hash_stores,
addr_tx_index_stores,
addr_outpoint_stores,
&mut txouts,
addresses,
)?;
txin::finalize_inputs(
transactions,
base_tx_index,
base_txin_index,
first_txin_index,
inputs,
addr_tx_index_stores,
addr_outpoint_stores,
txins,
&txouts,
)
},
|| {
store_tx_metadata(
height,
txs,
transaction_analyses,
txid_prefix_store,
&mut tx_metadata,
transaction_features,
)
},
);
finalize_result?;
metadata_result?;
Ok(())
}
}
fn store_tx_metadata(
height: Height,
txs: Vec<ComputedTx>,
transaction_analyses: Vec<TransactionAnalysis>,
store: &mut Store<TxidPrefix, TxIndex>,
md: &mut TxMetadataVecs<'_>,
features: &mut TransactionFeaturesVecs,
) -> Result<()> {
debug_assert_eq!(txs.len(), transaction_analyses.len());
let mut counts = TransactionCounts::default();
for (ct, analysis) in txs.into_iter().zip(transaction_analyses) {
if likely(ct.insert_txid_prefix) {
store.insert(ct.txid_prefix(), ct.tx_index);
}
let tx_version = TxVersion::from(ct.tx.version);
md.tx_version.checked_push(ct.tx_index, tx_version)?;
md.txid.checked_push(ct.tx_index, ct.txid)?;
md.raw_locktime
.checked_push(ct.tx_index, ct.tx.lock_time.into())?;
md.weight.checked_push(ct.tx_index, ct.weight().into())?;
md.total_size
.checked_push(ct.tx_index, ct.total_size.into())?;
md.total_sigop_cost
.checked_push(ct.tx_index, analysis.total_sigop_cost)?;
md.is_explicitly_rbf
.checked_push(ct.tx_index, StoredBool::from(analysis.explicitly_rbf))?;
counts.add_base(
ct.tx.input.len(),
ct.tx.output.len(),
tx_version,
analysis.explicitly_rbf,
);
features.push_and_count(analysis.features, &mut counts);
}
features.count.checked_push(height, counts)?;
Ok(())
}
crates/brk_indexer/src/processor/tx.rs
-116
View File
@@ -1,116 +0,0 @@
use brk_error::{Error, Result};
use brk_store::Store;
use brk_types::{SigOps, StoredBool, TxIndex, Txid, TxidPrefix};
use rayon::prelude::*;
use tracing::error;
use vecdb::{AnyVec, WritableVec, likely};
use crate::TxMetadataVecs;
use crate::constants::DUPLICATE_TXIDS;
use super::{BlockProcessor, ComputedTx};
impl<'a> BlockProcessor<'a> {
pub fn compute_txids(&self) -> Result<Vec<ComputedTx<'a>>> {
let will_check_collisions = self.check_collisions;
let base_tx_index = self.lengths.tx_index;
self.block
.txdata
.par_iter()
.enumerate()
.map(|(index, tx)| {
let (btc_txid, base_size, total_size) = self.block.compute_tx_id_and_sizes(index);
let txid = Txid::from(btc_txid);
let txid_prefix = TxidPrefix::from(&txid);
let prev_tx_index_opt = if will_check_collisions {
self.stores
.txid_prefix_to_tx_index
.get(&txid_prefix)?
.map(|v| *v)
} else {
None
};
Ok(ComputedTx {
tx_index: base_tx_index + TxIndex::from(index),
tx,
txid,
txid_prefix,
prev_tx_index_opt,
base_size,
total_size,
})
})
.collect()
}
/// Only for known duplicate TXIDs (BIP-30).
pub fn check_txid_collisions(&self, txs: &[ComputedTx]) -> Result<()> {
if likely(!self.check_collisions) {
return Ok(());
}
for ct in txs.iter() {
let Some(prev_tx_index) = ct.prev_tx_index_opt else {
continue;
};
if ct.tx_index == prev_tx_index {
continue;
}
let len = self.vecs.transactions.txid.len();
let prev_txid = self
.vecs
.transactions
.txid
.get_pushed_or_read(prev_tx_index, &self.readers.txid)
.ok_or(Error::Internal("Missing txid for tx_index"))
.inspect_err(|_| {
error!(tx_index = ?ct.tx_index, len, "Missing txid for tx_index");
})?;
let is_dup = DUPLICATE_TXIDS.contains(&prev_txid);
if !is_dup {
error!(
height = ?self.height, tx_index = ?ct.tx_index,
?prev_txid, ?prev_tx_index,
"Unexpected TXID collision"
);
return Err(Error::Internal("Unexpected TXID collision"));
}
}
Ok(())
}
}
pub(super) fn store_tx_metadata(
txs: Vec<ComputedTx>,
sigops: Vec<SigOps>,
store: &mut Store<TxidPrefix, TxIndex>,
md: &mut TxMetadataVecs<'_>,
) -> Result<()> {
debug_assert_eq!(txs.len(), sigops.len());
for (ct, sigops) in txs.into_iter().zip(sigops) {
if ct.prev_tx_index_opt.is_none() {
store.insert(ct.txid_prefix, ct.tx_index);
}
md.tx_version
.checked_push(ct.tx_index, ct.tx.version.into())?;
md.txid.checked_push(ct.tx_index, ct.txid)?;
md.raw_locktime
.checked_push(ct.tx_index, ct.tx.lock_time.into())?;
md.base_size
.checked_push(ct.tx_index, ct.base_size.into())?;
md.total_size
.checked_push(ct.tx_index, ct.total_size.into())?;
md.total_sigop_cost.checked_push(ct.tx_index, sigops)?;
md.is_explicitly_rbf
.checked_push(ct.tx_index, StoredBool::from(ct.tx.is_explicitly_rbf()))?;
}
Ok(())
}
crates/brk_indexer/src/processor/txin.rs
-222
View File
@@ -1,222 +0,0 @@
use brk_cohort::ByAddrType;
use brk_error::{Error, Result};
use brk_store::Store;
use brk_types::{
AddrIndexOutPoint, AddrIndexTxIndex, OutPoint, OutputType, TxInIndex, TxIndex, Txid,
TxidPrefix, TypeIndex, Unit, Vin, Vout,
};
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::error;
use vecdb::{PcoVec, WritableVec};
use super::{BlockProcessor, ComputedTx, InputSource, SameBlockOutputInfo};
use crate::InputsVecs;
impl<'a> BlockProcessor<'a> {
pub fn process_inputs(
&self,
txs: &[ComputedTx],
txid_prefix_to_tx_index: &mut FxHashMap<TxidPrefix, TxIndex>,
) -> Result<Vec<(TxInIndex, InputSource)>> {
txid_prefix_to_tx_index.clear();
txid_prefix_to_tx_index.extend(txs.iter().map(|ct| (ct.txid_prefix, ct.tx_index)));
let base_tx_index = self.lengths.tx_index;
let base_txin_index = self.lengths.txin_index;
let total_inputs: usize = self.block.txdata.iter().map(|tx| tx.input.len()).sum();
let mut items = Vec::with_capacity(total_inputs);
for (index, tx) in self.block.txdata.iter().enumerate() {
for (vin, txin) in tx.input.iter().enumerate() {
items.push((TxIndex::from(index), Vin::from(vin), txin, tx));
}
}
let txid_prefix_to_tx_index = &*txid_prefix_to_tx_index;
let txins = items
.into_par_iter()
.enumerate()
.map(
|(block_txin_index, (block_tx_index, vin, txin, tx))| -> Result<(TxInIndex, InputSource)> {
let tx_index = base_tx_index + block_tx_index;
let txin_index = base_txin_index + TxInIndex::from(block_txin_index);
if tx.is_coinbase() {
return Ok((
txin_index,
InputSource::SameBlock {
tx_index,
vin,
outpoint: OutPoint::COINBASE,
},
));
}
let outpoint = txin.previous_output;
let txid = Txid::from(outpoint.txid);
let txid_prefix = TxidPrefix::from(&txid);
let vout = Vout::from(outpoint.vout);
if let Some(&same_block_tx_index) = txid_prefix_to_tx_index
.get(&txid_prefix) {
let outpoint = OutPoint::new(same_block_tx_index, vout);
return Ok((
txin_index,
InputSource::SameBlock {
tx_index,
vin,
outpoint,
},
));
}
let store_result = self
.stores
.txid_prefix_to_tx_index
.get(&txid_prefix)?
.map(|v| *v);
let prev_tx_index = match store_result {
Some(tx_index) if tx_index < self.lengths.tx_index => tx_index,
_ => {
error!(
"UnknownTxid: txid={}, prefix={:?}, store_result={:?}, current_tx_index={:?}",
txid, txid_prefix, store_result, self.lengths.tx_index
);
return Err(Error::UnknownTxid);
}
};
let txout_index = self
.vecs
.transactions
.first_txout_index
.get_pushed_or_read(prev_tx_index, &self.readers.tx_index_to_first_txout_index)
.ok_or(Error::Internal("Missing txout_index"))?
+ vout;
let outpoint = OutPoint::new(prev_tx_index, vout);
let output_type = self
.vecs
.outputs
.output_type
.get_pushed_or_read(txout_index, &self.readers.txout_index_to_output_type)
.ok_or(Error::Internal("Missing output_type"))?;
let type_index = self
.vecs
.outputs
.type_index
.get_pushed_or_read(txout_index, &self.readers.txout_index_to_type_index)
.ok_or(Error::Internal("Missing type_index"))?;
Ok((
txin_index,
InputSource::PreviousBlock {
vin,
tx_index,
outpoint,
output_type,
type_index,
},
))
},
)
.collect::<Result<Vec<_>>>()?;
Ok(txins)
}
pub fn collect_same_block_spent_outpoints(
txins: &[(TxInIndex, InputSource)],
out: &mut FxHashSet<OutPoint>,
) {
out.clear();
out.extend(
txins
.iter()
.filter_map(|(_, input_source)| match input_source {
InputSource::SameBlock { outpoint, .. } if !outpoint.is_coinbase() => {
Some(*outpoint)
}
_ => None,
}),
);
}
}
pub(super) fn finalize_inputs(
first_txin_index: &mut PcoVec<TxIndex, TxInIndex>,
inputs: &mut InputsVecs,
addr_tx_index_stores: &mut ByAddrType<Store<AddrIndexTxIndex, Unit>>,
addr_outpoint_stores: &mut ByAddrType<Store<AddrIndexOutPoint, Unit>>,
txins: Vec<(TxInIndex, InputSource)>,
same_block_output_info: &mut FxHashMap<OutPoint, SameBlockOutputInfo>,
) -> Result<()> {
for (txin_index, input_source) in txins {
let (vin, tx_index, outpoint, output_type, type_index) = match input_source {
InputSource::PreviousBlock {
vin,
tx_index,
outpoint,
output_type,
type_index,
} => (vin, tx_index, outpoint, output_type, type_index),
InputSource::SameBlock {
tx_index,
vin,
outpoint,
} => {
if outpoint.is_coinbase() {
(
vin,
tx_index,
outpoint,
OutputType::Unknown,
TypeIndex::COINBASE,
)
} else {
let info = same_block_output_info
.remove(&outpoint)
.ok_or(Error::Internal("Same-block output not found"))
.inspect_err(|_| {
error!(
?outpoint,
remaining = same_block_output_info.len(),
"Same-block output not found"
);
})?;
(vin, tx_index, outpoint, info.output_type, info.type_index)
}
}
};
if vin.is_zero() {
first_txin_index.checked_push(tx_index, txin_index)?;
}
inputs.tx_index.checked_push(txin_index, tx_index)?;
inputs.outpoint.checked_push(txin_index, outpoint)?;
inputs.output_type.checked_push(txin_index, output_type)?;
inputs.type_index.checked_push(txin_index, type_index)?;
if !output_type.is_addr() {
continue;
}
let addr_type = output_type;
let addr_index = type_index;
addr_tx_index_stores
.get_mut_unwrap(addr_type)
.insert(AddrIndexTxIndex::from((addr_index, tx_index)), Unit);
addr_outpoint_stores
.get_mut_unwrap(addr_type)
.remove(AddrIndexOutPoint::from((addr_index, outpoint)));
}
Ok(())
}
crates/brk_indexer/src/processor/txin/mod.rs
+98
View File
@@ -0,0 +1,98 @@
mod resolver;
mod source;
pub(crate) use resolver::InputResolver;
pub(super) use source::InputSource;
use bitcoin::Transaction;
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_store::Store;
use brk_types::{
AddrIndexOutPoint, AddrIndexTxIndex, OutPoint, OutputType, TxInIndex, TxIndex, TypeIndex, Unit,
Vin,
};
use vecdb::{PcoVec, WritableVec, unlikely};
use super::{BlockProcessor, transaction::ComputedTx, txout::ProcessedOutput};
use crate::InputsVecs;
impl<'a> BlockProcessor<'a> {
pub(crate) fn process_inputs(
&self,
txs: &[ComputedTx],
resolver: &mut InputResolver,
) -> Result<Vec<InputSource>> {
resolver.resolve(self, txs)
}
}
#[allow(clippy::too_many_arguments)]
pub(super) fn finalize_inputs(
transactions: &[Transaction],
base_tx_index: TxIndex,
base_txin_index: TxInIndex,
first_txin_index: &mut PcoVec<TxIndex, TxInIndex>,
inputs: &mut InputsVecs,
addr_tx_index_stores: &mut ByAddrType<Store<AddrIndexTxIndex, Unit>>,
addr_outpoint_stores: &mut ByAddrType<Store<AddrIndexOutPoint, Unit>>,
txins: Vec<InputSource>,
txouts: &[ProcessedOutput],
) -> Result<()> {
let mut input_offset = 0;
for (block_tx_index, tx) in transactions.iter().enumerate() {
let tx_index = base_tx_index + TxIndex::from(block_tx_index);
let next_input_offset = input_offset + tx.input.len();
for (vin, input_source) in txins[input_offset..next_input_offset].iter().enumerate() {
let block_txin_index = input_offset + vin;
let txin_index = base_txin_index + TxInIndex::from(block_txin_index);
let vin = Vin::from(vin);
let (outpoint, output_type, type_index) = match input_source {
InputSource::PreviousBlock {
outpoint,
output_type,
legacy_sigops: _,
type_index,
} => (*outpoint, *output_type, *type_index),
InputSource::Coinbase => {
(OutPoint::COINBASE, OutputType::Unknown, TypeIndex::COINBASE)
}
InputSource::SameBlock {
outpoint,
txout_offset,
} => {
let output = &txouts[*txout_offset];
(*outpoint, output.output_type, output.resolved_type_index())
}
};
if vin.is_zero() {
first_txin_index.checked_push(tx_index, txin_index)?;
}
inputs.tx_index.checked_push(txin_index, tx_index)?;
inputs.outpoint.checked_push(txin_index, outpoint)?;
inputs.output_type.checked_push(txin_index, output_type)?;
inputs.type_index.checked_push(txin_index, type_index)?;
if unlikely(!output_type.is_addr()) {
continue;
}
let addr_type = output_type;
let addr_index = type_index;
addr_tx_index_stores
.get_mut_unwrap(addr_type)
.insert(AddrIndexTxIndex::from((addr_index, tx_index)), Unit);
addr_outpoint_stores
.get_mut_unwrap(addr_type)
.remove(AddrIndexOutPoint::from((addr_index, outpoint)));
}
input_offset = next_input_offset;
}
Ok(())
}
crates/brk_indexer/src/processor/txin/resolver.rs
+205
View File
@@ -0,0 +1,205 @@
use std::collections::hash_map::Entry;
use brk_error::{Error, Result};
use brk_types::{OutPoint, SigOps, TxIndex, TxOutIndex, Txid, TxidPrefix, Vout};
use rayon::prelude::*;
use rustc_hash::FxHashMap;
use tracing::error;
use vecdb::unlikely;
use super::InputSource;
use crate::processor::{BlockProcessor, transaction::ComputedTx};
#[derive(Default)]
pub(crate) struct InputResolver {
same_block_transactions: FxHashMap<TxidPrefix, TxIndex>,
previous_parent_indexes: FxHashMap<TxidPrefix, usize>,
previous_parents: Vec<PreviousParent>,
inputs: Vec<UnresolvedInput>,
}
impl InputResolver {
pub(crate) fn resolve(
&mut self,
processor: &BlockProcessor<'_>,
txs: &[ComputedTx<'_>],
) -> Result<Vec<InputSource>> {
self.prepare(txs, processor.lengths.tx_index);
let current_tx_index = processor.lengths.tx_index;
let parents = self
.previous_parents
.par_iter()
.map(|parent| {
let store_result = processor
.stores
.txid_prefix_to_tx_index
.get(&parent.txid_prefix)?
.map(|value| *value);
let tx_index = match store_result {
Some(tx_index) if tx_index < current_tx_index => tx_index,
_ => {
error!(
"UnknownTxid: txid={}, prefix={:?}, store_result={:?}, current_tx_index={:?}",
parent.txid,
parent.txid_prefix,
store_result,
current_tx_index
);
return Err(Error::UnknownTxid);
}
};
let first_txout_index = processor
.vecs
.transactions
.first_txout_index
.get_pushed_or_read(
tx_index,
&processor.readers.tx_index_to_first_txout_index,
)
.ok_or(Error::Internal("Missing txout_index"))?;
Ok(ResolvedParent {
tx_index,
first_txout_index,
})
})
.collect::<Result<Vec<_>>>()?;
let tracks_executed_legacy_sigops = processor.tracks_executed_legacy_sigops();
self.inputs
.par_iter()
.map(|input| match *input {
UnresolvedInput::Coinbase => Ok(InputSource::Coinbase),
UnresolvedInput::SameBlock {
outpoint,
txout_offset,
} => Ok(InputSource::SameBlock {
outpoint,
txout_offset,
}),
UnresolvedInput::PreviousBlock { parent_index, vout } => {
let parent = parents[parent_index];
let txout_index = parent.first_txout_index + vout;
let outpoint = OutPoint::new(parent.tx_index, vout);
let output_type = processor
.vecs
.outputs
.output_type
.get_pushed_or_read(
txout_index,
&processor.readers.txout_index_to_output_type,
)
.ok_or(Error::Internal("Missing output_type"))?;
let type_index = processor
.vecs
.outputs
.type_index
.get_pushed_or_read(
txout_index,
&processor.readers.txout_index_to_type_index,
)
.ok_or(Error::Internal("Missing type_index"))?;
let legacy_sigops = if tracks_executed_legacy_sigops {
processor
.vecs
.scripts
.legacy_sigops(output_type, type_index, &processor.readers.scripts)
.ok_or(Error::Internal("Missing legacy_sigops"))?
} else {
SigOps::ZERO
};
Ok(InputSource::PreviousBlock {
outpoint,
output_type,
legacy_sigops,
type_index,
})
}
})
.collect()
}
fn prepare(&mut self, txs: &[ComputedTx<'_>], block_first_tx_index: TxIndex) {
self.same_block_transactions.clear();
self.previous_parent_indexes.clear();
self.previous_parents.clear();
self.inputs.clear();
self.same_block_transactions.reserve(txs.len());
self.same_block_transactions
.extend(txs.iter().map(|tx| (tx.txid_prefix(), tx.tx_index)));
let total_inputs = txs.iter().map(|tx| tx.tx.input.len()).sum();
self.inputs.reserve(total_inputs);
for tx in txs {
for txin in &tx.tx.input {
let previous_output = &txin.previous_output;
if unlikely(previous_output.is_null()) {
self.inputs.push(UnresolvedInput::Coinbase);
continue;
}
let txid = *<&Txid>::from(&previous_output.txid);
let txid_prefix = TxidPrefix::from(&txid);
let vout = Vout::from(previous_output.vout);
if let Some(&tx_index) = self.same_block_transactions.get(&txid_prefix) {
let block_tx_index = usize::from(tx_index) - usize::from(block_first_tx_index);
self.inputs.push(UnresolvedInput::SameBlock {
outpoint: OutPoint::new(tx_index, vout),
txout_offset: txs[block_tx_index].txout_offset(vout),
});
continue;
}
let parent_index = match self.previous_parent_indexes.entry(txid_prefix) {
Entry::Occupied(entry) => *entry.get(),
Entry::Vacant(entry) => {
let parent_index = self.previous_parents.len();
entry.insert(parent_index);
self.previous_parents
.push(PreviousParent { txid, txid_prefix });
parent_index
}
};
self.inputs
.push(UnresolvedInput::PreviousBlock { parent_index, vout });
}
}
}
}
#[derive(Clone, Copy)]
struct PreviousParent {
txid: Txid,
txid_prefix: TxidPrefix,
}
#[derive(Clone, Copy)]
struct ResolvedParent {
tx_index: TxIndex,
first_txout_index: TxOutIndex,
}
#[derive(Clone, Copy)]
enum UnresolvedInput {
Coinbase,
PreviousBlock {
parent_index: usize,
vout: Vout,
},
SameBlock {
outpoint: OutPoint,
txout_offset: usize,
},
}
crates/brk_indexer/src/processor/txin/source.rs
+16
View File
@@ -0,0 +1,16 @@
use brk_types::{OutPoint, OutputType, SigOps, TypeIndex};
#[derive(Debug)]
pub(crate) enum InputSource {
Coinbase,
PreviousBlock {
outpoint: OutPoint,
output_type: OutputType,
legacy_sigops: SigOps,
type_index: TypeIndex,
},
SameBlock {
outpoint: OutPoint,
txout_offset: usize,
},
}
crates/brk_indexer/src/processor/txout.rs
-235
View File
@@ -1,235 +0,0 @@
use brk_cohort::ByAddrType;
use brk_error::{Error, Result};
use brk_store::Store;
use brk_types::{
AddrBytes, AddrHash, AddrIndexOutPoint, AddrIndexTxIndex, OutPoint, OutputType, Sats, TxIndex,
TxOutIndex, TypeIndex, Unit, Vout,
};
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::error;
use vecdb::{BytesVec, WritableVec};
use super::{BlockProcessor, ProcessedOutput, SameBlockOutputInfo};
use crate::{AddrsVecs, Lengths, OutputsVecs, ScriptsVecs};
impl<'a> BlockProcessor<'a> {
pub fn process_outputs(&self) -> Result<Vec<ProcessedOutput<'a>>> {
let height = self.height;
let check_collisions = self.check_collisions;
let base_tx_index = self.lengths.tx_index;
let base_txout_index = self.lengths.txout_index;
let total_outputs: usize = self.block.txdata.iter().map(|tx| tx.output.len()).sum();
let mut items = Vec::with_capacity(total_outputs);
for (index, tx) in self.block.txdata.iter().enumerate() {
for (vout, txout) in tx.output.iter().enumerate() {
items.push((TxIndex::from(index), Vout::from(vout), txout));
}
}
items
.into_par_iter()
.enumerate()
.map(
|(block_txout_index, (block_tx_index, vout, txout))| -> Result<ProcessedOutput> {
let tx_index = base_tx_index + block_tx_index;
let txout_index = base_txout_index + TxOutIndex::from(block_txout_index);
let script = &txout.script_pubkey;
let output_type = OutputType::from(script);
if output_type.is_not_addr() {
return Ok(ProcessedOutput {
txout_index,
txout,
tx_index,
vout,
output_type,
addr_info: None,
existing_type_index: None,
});
}
let addr_type = output_type;
let addr_bytes = AddrBytes::try_from((script, addr_type)).unwrap();
let addr_hash = AddrHash::from(&addr_bytes);
let existing_type_index = self
.stores
.addr_type_to_addr_hash_to_addr_index
.get_unwrap(addr_type)
.get(&addr_hash)?
.map(|v| *v)
.and_then(|type_index_local| {
(type_index_local < self.lengths.to_type_index(addr_type))
.then_some(type_index_local)
});
if check_collisions && let Some(type_index) = existing_type_index {
let prev_addrbytes = self
.vecs
.addrs
.get_bytes_by_type(addr_type, type_index, &self.readers.addrbytes)
.ok_or(Error::Internal("Missing addrbytes"))?;
if prev_addrbytes != addr_bytes {
error!(
?height,
?vout,
?block_tx_index,
?addr_type,
?prev_addrbytes,
?addr_bytes,
?type_index,
"Address hash collision"
);
return Err(Error::Internal("Address hash collision"));
}
}
Ok(ProcessedOutput {
txout_index,
txout,
tx_index,
vout,
output_type,
addr_info: Some((addr_bytes, addr_hash)),
existing_type_index,
})
},
)
.collect()
}
}
#[allow(clippy::too_many_arguments)]
pub(super) fn finalize_outputs(
lengths: &mut Lengths,
first_txout_index: &mut BytesVec<TxIndex, TxOutIndex>,
outputs: &mut OutputsVecs,
addrs: &mut AddrsVecs,
scripts: &mut ScriptsVecs,
addr_hash_stores: &mut ByAddrType<Store<AddrHash, TypeIndex>>,
addr_tx_index_stores: &mut ByAddrType<Store<AddrIndexTxIndex, Unit>>,
addr_outpoint_stores: &mut ByAddrType<Store<AddrIndexOutPoint, Unit>>,
txouts: Vec<ProcessedOutput>,
same_block_spent_outpoints: &FxHashSet<OutPoint>,
already_added_addr_hash: &mut ByAddrType<FxHashMap<AddrHash, TypeIndex>>,
same_block_output_info: &mut FxHashMap<OutPoint, SameBlockOutputInfo>,
) -> Result<()> {
already_added_addr_hash.values_mut().for_each(|m| m.clear());
same_block_output_info.clear();
for ProcessedOutput {
txout_index,
txout,
tx_index,
vout,
output_type,
addr_info,
existing_type_index,
} in txouts
{
let sats = Sats::from(txout.value);
if vout.is_zero() {
first_txout_index.checked_push(tx_index, txout_index)?;
}
outputs.tx_index.checked_push(txout_index, tx_index)?;
let type_index = if let Some(ti) = existing_type_index {
ti
} else if let Some((addr_bytes, addr_hash)) = addr_info {
let addr_type = output_type;
if let Some(&ti) = already_added_addr_hash
.get_unwrap(addr_type)
.get(&addr_hash)
{
ti
} else {
let ti = lengths.increment_addr_index(addr_type);
already_added_addr_hash
.get_mut_unwrap(addr_type)
.insert(addr_hash, ti);
addr_hash_stores
.get_mut_unwrap(addr_type)
.insert(addr_hash, ti);
addrs.push_bytes_if_needed(ti, addr_bytes)?;
ti
}
} else {
match output_type {
OutputType::P2MS => {
scripts
.p2ms
.to_tx_index
.checked_push(lengths.p2ms_output_index, tx_index)?;
lengths.p2ms_output_index.copy_then_increment()
}
OutputType::OpReturn => {
scripts
.op_return
.to_tx_index
.checked_push(lengths.op_return_index, tx_index)?;
lengths.op_return_index.copy_then_increment()
}
OutputType::Empty => {
scripts
.empty
.to_tx_index
.checked_push(lengths.empty_output_index, tx_index)?;
lengths.empty_output_index.copy_then_increment()
}
OutputType::Unknown => {
scripts
.unknown
.to_tx_index
.checked_push(lengths.unknown_output_index, tx_index)?;
lengths.unknown_output_index.copy_then_increment()
}
_ => unreachable!(),
}
};
outputs.value.checked_push(txout_index, sats)?;
outputs.output_type.checked_push(txout_index, output_type)?;
outputs.type_index.checked_push(txout_index, type_index)?;
if output_type.is_unspendable() {
continue;
} else if output_type.is_addr() {
let addr_type = output_type;
let addr_index = type_index;
addr_tx_index_stores
.get_mut_unwrap(addr_type)
.insert(AddrIndexTxIndex::from((addr_index, tx_index)), Unit);
}
let outpoint = OutPoint::new(tx_index, vout);
if same_block_spent_outpoints.contains(&outpoint) {
same_block_output_info.insert(
outpoint,
SameBlockOutputInfo {
output_type,
type_index,
},
);
} else if output_type.is_addr() {
let addr_type = output_type;
let addr_index = type_index;
addr_outpoint_stores
.get_mut_unwrap(addr_type)
.insert(AddrIndexOutPoint::from((addr_index, outpoint)), Unit);
}
}
Ok(())
}
crates/brk_indexer/src/processor/txout/address.rs
+172
View File
@@ -0,0 +1,172 @@
mod cache;
use std::collections::hash_map::Entry;
use brk_cohort::ByAddrType;
use brk_error::{Error, Result};
use brk_types::{AddrBytes, AddrHash, OutputType, TxIndex, TypeIndex, Vout};
use rayon::prelude::*;
use rustc_hash::FxHashMap;
use tracing::error;
use vecdb::likely;
use self::cache::AddressCache;
use super::{ProcessedOutput, processed::ProcessedOutputData};
use crate::processor::BlockProcessor;
struct Lookup {
index: usize,
output_type: OutputType,
hash: AddrHash,
type_index: Option<TypeIndex>,
}
#[derive(Default)]
pub(crate) struct BlockAddresses {
cache: AddressCache,
indexes: ByAddrType<FxHashMap<AddrHash, usize>>,
lookups: Vec<Lookup>,
unique: Vec<(OutputType, AddrHash)>,
resolved: Vec<Option<TypeIndex>>,
}
impl BlockAddresses {
pub(super) fn resolve(
&mut self,
processor: &BlockProcessor,
outputs: &[ProcessedOutput],
) -> Result<()> {
self.clear_block();
for output in outputs {
let ProcessedOutputData::Address(addr_hash) = &output.data else {
continue;
};
if let Entry::Vacant(entry) = self
.indexes
.get_mut_unwrap(output.output_type)
.entry(*addr_hash)
{
entry.insert(self.unique.len());
self.unique.push((output.output_type, *addr_hash));
}
}
self.resolved.resize(self.unique.len(), None);
for (index, &(output_type, hash)) in self.unique.iter().enumerate() {
if let Some(type_index) = self.cache.get(output_type, hash) {
self.resolved[index] = Some(type_index);
} else {
self.lookups.push(Lookup {
index,
output_type,
hash,
type_index: None,
});
}
}
let stores = &processor.stores.addr_type_to_addr_hash_to_addr_index;
let lengths = &*processor.lengths;
self.lookups
.par_iter_mut()
.try_for_each(|lookup| -> Result<()> {
lookup.type_index = stores
.get_unwrap(lookup.output_type)
.get(&lookup.hash)?
.map(|type_index| *type_index)
.filter(|type_index| *type_index < lengths.to_type_index(lookup.output_type));
Ok(())
})?;
for lookup in &self.lookups {
self.resolved[lookup.index] = lookup.type_index;
if let Some(type_index) = lookup.type_index {
self.cache
.insert(lookup.output_type, lookup.hash, type_index);
}
}
if likely(!processor.check_collisions) {
return Ok(());
}
let mut output_offset = 0;
for (block_tx_index, tx) in processor.block.txdata.iter().enumerate() {
let tx_index = processor.lengths.tx_index + TxIndex::from(block_tx_index);
let next_output_offset = output_offset + tx.output.len();
for (vout, (txout, output)) in tx
.output
.iter()
.zip(&outputs[output_offset..next_output_offset])
.enumerate()
{
let ProcessedOutputData::Address(addr_hash) = &output.data else {
continue;
};
let Some(type_index) = self.index(output.output_type, addr_hash) else {
continue;
};
let addr_bytes =
AddrBytes::try_from((&txout.script_pubkey, output.output_type)).unwrap();
let prev_addrbytes = processor
.vecs
.addrs
.get_bytes_by_type(output.output_type, type_index, &processor.readers.addrbytes)
.ok_or(Error::Internal("Missing addrbytes"))?;
if prev_addrbytes != addr_bytes {
error!(
height = ?processor.height,
vout = ?Vout::from(vout),
?tx_index,
addr_type = ?output.output_type,
?prev_addrbytes,
?addr_bytes,
?type_index,
"Address hash collision"
);
return Err(Error::Internal("Address hash collision"));
}
}
output_offset = next_output_offset;
}
Ok(())
}
pub(super) fn index_mut(
&mut self,
addr_type: OutputType,
addr_hash: &AddrHash,
) -> &mut Option<TypeIndex> {
let index = *self
.indexes
.get_mut_unwrap(addr_type)
.get_mut(addr_hash)
.unwrap();
&mut self.resolved[index]
}
fn index(&self, addr_type: OutputType, addr_hash: &AddrHash) -> Option<TypeIndex> {
let index = *self.indexes.get_unwrap(addr_type).get(addr_hash).unwrap();
self.resolved[index]
}
pub(crate) fn clear_cache(&mut self) {
self.cache.clear();
}
fn clear_block(&mut self) {
self.indexes.values_mut().for_each(FxHashMap::clear);
self.lookups.clear();
self.unique.clear();
self.resolved.clear();
}
}
crates/brk_indexer/src/processor/txout/address/cache.rs
+140
View File
@@ -0,0 +1,140 @@
use std::mem;
use brk_types::{AddrHash, OutputType, TypeIndex};
const ASSOCIATIVITY: usize = 4;
const SET_COUNT: usize = 1 << 18;
const SET_MASK: usize = SET_COUNT - 1;
const TYPE_MIX: u64 = 0x9e37_79b9_7f4a_7c15;
const REFERENCE_SHIFT: usize = 2;
#[derive(Clone, Copy)]
#[repr(C)]
struct Entry {
hash: AddrHash,
type_index: TypeIndex,
output_type: OutputType,
}
impl Entry {
const EMPTY: Self = Self {
hash: AddrHash::new(0),
type_index: TypeIndex::COINBASE,
output_type: OutputType::Unknown,
};
#[inline]
fn matches(&self, output_type: OutputType, hash: AddrHash) -> bool {
self.output_type == output_type && self.hash == hash
}
#[inline]
fn is_empty(&self) -> bool {
self.output_type == OutputType::Unknown
}
}
#[derive(Clone, Copy)]
#[repr(C, align(64))]
struct Set([Entry; ASSOCIATIVITY]);
impl Set {
const EMPTY: Self = Self([Entry::EMPTY; ASSOCIATIVITY]);
}
#[derive(Clone, Copy, Default)]
struct SetState(u8);
impl SetState {
#[inline]
fn mark_referenced(&mut self, entry_index: usize) {
self.0 |= 1 << (REFERENCE_SHIFT + entry_index);
}
#[inline]
fn victim(&mut self) -> usize {
loop {
let entry_index = usize::from(self.0 & (ASSOCIATIVITY as u8 - 1));
self.0 = (self.0 & !(ASSOCIATIVITY as u8 - 1))
| ((entry_index + 1) & (ASSOCIATIVITY - 1)) as u8;
let reference = 1 << (REFERENCE_SHIFT + entry_index);
if self.0 & reference == 0 {
return entry_index;
}
self.0 &= !reference;
}
}
}
const _: () = assert!(mem::size_of::<Entry>() == 16);
const _: () = assert!(mem::size_of::<Set>() == 64);
const _: () = assert!(mem::size_of::<SetState>() == 1);
struct Storage {
sets: Box<[Set]>,
states: Box<[SetState]>,
}
impl Storage {
fn new() -> Self {
Self {
sets: vec![Set::EMPTY; SET_COUNT].into_boxed_slice(),
states: vec![SetState::default(); SET_COUNT].into_boxed_slice(),
}
}
}
#[derive(Default)]
pub(super) struct AddressCache {
storage: Option<Storage>,
}
impl AddressCache {
#[inline]
pub(super) fn get(&mut self, output_type: OutputType, hash: AddrHash) -> Option<TypeIndex> {
let storage = self.storage.as_mut()?;
let set_index = Self::set_index(output_type, hash);
let entry_index = storage.sets[set_index]
.0
.iter()
.position(|entry| entry.matches(output_type, hash))?;
storage.states[set_index].mark_referenced(entry_index);
Some(storage.sets[set_index].0[entry_index].type_index)
}
#[inline]
pub(super) fn insert(
&mut self,
output_type: OutputType,
hash: AddrHash,
type_index: TypeIndex,
) {
let set_index = Self::set_index(output_type, hash);
let storage = self.storage.get_or_insert_with(Storage::new);
let set = &mut storage.sets[set_index].0;
let state = &mut storage.states[set_index];
let entry_index = set
.iter()
.position(Entry::is_empty)
.unwrap_or_else(|| state.victim());
set[entry_index] = Entry {
hash,
type_index,
output_type,
};
state.mark_referenced(entry_index);
}
pub(super) fn clear(&mut self) {
self.storage = None;
}
#[inline]
fn set_index(output_type: OutputType, hash: AddrHash) -> usize {
let hash = *hash ^ (output_type as u64).wrapping_mul(TYPE_MIX);
hash as usize & SET_MASK
}
}
crates/brk_indexer/src/processor/txout/mod.rs
+207
View File
@@ -0,0 +1,207 @@
mod address;
mod op_return;
mod processed;
pub(crate) use address::BlockAddresses;
pub(super) use processed::ProcessedOutput;
use processed::ProcessedOutputData;
use bitcoin::{Script, Transaction, constants::WITNESS_SCALE_FACTOR};
use brk_cohort::ByAddrType;
use brk_error::Result;
use brk_store::Store;
use brk_types::{
AddrBytes, AddrHash, AddrIndexOutPoint, AddrIndexTxIndex, OutPoint, OutputType, Sats, SigOps,
TxIndex, TxOutIndex, TypeIndex, Unit, Vout,
};
use rayon::prelude::*;
use vecdb::{BytesVec, WritableVec, likely};
use super::BlockProcessor;
use crate::{AddrsVecs, Lengths, OpReturnVecs, OutputsVecs, ScriptsVecs};
impl<'a> BlockProcessor<'a> {
pub(crate) fn process_outputs(
&self,
addresses: &mut BlockAddresses,
) -> Result<Vec<ProcessedOutput>> {
let total_outputs: usize = self.block.txdata.iter().map(|tx| tx.output.len()).sum();
let mut items = Vec::with_capacity(total_outputs);
for tx in &self.block.txdata {
items.extend(&tx.output);
}
let outputs = items
.into_par_iter()
.map(|txout| {
let script = &txout.script_pubkey;
let output_type = OutputType::from(script);
let legacy_sigops = executed_legacy_sigops_for_output(output_type, script);
let data = if output_type.is_addr() {
ProcessedOutputData::Address(
AddrHash::from_script(script, output_type).unwrap(),
)
} else if likely(output_type == OutputType::OpReturn) {
ProcessedOutputData::OpReturn(op_return::analyze(script))
} else {
ProcessedOutputData::None
};
ProcessedOutput {
output_type,
legacy_sigops,
data,
}
})
.collect::<Vec<_>>();
addresses.resolve(self, &outputs)?;
Ok(outputs)
}
}
fn executed_legacy_sigops_for_output(output_type: OutputType, script_pubkey: &Script) -> SigOps {
SigOps::from(
match output_type {
OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65 => 1,
OutputType::P2MS | OutputType::Unknown => script_pubkey.count_sigops(),
_ => 0,
}
.saturating_mul(WITNESS_SCALE_FACTOR),
)
}
#[allow(clippy::too_many_arguments)]
pub(super) fn finalize_outputs(
transactions: &[Transaction],
base_tx_index: TxIndex,
lengths: &mut Lengths,
first_txout_index: &mut BytesVec<TxIndex, TxOutIndex>,
outputs: &mut OutputsVecs,
addrs: &mut AddrsVecs,
scripts: &mut ScriptsVecs,
op_return_vecs: &mut OpReturnVecs,
addr_hash_stores: &mut ByAddrType<Store<AddrHash, TypeIndex>>,
addr_tx_index_stores: &mut ByAddrType<Store<AddrIndexTxIndex, Unit>>,
addr_outpoint_stores: &mut ByAddrType<Store<AddrIndexOutPoint, Unit>>,
txouts: &mut [ProcessedOutput],
addresses: &mut BlockAddresses,
) -> Result<()> {
let base_txout_index = lengths.txout_index;
let mut output_offset = 0;
for (block_tx_index, tx) in transactions.iter().enumerate() {
let tx_index = base_tx_index + TxIndex::from(block_tx_index);
let next_output_offset = output_offset + tx.output.len();
for (vout, (txout, processed)) in tx
.output
.iter()
.zip(&mut txouts[output_offset..next_output_offset])
.enumerate()
{
let block_txout_index = output_offset + vout;
let txout_index = base_txout_index + TxOutIndex::from(block_txout_index);
let vout = Vout::from(vout);
let output_type = processed.output_type;
let legacy_sigops = processed.legacy_sigops;
let data = processed.data;
let sats = Sats::from(txout.value);
if vout.is_zero() {
first_txout_index.checked_push(tx_index, txout_index)?;
}
outputs.tx_index.checked_push(txout_index, tx_index)?;
let type_index = match data {
ProcessedOutputData::Address(addr_hash) => {
let addr_type = output_type;
let type_index = addresses.index_mut(addr_type, &addr_hash);
if let Some(ti) = *type_index {
ti
} else {
let ti = lengths.increment_addr_index(addr_type);
*type_index = Some(ti);
addr_hash_stores
.get_mut_unwrap(addr_type)
.insert(addr_hash, ti);
let addr_bytes =
AddrBytes::try_from((&txout.script_pubkey, addr_type)).unwrap();
addrs.push_bytes_if_needed(ti, addr_bytes)?;
ti
}
}
ProcessedOutputData::OpReturn(op_return) => {
let op_return_index = lengths.op_return_index;
op_return_vecs
.to_tx_index
.checked_push(lengths.op_return_index, tx_index)?;
op_return_vecs
.kind
.checked_push(op_return_index, op_return.kind)?;
op_return_vecs
.post_op_return_bytes
.checked_push(op_return_index, op_return.post_op_return_bytes)?;
lengths.op_return_index.copy_then_increment()
}
ProcessedOutputData::None => match output_type {
OutputType::P2MS => {
let index = lengths.p2ms_output_index;
scripts.p2ms.to_tx_index.checked_push(index, tx_index)?;
scripts
.p2ms
.legacy_sigops
.checked_push(index, legacy_sigops)?;
lengths.p2ms_output_index.copy_then_increment()
}
OutputType::Empty => {
scripts
.empty
.to_tx_index
.checked_push(lengths.empty_output_index, tx_index)?;
lengths.empty_output_index.copy_then_increment()
}
OutputType::Unknown => {
let index = lengths.unknown_output_index;
scripts.unknown.to_tx_index.checked_push(index, tx_index)?;
scripts
.unknown
.legacy_sigops
.checked_push(index, legacy_sigops)?;
lengths.unknown_output_index.copy_then_increment()
}
_ => unreachable!(),
},
ProcessedOutputData::Resolved(_) => unreachable!(),
};
outputs.value.checked_push(txout_index, sats)?;
outputs.output_type.checked_push(txout_index, output_type)?;
outputs.type_index.checked_push(txout_index, type_index)?;
processed.data = ProcessedOutputData::Resolved(type_index);
if likely(output_type.is_addr()) {
let addr_type = output_type;
let addr_index = type_index;
addr_tx_index_stores
.get_mut_unwrap(addr_type)
.insert(AddrIndexTxIndex::from((addr_index, tx_index)), Unit);
addr_outpoint_stores.get_mut_unwrap(addr_type).insert(
AddrIndexOutPoint::from((addr_index, OutPoint::new(tx_index, vout))),
Unit,
);
}
}
output_offset = next_output_offset;
}
Ok(())
}
crates/brk_indexer/src/processor/txout/op_return.rs
+221
View File
@@ -0,0 +1,221 @@
use bitcoin::{
Script,
opcodes::all::{OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY, OP_CHECKSIG, OP_CHECKSIGVERIFY},
script::Instruction,
};
use brk_types::{OpReturnKind, StoredU32};
#[derive(Debug, Clone, Copy)]
pub(super) struct Facts {
pub(super) kind: OpReturnKind,
pub(super) legacy_sigops: StoredU32,
pub(super) post_op_return_bytes: StoredU32,
}
pub(super) fn analyze(script: &Script) -> Facts {
let data = &script.as_bytes()[1..];
let (prefix, legacy_sigops) = scan(script);
Facts {
kind: classify(data, prefix),
legacy_sigops: StoredU32::from(legacy_sigops),
post_op_return_bytes: StoredU32::from(data.len()),
}
}
fn classify(data: &[u8], prefix: Option<&[u8]>) -> OpReturnKind {
if data.first() == Some(&0x5d) {
return OpReturnKind::Runes;
}
let Some(prefix) = prefix else {
return OpReturnKind::Empty;
};
if prefix.starts_with(b"omni") {
OpReturnKind::Omni
} else if prefix.starts_with(b"X2") || prefix.starts_with(b"X1") {
OpReturnKind::Stacks
} else if prefix.starts_with(b"id") {
OpReturnKind::Blockstack
} else if prefix.starts_with(b"CC") {
OpReturnKind::Colu
} else if prefix.starts_with(b"OA\x01\x00") {
OpReturnKind::OpenAssets
} else if prefix.starts_with(b"SPK") {
OpReturnKind::CoinSpark
} else if prefix.starts_with(b"POET") {
OpReturnKind::Poet
} else if prefix.starts_with(b"DOCPROOF") {
OpReturnKind::Docproof
} else if prefix.starts_with(b"\x05\x88\x96\x0d\x73\xd7\x19\x01") {
OpReturnKind::OpenTimestamps
} else if prefix.starts_with(b"Factom!!") {
OpReturnKind::Factom
} else if prefix.starts_with(b"EW") {
OpReturnKind::EternityWall
} else if is_memo(prefix) {
OpReturnKind::Memo
} else if prefix.starts_with(b"BP") {
OpReturnKind::Bitproof
} else if prefix.starts_with(b"ASCRIBE\0") {
OpReturnKind::Ascribe
} else if prefix.starts_with(b"Stampery") {
OpReturnKind::Stampery
} else if prefix.starts_with(b"EPOBC") {
OpReturnKind::Epobc
} else if data.len() == 82 {
OpReturnKind::VeriBlock
} else if (36..=38).contains(&data.len()) {
OpReturnKind::Komodo
} else if matches!(prefix.len(), 20 | 32) {
OpReturnKind::BareHash
} else if is_text(prefix) {
OpReturnKind::Text
} else {
OpReturnKind::Unknown
}
}
fn scan(script: &Script) -> (Option<&[u8]>, usize) {
let mut first_push = None;
let mut legacy_sigops = 0;
for instruction in script.instructions().skip(1) {
match instruction {
Ok(Instruction::PushBytes(bytes)) => {
if first_push.is_none() && !bytes.is_empty() {
first_push = Some(bytes.as_bytes());
}
}
Ok(Instruction::Op(opcode)) => match opcode {
OP_CHECKSIG | OP_CHECKSIGVERIFY => legacy_sigops += 1,
OP_CHECKMULTISIG | OP_CHECKMULTISIGVERIFY => legacy_sigops += 20,
_ => {}
},
Err(_) => break,
}
}
(first_push, legacy_sigops)
}
fn is_memo(prefix: &[u8]) -> bool {
prefix.len() >= 2
&& prefix[0] == 0x6d
&& (matches!(prefix[1], 0x01..=0x07) || prefix[1] == 0x0c)
}
fn is_text(prefix: &[u8]) -> bool {
prefix.len() >= 4
&& prefix
.iter()
.filter(|byte| byte.is_ascii_graphic() || **byte == b' ')
.count()
* 10
>= prefix.len() * 9
}
#[cfg(test)]
mod tests {
use bitcoin::ScriptBuf;
use bitcoin::opcodes::all::OP_RETURN;
use bitcoin::script::{Builder, PushBytesBuf};
use super::*;
fn pushed(data: &[u8]) -> ScriptBuf {
Builder::new()
.push_opcode(OP_RETURN)
.push_slice(PushBytesBuf::try_from(data.to_vec()).unwrap())
.into_script()
}
#[test]
fn classifies_exact_prefix_before_heuristics() {
let mut payload = b"omni".to_vec();
payload.resize(80, 0);
assert_eq!(analyze(&pushed(&payload)).kind, OpReturnKind::Omni);
}
#[test]
fn classifies_runes_opcode() {
assert_eq!(
analyze(&ScriptBuf::from_bytes(vec![OP_RETURN.to_u8(), 0x5d])).kind,
OpReturnKind::Runes
);
}
#[test]
fn classifies_empty_and_unknown() {
assert_eq!(
analyze(&ScriptBuf::from_bytes(vec![OP_RETURN.to_u8()])).kind,
OpReturnKind::Empty
);
assert_eq!(
analyze(&pushed(&[0, 1, 2, 3, 4])).kind,
OpReturnKind::Unknown
);
}
#[test]
fn classifies_known_protocol_prefixes() {
let cases: &[(&[u8], OpReturnKind)] = &[
(b"omni", OpReturnKind::Omni),
(b"X2", OpReturnKind::Stacks),
(b"id", OpReturnKind::Blockstack),
(b"CC", OpReturnKind::Colu),
(b"OA\x01\x00", OpReturnKind::OpenAssets),
(b"SPK", OpReturnKind::CoinSpark),
(b"POET", OpReturnKind::Poet),
(b"DOCPROOF", OpReturnKind::Docproof),
(
b"\x05\x88\x96\x0d\x73\xd7\x19\x01",
OpReturnKind::OpenTimestamps,
),
(b"Factom!!", OpReturnKind::Factom),
(b"EW", OpReturnKind::EternityWall),
(b"\x6d\x01", OpReturnKind::Memo),
(b"BP", OpReturnKind::Bitproof),
(b"ASCRIBE\0", OpReturnKind::Ascribe),
(b"Stampery", OpReturnKind::Stampery),
(b"EPOBC", OpReturnKind::Epobc),
];
for (prefix, expected) in cases {
assert_eq!(analyze(&pushed(prefix)).kind, *expected);
}
}
#[test]
fn classifies_length_and_content_heuristics() {
assert_eq!(analyze(&pushed(&[1; 80])).kind, OpReturnKind::VeriBlock);
assert_eq!(analyze(&pushed(&[1; 35])).kind, OpReturnKind::Komodo);
assert_eq!(analyze(&pushed(&[1; 20])).kind, OpReturnKind::BareHash);
assert_eq!(analyze(&pushed(b"plain text")).kind, OpReturnKind::Text);
}
#[test]
fn counts_only_executed_legacy_sigop_opcodes() {
use bitcoin::opcodes::all::{
OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY, OP_CHECKSIG, OP_CHECKSIGVERIFY,
};
let opcodes = ScriptBuf::from_bytes(vec![
OP_RETURN.to_u8(),
OP_CHECKSIG.to_u8(),
OP_CHECKSIGVERIFY.to_u8(),
OP_CHECKMULTISIG.to_u8(),
OP_CHECKMULTISIGVERIFY.to_u8(),
]);
assert_eq!(u32::from(analyze(&opcodes).legacy_sigops), 42);
let pushed = pushed(&[
OP_CHECKSIG.to_u8(),
OP_CHECKSIGVERIFY.to_u8(),
OP_CHECKMULTISIG.to_u8(),
OP_CHECKMULTISIGVERIFY.to_u8(),
]);
assert_eq!(u32::from(analyze(&pushed).legacy_sigops), 0);
}
}
crates/brk_indexer/src/processor/txout/processed.rs
+34
View File
@@ -0,0 +1,34 @@
use brk_types::{AddrHash, OutputType, SigOps, TypeIndex};
use super::op_return;
#[derive(Clone, Copy)]
pub(super) enum ProcessedOutputData {
Address(AddrHash),
OpReturn(op_return::Facts),
None,
Resolved(TypeIndex),
}
#[derive(Clone, Copy)]
pub(crate) struct ProcessedOutput {
pub(crate) output_type: OutputType,
pub(crate) legacy_sigops: SigOps,
pub(super) data: ProcessedOutputData,
}
impl ProcessedOutput {
pub(crate) fn op_return_legacy_sigops(&self) -> usize {
let ProcessedOutputData::OpReturn(facts) = self.data else {
unreachable!();
};
u32::from(facts.legacy_sigops) as usize
}
pub(crate) fn resolved_type_index(&self) -> TypeIndex {
let ProcessedOutputData::Resolved(type_index) = self.data else {
unreachable!();
};
type_index
}
}
crates/brk_indexer/src/processor/types.rs
-70
View File
@@ -1,70 +0,0 @@
use bitcoin::{Transaction, TxOut};
use brk_cohort::ByAddrType;
use brk_types::{
AddrBytes, AddrHash, OutPoint, OutputType, TxIndex, TxOutIndex, Txid, TxidPrefix, TypeIndex,
Vin, Vout,
};
use rustc_hash::{FxHashMap, FxHashSet};
#[derive(Debug)]
pub enum InputSource {
PreviousBlock {
vin: Vin,
tx_index: TxIndex,
outpoint: OutPoint,
output_type: OutputType,
type_index: TypeIndex,
},
SameBlock {
tx_index: TxIndex,
vin: Vin,
outpoint: OutPoint,
},
}
#[derive(Debug, Clone, Copy)]
pub struct SameBlockOutputInfo {
pub output_type: OutputType,
pub type_index: TypeIndex,
}
pub struct ProcessedOutput<'a> {
pub txout_index: TxOutIndex,
pub txout: &'a TxOut,
pub tx_index: TxIndex,
pub vout: Vout,
pub output_type: OutputType,
pub addr_info: Option<(AddrBytes, AddrHash)>,
pub existing_type_index: Option<TypeIndex>,
}
pub struct ComputedTx<'a> {
pub tx_index: TxIndex,
pub tx: &'a Transaction,
pub txid: Txid,
pub txid_prefix: TxidPrefix,
pub prev_tx_index_opt: Option<TxIndex>,
pub base_size: u32,
pub total_size: u32,
}
impl ComputedTx<'_> {
#[inline]
pub fn is_segwit(&self) -> bool {
self.base_size != self.total_size
}
#[inline]
pub fn weight(&self) -> usize {
self.base_size as usize * 3 + self.total_size as usize
}
}
/// Reusable buffers cleared and refilled each block to avoid allocation churn.
#[derive(Default)]
pub struct BlockBuffers {
pub txid_prefix_map: FxHashMap<TxidPrefix, TxIndex>,
pub same_block_spent: FxHashSet<OutPoint>,
pub already_added_addrs: ByAddrType<FxHashMap<AddrHash, TypeIndex>>,
pub same_block_output_info: FxHashMap<OutPoint, SameBlockOutputInfo>,
}
crates/brk_indexer/src/readers.rs
+14 -3
View File
@@ -1,8 +1,9 @@
use bitcoin::ScriptBuf;
use brk_types::{
AddrBytes, OutputType, P2AAddrIndex, P2ABytes, P2PK33AddrIndex, P2PK33Bytes, P2PK65AddrIndex,
P2PK65Bytes, P2PKHAddrIndex, P2PKHBytes, P2SHAddrIndex, P2SHBytes, P2TRAddrIndex, P2TRBytes,
P2WPKHAddrIndex, P2WPKHBytes, P2WSHAddrIndex, P2WSHBytes, TxIndex, TxOutIndex, Txid, TypeIndex,
AddrBytes, OutputType, P2AAddrIndex, P2ABytes, P2MSOutputIndex, P2PK33AddrIndex, P2PK33Bytes,
P2PK65AddrIndex, P2PK65Bytes, P2PKHAddrIndex, P2PKHBytes, P2SHAddrIndex, P2SHBytes,
P2TRAddrIndex, P2TRBytes, P2WPKHAddrIndex, P2WPKHBytes, P2WSHAddrIndex, P2WSHBytes, SigOps,
TxIndex, TxOutIndex, Txid, TypeIndex, UnknownOutputIndex,
};
use vecdb::{BytesStrategy, VecReader};
@@ -46,6 +47,7 @@ pub struct Readers {
pub tx_index_to_first_txout_index: VecReader<TxIndex, TxOutIndex, BytesStrategy<TxOutIndex>>,
pub txout_index_to_output_type: VecReader<TxOutIndex, OutputType, BytesStrategy<OutputType>>,
pub txout_index_to_type_index: VecReader<TxOutIndex, TypeIndex, BytesStrategy<TypeIndex>>,
pub scripts: ScriptReaders,
pub addrbytes: AddrReaders,
}
@@ -56,7 +58,16 @@ impl Readers {
tx_index_to_first_txout_index: vecs.transactions.first_txout_index.reader(),
txout_index_to_output_type: vecs.outputs.output_type.reader(),
txout_index_to_type_index: vecs.outputs.type_index.reader(),
scripts: ScriptReaders {
p2ms_legacy_sigops: vecs.scripts.p2ms.legacy_sigops.reader(),
unknown_legacy_sigops: vecs.scripts.unknown.legacy_sigops.reader(),
},
addrbytes: vecs.addrs.addr_readers(),
}
}
}
pub struct ScriptReaders {
pub p2ms_legacy_sigops: VecReader<P2MSOutputIndex, SigOps, BytesStrategy<SigOps>>,
pub unknown_legacy_sigops: VecReader<UnknownOutputIndex, SigOps, BytesStrategy<SigOps>>,
}
crates/brk_indexer/src/vecs/mod.rs
+36 -2
View File
@@ -14,6 +14,7 @@ mod addrs;
mod blocks;
mod inputs;
mod macros;
mod op_return;
mod outputs;
mod scripts;
mod transactions;
@@ -21,6 +22,7 @@ mod transactions;
pub use addrs::*;
pub use blocks::*;
pub use inputs::*;
pub use op_return::*;
pub use outputs::*;
pub use scripts::*;
pub use transactions::*;
@@ -34,6 +36,8 @@ pub struct Vecs<M: StorageMode = Rw> {
pub blocks: BlocksVecs<M>,
#[traversable(wrap = "transactions", rename = "raw")]
pub transactions: TransactionsVecs<M>,
#[traversable(wrap = "transactions", rename = "features")]
pub transaction_features: TransactionFeaturesVecs<M>,
#[traversable(wrap = "inputs", rename = "raw")]
pub inputs: InputsVecs<M>,
#[traversable(wrap = "outputs", rename = "raw")]
@@ -42,6 +46,8 @@ pub struct Vecs<M: StorageMode = Rw> {
pub addrs: AddrsVecs<M>,
#[traversable(wrap = "scripts", rename = "raw")]
pub scripts: ScriptsVecs<M>,
#[traversable(wrap = "op_return", rename = "raw")]
pub op_return: OpReturnVecs<M>,
}
impl Vecs {
@@ -51,23 +57,36 @@ impl Vecs {
tracing::debug!("Setting min len...");
db.set_min_len(PAGE_SIZE * 60_000_000)?;
let (blocks, transactions, inputs, outputs, addrs, scripts) = parallel_import! {
let (
blocks,
transactions,
transaction_features,
inputs,
outputs,
addrs,
scripts,
op_return,
) = parallel_import! {
blocks = BlocksVecs::forced_import(&db, version),
transactions = TransactionsVecs::forced_import(&db, version),
transaction_features = TransactionFeaturesVecs::forced_import(&db, version),
inputs = InputsVecs::forced_import(&db, version),
outputs = OutputsVecs::forced_import(&db, version),
addrs = AddrsVecs::forced_import(&db, version),
scripts = ScriptsVecs::forced_import(&db, version),
op_return = OpReturnVecs::forced_import(&db, version),
};
let this = Self {
db,
blocks,
transactions,
transaction_features,
inputs,
outputs,
addrs,
scripts,
op_return,
};
this.db.retain_regions(
@@ -89,6 +108,12 @@ impl Vecs {
self.transactions
.truncate(starting_lengths.height, starting_lengths.tx_index, stamp)?;
self.transaction_features.truncate(
starting_lengths.height,
starting_lengths.tx_index,
stamp,
)?;
self.inputs
.truncate(starting_lengths.height, starting_lengths.txin_index, stamp)?;
@@ -111,12 +136,17 @@ impl Vecs {
self.scripts.truncate(
starting_lengths.height,
starting_lengths.empty_output_index,
starting_lengths.op_return_index,
starting_lengths.p2ms_output_index,
starting_lengths.unknown_output_index,
stamp,
)?;
self.op_return.truncate(
starting_lengths.height,
starting_lengths.op_return_index,
stamp,
)?;
Ok(())
}
@@ -167,19 +197,23 @@ impl Vecs {
self.blocks
.par_iter_mut_any()
.chain(self.transactions.par_iter_mut_any())
.chain(self.transaction_features.par_iter_mut_any())
.chain(self.inputs.par_iter_mut_any())
.chain(self.outputs.par_iter_mut_any())
.chain(self.addrs.par_iter_mut_any())
.chain(self.scripts.par_iter_mut_any())
.chain(self.op_return.par_iter_mut_any())
}
fn iter_any_stored_vec(&self) -> impl Iterator<Item = &dyn AnyStoredVec> {
self.blocks
.iter_any()
.chain(self.transactions.iter_any())
.chain(self.transaction_features.iter_any())
.chain(self.inputs.iter_any())
.chain(self.outputs.iter_any())
.chain(self.addrs.iter_any())
.chain(self.scripts.iter_any())
.chain(self.op_return.iter_any())
}
}
crates/brk_indexer/src/vecs/op_return.rs
+70
View File
@@ -0,0 +1,70 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, OpReturnIndex, OpReturnKind, StoredU32, TxIndex, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, ImportableVec, PcoVec, Rw, Stamp, StorageMode, WritableVec};
use crate::parallel_import;
#[derive(Traversable)]
pub struct OpReturnVecs<M: StorageMode = Rw> {
pub first_index: M::Stored<PcoVec<Height, OpReturnIndex>>,
pub to_tx_index: M::Stored<PcoVec<OpReturnIndex, TxIndex>>,
pub kind: M::Stored<PcoVec<OpReturnIndex, OpReturnKind>>,
pub post_op_return_bytes: M::Stored<PcoVec<OpReturnIndex, StoredU32>>,
}
impl OpReturnVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (first_index, to_tx_index, kind, post_op_return_bytes) = parallel_import! {
first_index = PcoVec::forced_import(db, "first_op_return_index", version),
to_tx_index = PcoVec::forced_import(db, "tx_index", version),
kind = PcoVec::forced_import(db, "kind", version),
post_op_return_bytes =
PcoVec::forced_import(db, "op_return_post_op_return_bytes", version),
};
Ok(Self {
first_index,
to_tx_index,
kind,
post_op_return_bytes,
})
}
pub fn truncate(
&mut self,
height: Height,
op_return_index: OpReturnIndex,
stamp: Stamp,
) -> Result<()> {
self.first_index
.truncate_if_needed_with_stamp(height, stamp)?;
self.to_tx_index
.truncate_if_needed_with_stamp(op_return_index, stamp)?;
self.kind
.truncate_if_needed_with_stamp(op_return_index, stamp)?;
self.post_op_return_bytes
.truncate_if_needed_with_stamp(op_return_index, stamp)?;
Ok(())
}
pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.first_index as &mut dyn AnyStoredVec,
&mut self.to_tx_index,
&mut self.kind,
&mut self.post_op_return_bytes,
]
.into_par_iter()
}
pub fn iter_any(&self) -> impl Iterator<Item = &dyn AnyStoredVec> {
[
&self.first_index as &dyn AnyStoredVec,
&self.to_tx_index,
&self.kind,
&self.post_op_return_bytes,
]
.into_iter()
}
}
crates/brk_indexer/src/vecs/scripts.rs
+53 -26
View File
@@ -1,13 +1,14 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{
EmptyOutputIndex, Height, OpReturnIndex, P2MSOutputIndex, TxIndex, UnknownOutputIndex, Version,
EmptyOutputIndex, Height, OutputType, P2MSOutputIndex, SigOps, TxIndex, TypeIndex,
UnknownOutputIndex, Version,
};
use rayon::prelude::*;
use schemars::JsonSchema;
use serde::Serialize;
use vecdb::{
AnyStoredVec, Database, Formattable, ImportableVec, PcoVec, PcoVecValue, Rw, Stamp,
AnyStoredVec, BytesVec, Database, Formattable, ImportableVec, PcoVec, PcoVecValue, Rw, Stamp,
StorageMode, VecIndex, WritableVec,
};
@@ -22,51 +23,58 @@ pub struct ScriptTypeVecs<
pub to_tx_index: M::Stored<PcoVec<I, TxIndex>>,
}
#[derive(Traversable)]
pub struct ScriptTypeWithSigOpsVecs<
I: VecIndex + PcoVecValue + Formattable + Serialize + JsonSchema,
M: StorageMode = Rw,
> {
pub first_index: M::Stored<PcoVec<Height, I>>,
pub to_tx_index: M::Stored<PcoVec<I, TxIndex>>,
pub legacy_sigops: M::Stored<BytesVec<I, SigOps>>,
}
#[derive(Traversable)]
pub struct ScriptsVecs<M: StorageMode = Rw> {
pub empty: ScriptTypeVecs<EmptyOutputIndex, M>,
pub op_return: ScriptTypeVecs<OpReturnIndex, M>,
pub p2ms: ScriptTypeVecs<P2MSOutputIndex, M>,
pub unknown: ScriptTypeVecs<UnknownOutputIndex, M>,
pub p2ms: ScriptTypeWithSigOpsVecs<P2MSOutputIndex, M>,
pub unknown: ScriptTypeWithSigOpsVecs<UnknownOutputIndex, M>,
}
impl ScriptsVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
first_empty_output_index,
first_op_return_index,
first_p2ms_output_index,
first_unknown_output_index,
empty_output_index_to_tx_index,
op_return_index_to_tx_index,
p2ms_output_index_to_tx_index,
unknown_output_index_to_tx_index,
p2ms_legacy_sigops,
unknown_legacy_sigops,
) = parallel_import! {
first_empty_output_index = PcoVec::forced_import(db, "first_empty_output_index", version),
first_op_return_index = PcoVec::forced_import(db, "first_op_return_index", version),
first_p2ms_output_index = PcoVec::forced_import(db, "first_p2ms_output_index", version),
first_unknown_output_index = PcoVec::forced_import(db, "first_unknown_output_index", version),
empty_output_index_to_tx_index = PcoVec::forced_import(db, "tx_index", version),
op_return_index_to_tx_index = PcoVec::forced_import(db, "tx_index", version),
p2ms_output_index_to_tx_index = PcoVec::forced_import(db, "tx_index", version),
unknown_output_index_to_tx_index = PcoVec::forced_import(db, "tx_index", version),
p2ms_legacy_sigops = BytesVec::forced_import(db, "p2ms_legacy_sigops", version),
unknown_legacy_sigops = BytesVec::forced_import(db, "unknown_legacy_sigops", version),
};
Ok(Self {
empty: ScriptTypeVecs {
first_index: first_empty_output_index,
to_tx_index: empty_output_index_to_tx_index,
},
op_return: ScriptTypeVecs {
first_index: first_op_return_index,
to_tx_index: op_return_index_to_tx_index,
},
p2ms: ScriptTypeVecs {
p2ms: ScriptTypeWithSigOpsVecs {
first_index: first_p2ms_output_index,
to_tx_index: p2ms_output_index_to_tx_index,
legacy_sigops: p2ms_legacy_sigops,
},
unknown: ScriptTypeVecs {
unknown: ScriptTypeWithSigOpsVecs {
first_index: first_unknown_output_index,
to_tx_index: unknown_output_index_to_tx_index,
legacy_sigops: unknown_legacy_sigops,
},
})
}
@@ -75,7 +83,6 @@ impl ScriptsVecs {
&mut self,
height: Height,
empty_output_index: EmptyOutputIndex,
op_return_index: OpReturnIndex,
p2ms_output_index: P2MSOutputIndex,
unknown_output_index: UnknownOutputIndex,
stamp: Stamp,
@@ -83,9 +90,6 @@ impl ScriptsVecs {
self.empty
.first_index
.truncate_if_needed_with_stamp(height, stamp)?;
self.op_return
.first_index
.truncate_if_needed_with_stamp(height, stamp)?;
self.p2ms
.first_index
.truncate_if_needed_with_stamp(height, stamp)?;
@@ -95,28 +99,31 @@ impl ScriptsVecs {
self.empty
.to_tx_index
.truncate_if_needed_with_stamp(empty_output_index, stamp)?;
self.op_return
.to_tx_index
.truncate_if_needed_with_stamp(op_return_index, stamp)?;
self.p2ms
.to_tx_index
.truncate_if_needed_with_stamp(p2ms_output_index, stamp)?;
self.p2ms
.legacy_sigops
.truncate_if_needed_with_stamp(p2ms_output_index, stamp)?;
self.unknown
.to_tx_index
.truncate_if_needed_with_stamp(unknown_output_index, stamp)?;
self.unknown
.legacy_sigops
.truncate_if_needed_with_stamp(unknown_output_index, stamp)?;
Ok(())
}
pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.empty.first_index as &mut dyn AnyStoredVec,
&mut self.op_return.first_index,
&mut self.p2ms.first_index,
&mut self.unknown.first_index,
&mut self.empty.to_tx_index,
&mut self.op_return.to_tx_index,
&mut self.p2ms.to_tx_index,
&mut self.p2ms.legacy_sigops,
&mut self.unknown.to_tx_index,
&mut self.unknown.legacy_sigops,
]
.into_par_iter()
}
@@ -124,14 +131,34 @@ impl ScriptsVecs {
pub fn iter_any(&self) -> impl Iterator<Item = &dyn AnyStoredVec> {
[
&self.empty.first_index as &dyn AnyStoredVec,
&self.op_return.first_index,
&self.p2ms.first_index,
&self.unknown.first_index,
&self.empty.to_tx_index,
&self.op_return.to_tx_index,
&self.p2ms.to_tx_index,
&self.p2ms.legacy_sigops,
&self.unknown.to_tx_index,
&self.unknown.legacy_sigops,
]
.into_iter()
}
pub(crate) fn legacy_sigops(
&self,
output_type: OutputType,
type_index: TypeIndex,
readers: &crate::readers::ScriptReaders,
) -> Option<SigOps> {
match output_type {
OutputType::P2PK65 | OutputType::P2PK33 | OutputType::P2PKH => Some(SigOps::new(4)),
OutputType::P2MS => self
.p2ms
.legacy_sigops
.get_pushed_or_read(type_index.into(), &readers.p2ms_legacy_sigops),
OutputType::Unknown => self
.unknown
.legacy_sigops
.get_pushed_or_read(type_index.into(), &readers.unknown_legacy_sigops),
_ => Some(SigOps::ZERO),
}
}
}
crates/brk_indexer/src/vecs/transactions/features/counts.rs
+179
View File
@@ -0,0 +1,179 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, StoredU64, TxVersion, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, ImportableVec, PcoVec, Rw, Stamp, StorageMode, WritableVec};
use super::schema::with_transaction_features;
macro_rules! define_counts {
($($(#[$attribute:meta])* $vector:ident: $flag:ident = $bit:literal $(, count: $count:ident)?;)+) => {
#[derive(Default)]
pub(crate) struct TransactionCounts {
v1: u64,
v2: u64,
v3: u64,
other_version: u64,
explicitly_rbf: u64,
one_input: u64,
one_output: u64,
$($(pub(super) $count: u64,)?) +
}
impl TransactionCounts {
pub(crate) fn add_base(
&mut self,
input_count: usize,
output_count: usize,
version: TxVersion,
explicitly_rbf: bool,
) {
match version {
TxVersion::ONE => self.v1 += 1,
TxVersion::TWO => self.v2 += 1,
TxVersion::THREE => self.v3 += 1,
_ => self.other_version += 1,
}
self.explicitly_rbf += explicitly_rbf as u64;
self.one_input += (input_count == 1) as u64;
self.one_output += (output_count == 1) as u64;
}
}
#[derive(Traversable)]
pub struct TransactionCountVecs<M: StorageMode = Rw> {
pub v1: M::Stored<PcoVec<Height, StoredU64>>,
pub v2: M::Stored<PcoVec<Height, StoredU64>>,
pub v3: M::Stored<PcoVec<Height, StoredU64>>,
pub other_version: M::Stored<PcoVec<Height, StoredU64>>,
pub explicitly_rbf: M::Stored<PcoVec<Height, StoredU64>>,
pub one_input: M::Stored<PcoVec<Height, StoredU64>>,
pub one_output: M::Stored<PcoVec<Height, StoredU64>>,
$($(pub $count: M::Stored<PcoVec<Height, StoredU64>>,)?) +
}
impl TransactionCountVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
v1,
v2,
v3,
other_version,
explicitly_rbf,
one_input,
one_output,
$($($count,)?) +
) = crate::parallel_import! {
v1 = PcoVec::forced_import(db, "tx_count_v1", version),
v2 = PcoVec::forced_import(db, "tx_count_v2", version),
v3 = PcoVec::forced_import(db, "tx_count_v3", version),
other_version = PcoVec::forced_import(db, "tx_count_other_version", version),
explicitly_rbf = PcoVec::forced_import(db, "tx_count_explicitly_rbf", version),
one_input = PcoVec::forced_import(db, "tx_count_one_input", version),
one_output = PcoVec::forced_import(db, "tx_count_one_output", version),
$($($count = PcoVec::forced_import(
db,
concat!("tx_count_", stringify!($count)),
version,
),)?) +
};
Ok(Self {
v1,
v2,
v3,
other_version,
explicitly_rbf,
one_input,
one_output,
$($($count,)?) +
})
}
pub(crate) fn checked_push(
&mut self,
height: Height,
counts: TransactionCounts,
) -> Result<()> {
self.v1.checked_push(height, counts.v1.into())?;
self.v2.checked_push(height, counts.v2.into())?;
self.v3.checked_push(height, counts.v3.into())?;
self.other_version
.checked_push(height, counts.other_version.into())?;
self.explicitly_rbf
.checked_push(height, counts.explicitly_rbf.into())?;
self.one_input.checked_push(height, counts.one_input.into())?;
self.one_output.checked_push(height, counts.one_output.into())?;
$($(self.$count.checked_push(height, counts.$count.into())?;)?) +
Ok(())
}
pub(crate) fn truncate(&mut self, height: Height, stamp: Stamp) -> Result<()> {
self.v1.truncate_if_needed_with_stamp(height, stamp)?;
self.v2.truncate_if_needed_with_stamp(height, stamp)?;
self.v3.truncate_if_needed_with_stamp(height, stamp)?;
self.other_version
.truncate_if_needed_with_stamp(height, stamp)?;
self.explicitly_rbf
.truncate_if_needed_with_stamp(height, stamp)?;
self.one_input.truncate_if_needed_with_stamp(height, stamp)?;
self.one_output.truncate_if_needed_with_stamp(height, stamp)?;
$($(self.$count.truncate_if_needed_with_stamp(height, stamp)?;)?) +
Ok(())
}
pub(crate) fn par_iter_mut_any(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.v1 as &mut dyn AnyStoredVec,
&mut self.v2,
&mut self.v3,
&mut self.other_version,
&mut self.explicitly_rbf,
&mut self.one_input,
&mut self.one_output,
$($(&mut self.$count,)?) +
]
.into_par_iter()
}
pub(crate) fn iter_any(&self) -> impl Iterator<Item = &dyn AnyStoredVec> {
[
&self.v1 as &dyn AnyStoredVec,
&self.v2,
&self.v3,
&self.other_version,
&self.explicitly_rbf,
&self.one_input,
&self.one_output,
$($(&self.$count,)?) +
]
.into_iter()
}
}
};
}
with_transaction_features!(define_counts);
#[cfg(test)]
mod tests {
use brk_types::TxVersion;
use super::TransactionCounts;
#[test]
fn counts_base_transaction_properties() {
let mut counts = TransactionCounts::default();
counts.add_base(1, 2, TxVersion::TWO, true);
counts.add_base(2, 1, TxVersion::NON_STANDARD, false);
assert_eq!(counts.v1, 0);
assert_eq!(counts.v2, 1);
assert_eq!(counts.v3, 0);
assert_eq!(counts.other_version, 1);
assert_eq!(counts.explicitly_rbf, 1);
assert_eq!(counts.one_input, 1);
assert_eq!(counts.one_output, 1);
}
}
crates/brk_indexer/src/vecs/transactions/features/flags.rs
+66
View File
@@ -0,0 +1,66 @@
use brk_types::OutputType;
use super::schema::with_transaction_features;
#[derive(Clone, Copy, Default)]
pub(crate) struct TxFeatureFlags(u32);
macro_rules! define_flags {
($($(#[$attribute:meta])* $vector:ident: $flag:ident = $bit:literal $(, count: $count:ident)?;)+) => {
impl TxFeatureFlags {
$(pub(crate) const $flag: u32 = 1 << $bit;)+
}
};
}
with_transaction_features!(define_flags);
impl TxFeatureFlags {
pub(crate) fn insert_type(&mut self, output_type: OutputType) {
self.0 |= match output_type {
OutputType::P2PK65 | OutputType::P2PK33 => Self::P2PK,
OutputType::P2MS => Self::P2MS,
OutputType::P2PKH => Self::P2PKH,
OutputType::P2SH => Self::P2SH,
OutputType::P2WPKH => Self::P2WPKH,
OutputType::P2WSH => Self::P2WSH,
OutputType::P2TR => Self::P2TR,
OutputType::P2A => Self::P2A,
OutputType::OpReturn => Self::OP_RETURN,
OutputType::Empty => Self::EMPTY,
OutputType::Unknown => Self::UNKNOWN,
};
}
#[inline]
pub(crate) fn insert(&mut self, flag: u32) {
self.0 |= flag;
}
#[inline]
pub(crate) fn contains_all(self, flags: u32) -> bool {
self.0 & flags == flags
}
#[inline]
pub(crate) fn is_set(self, flag: u32) -> bool {
self.0 & flag != 0
}
}
#[cfg(test)]
mod tests {
use brk_types::OutputType;
use super::TxFeatureFlags;
#[test]
fn type_flags_union_inputs_and_outputs() {
let mut flags = TxFeatureFlags::default();
flags.insert_type(OutputType::P2PKH);
flags.insert_type(OutputType::P2TR);
assert!(flags.is_set(TxFeatureFlags::P2PKH));
assert!(flags.is_set(TxFeatureFlags::P2TR));
}
}
crates/brk_indexer/src/vecs/transactions/features/mod.rs
+75
View File
@@ -0,0 +1,75 @@
mod counts;
mod flags;
mod schema;
pub use counts::TransactionCountVecs;
pub(crate) use counts::TransactionCounts;
pub(crate) use flags::TxFeatureFlags;
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, StoredBool, TxIndex, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, ImportableVec, PcoVec, Rw, Stamp, StorageMode, WritableVec};
use self::schema::with_transaction_features;
use crate::parallel_import;
macro_rules! define_vecs {
($($(#[$attribute:meta])* $vector:ident: $flag:ident = $bit:literal $(, count: $count:ident)?;)+) => {
#[derive(Traversable)]
pub struct TransactionFeaturesVecs<M: StorageMode = Rw> {
pub count: TransactionCountVecs<M>,
$($(#[$attribute])* pub $vector: M::Stored<PcoVec<TxIndex, StoredBool>>,) +
}
impl TransactionFeaturesVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (count, $($vector,) +) = parallel_import! {
count = TransactionCountVecs::forced_import(db, version),
$($vector = PcoVec::forced_import(db, stringify!($vector), version),) +
};
Ok(Self { count, $($vector,) + })
}
pub(crate) fn push_and_count(
&mut self,
flags: TxFeatureFlags,
counts: &mut TransactionCounts,
) {
$(
let is_set = flags.is_set(TxFeatureFlags::$flag);
self.$vector.push(StoredBool::from(is_set));
$(counts.$count += is_set as u64;)?
) +
}
pub fn truncate(
&mut self,
height: Height,
tx_index: TxIndex,
stamp: Stamp,
) -> Result<()> {
self.count.truncate(height, stamp)?;
$(self.$vector.truncate_if_needed_with_stamp(tx_index, stamp)?;) +
Ok(())
}
pub fn par_iter_mut_any(
&mut self,
) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[$( &mut self.$vector as &mut dyn AnyStoredVec, ) +]
.into_par_iter()
.chain(self.count.par_iter_mut_any())
}
pub fn iter_any(&self) -> impl Iterator<Item = &dyn AnyStoredVec> {
[$( &self.$vector as &dyn AnyStoredVec, ) +]
.into_iter()
.chain(self.count.iter_any())
}
}
};
}
with_transaction_features!(define_vecs);
crates/brk_indexer/src/vecs/transactions/features/schema.rs
+31
View File
@@ -0,0 +1,31 @@
macro_rules! with_transaction_features {
($macro:ident) => {
$macro! {
has_p2pk: P2PK = 0, count: p2pk;
has_p2ms: P2MS = 1, count: p2ms;
has_p2pkh: P2PKH = 2, count: p2pkh;
has_p2sh: P2SH = 3, count: p2sh;
has_p2wpkh: P2WPKH = 4, count: p2wpkh;
has_p2wsh: P2WSH = 5, count: p2wsh;
has_p2tr: P2TR = 6, count: p2tr;
has_p2a: P2A = 7, count: p2a;
has_op_return: OP_RETURN = 8, count: op_return;
has_empty: EMPTY = 9, count: empty;
has_unknown: UNKNOWN = 10, count: unknown;
has_fake_pubkey: FAKE_PUBKEY = 11, count: fake_pubkey;
has_fake_scripthash: FAKE_SCRIPTHASH = 12, count: fake_scripthash;
has_inscription: INSCRIPTION = 13, count: inscription;
has_annex: ANNEX = 14, count: annex;
has_sighash_all: SIGHASH_ALL = 15, count: sighash_all;
has_sighash_none: SIGHASH_NONE = 16, count: sighash_none;
has_sighash_single: SIGHASH_SINGLE = 17, count: sighash_single;
has_sighash_default: SIGHASH_DEFAULT = 18, count: sighash_default;
has_sighash_anyone_can_pay: SIGHASH_ANYONE_CAN_PAY = 19, count: sighash_anyone_can_pay;
#[traversable(hidden)]
is_unconditionally_nonstandard: UNCONDITIONALLY_NONSTANDARD = 20;
has_dust_output: DUST_OUTPUT = 21, count: dust_output;
}
};
}
pub(super) use with_transaction_features;
crates/brk_indexer/src/vecs/transactions.rs → crates/brk_indexer/src/vecs/transactions/mod.rs
+15 -11
View File
@@ -1,8 +1,13 @@
mod features;
pub use features::{TransactionCountVecs, TransactionFeaturesVecs};
pub(crate) use features::{TransactionCounts, TxFeatureFlags};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{
BlkPosition, Height, RawLockTime, SigOps, StoredBool, StoredU32, TxInIndex, TxIndex,
TxOutIndex, TxVersion, Txid, Version,
TxOutIndex, TxVersion, Txid, Version, Weight,
};
use rayon::prelude::*;
use vecdb::{
@@ -17,7 +22,7 @@ pub struct TransactionsVecs<M: StorageMode = Rw> {
pub txid: M::Stored<BytesVec<TxIndex, Txid>>,
pub tx_version: M::Stored<PcoVec<TxIndex, TxVersion>>,
pub raw_locktime: M::Stored<PcoVec<TxIndex, RawLockTime>>,
pub base_size: M::Stored<PcoVec<TxIndex, StoredU32>>,
pub weight: M::Stored<PcoVec<TxIndex, Weight>>,
pub total_size: M::Stored<PcoVec<TxIndex, StoredU32>>,
pub total_sigop_cost: M::Stored<PcoVec<TxIndex, SigOps>>,
pub is_explicitly_rbf: M::Stored<PcoVec<TxIndex, StoredBool>>,
@@ -31,7 +36,7 @@ pub struct TxMetadataVecs<'a> {
pub tx_version: &'a mut PcoVec<TxIndex, TxVersion>,
pub txid: &'a mut BytesVec<TxIndex, Txid>,
pub raw_locktime: &'a mut PcoVec<TxIndex, RawLockTime>,
pub base_size: &'a mut PcoVec<TxIndex, StoredU32>,
pub weight: &'a mut PcoVec<TxIndex, Weight>,
pub total_size: &'a mut PcoVec<TxIndex, StoredU32>,
pub total_sigop_cost: &'a mut PcoVec<TxIndex, SigOps>,
pub is_explicitly_rbf: &'a mut PcoVec<TxIndex, StoredBool>,
@@ -52,7 +57,7 @@ impl TransactionsVecs {
tx_version: &mut self.tx_version,
txid: &mut self.txid,
raw_locktime: &mut self.raw_locktime,
base_size: &mut self.base_size,
weight: &mut self.weight,
total_size: &mut self.total_size,
total_sigop_cost: &mut self.total_sigop_cost,
is_explicitly_rbf: &mut self.is_explicitly_rbf,
@@ -66,7 +71,7 @@ impl TransactionsVecs {
txid,
tx_version,
raw_locktime,
base_size,
weight,
total_size,
total_sigop_cost,
is_explicitly_rbf,
@@ -78,7 +83,7 @@ impl TransactionsVecs {
txid = BytesVec::forced_import(db, "txid", version),
tx_version = PcoVec::forced_import(db, "tx_version", version),
raw_locktime = PcoVec::forced_import(db, "raw_locktime", version),
base_size = PcoVec::forced_import(db, "base_size", version),
weight = PcoVec::forced_import(db, "tx_weight", version),
total_size = PcoVec::forced_import(db, "total_size", version),
total_sigop_cost = PcoVec::forced_import(db, "total_sigop_cost", version),
is_explicitly_rbf = PcoVec::forced_import(db, "is_explicitly_rbf", version),
@@ -91,7 +96,7 @@ impl TransactionsVecs {
txid,
tx_version,
raw_locktime,
base_size,
weight,
total_size,
total_sigop_cost,
is_explicitly_rbf,
@@ -109,8 +114,7 @@ impl TransactionsVecs {
.truncate_if_needed_with_stamp(tx_index, stamp)?;
self.raw_locktime
.truncate_if_needed_with_stamp(tx_index, stamp)?;
self.base_size
.truncate_if_needed_with_stamp(tx_index, stamp)?;
self.weight.truncate_if_needed_with_stamp(tx_index, stamp)?;
self.total_size
.truncate_if_needed_with_stamp(tx_index, stamp)?;
self.total_sigop_cost
@@ -132,7 +136,7 @@ impl TransactionsVecs {
&mut self.txid,
&mut self.tx_version,
&mut self.raw_locktime,
&mut self.base_size,
&mut self.weight,
&mut self.total_size,
&mut self.total_sigop_cost,
&mut self.is_explicitly_rbf,
@@ -149,7 +153,7 @@ impl TransactionsVecs {
&self.txid,
&self.tx_version,
&self.raw_locktime,
&self.base_size,
&self.weight,
&self.total_size,
&self.total_sigop_cost,
&self.is_explicitly_rbf,
crates/brk_mempool/src/snapshot/cluster.rs
+5 -8
View File
@@ -10,11 +10,11 @@ use smallvec::SmallVec;
use super::{SnapTx, TxIndex};
/// Matches Bitcoin Core 31's `MAX_CLUSTER_COUNT_LIMIT`.
pub const MAX_CLUSTER: usize = 64;
pub struct Cluster;
/// Matches Bitcoin Core 31's `MAX_CLUSTER_COUNT_LIMIT`.
const MAX_CLUSTER: usize = 64;
impl Cluster {
/// Capped DFS over the undirected dependency graph (`parents ∪
/// children`) starting from `seed`. Returns the connected component
@@ -99,11 +99,8 @@ impl Cluster {
/// follows all its in-cluster parents.
fn topo_sort(txs: &[SnapTx], component: &[TxIndex]) -> Vec<TxIndex> {
let n = component.len();
let pos: FxHashMap<TxIndex, usize> = component
.iter()
.enumerate()
.map(|(i, &x)| (x, i))
.collect();
let pos: FxHashMap<TxIndex, usize> =
component.iter().enumerate().map(|(i, &x)| (x, i)).collect();
let mut indeg: Vec<u32> = vec![0; n];
let mut children: Vec<Vec<usize>> = vec![Vec::new(); n];
for (i, &idx) in component.iter().enumerate() {
crates/brk_query/src/impl/cpfp.rs
+12 -325
View File
@@ -1,49 +1,17 @@
//! CPFP queries: dispatches between the live mempool path (handled by
//! `brk_mempool`) and the confirmed-tx path built here from indexer
//! and computer vecs.
//!
//! Confirmed clusters are materialized on demand by walking same-block
//! parent/child edges in `TxIndex` space (no `Transaction`
//! reconstruction, no `txid -> tx_index` lookup), then assembling the
//! wire shape directly. The seed's effective fee rate and the per-chunk
//! grouping both read precomputed `effective_fee_rate.tx_index`, which
//! carries the same chunk-rate semantics the live mempool produces.
//! CPFP queries shared by live mempool and confirmed transactions.
use brk_error::{Error, OptionData, Result};
use brk_types::{
CPFP_CHAIN_LIMIT, ChunkInput, CpfpCluster, CpfpClusterTx, CpfpClusterTxIndex, CpfpEntry,
CpfpInfo, FeeRate, Height, Sats, TxInIndex, TxIndex, Txid, TxidPrefix, VSize, Weight,
find_seed_chunk, linearize,
};
use rustc_hash::{FxBuildHasher, FxHashMap};
use smallvec::SmallVec;
use vecdb::{ReadableVec, VecIndex};
mod confirmed;
use brk_error::{Error, Result};
use brk_types::{FeeRate, Txid, TxidPrefix};
use vecdb::ReadableVec;
use crate::Query;
struct WalkResult {
/// Cluster members in `[ancestors..., seed, descendants...]` order,
/// each paired with its in-cluster parent edges resolved to the
/// member's local index. The seed's local index is `ancestors.len()`.
members: Vec<(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)>,
seed_local: CpfpClusterTxIndex,
}
struct Member {
txid: Txid,
fee: Sats,
weight: Weight,
vsize: VSize,
rate: FeeRate,
parents: SmallVec<[CpfpClusterTxIndex; 2]>,
}
impl Query {
/// CPFP cluster for `txid`. Returns the mempool cluster when the
/// txid is unconfirmed; otherwise reconstructs the confirmed
/// same-block cluster from indexer state. Works even when the
/// mempool feature is off.
pub fn cpfp(&self, txid: &Txid) -> Result<CpfpInfo> {
/// Returns live mempool information when available, otherwise
/// reconstructs the confirmed same-block cluster from indexed data.
pub fn cpfp(&self, txid: &Txid) -> Result<brk_types::CpfpInfo> {
let prefix = TxidPrefix::from(txid);
if let Some(info) = self.mempool().and_then(|m| m.cpfp_info(&prefix)) {
return Ok(info);
@@ -51,17 +19,7 @@ impl Query {
self.confirmed_cpfp(txid)
}
/// Effective fee rate for `txid` using the same chunk-rate semantics
/// across paths:
///
/// - Live mempool: snapshot's per-tx linearized `chunk_rate`. If
/// the tx is in the pool but not in the latest snapshot (e.g.
/// just added), falls back to the entry's simple `fee/vsize`.
/// - Confirmed: precomputed `effective_fee_rate.tx_index`.
/// - Graveyard-only RBF predecessor: linearized chunk rate
/// captured at burial.
///
/// Returns `Error::UnknownTxid` for txids not seen in any of those.
/// Effective SFL chunk rate for live, confirmed, or replaced transactions.
pub fn effective_fee_rate(&self, txid: &Txid) -> Result<FeeRate> {
let prefix = TxidPrefix::from(txid);
@@ -71,14 +29,14 @@ impl Query {
return Ok(rate);
}
if let Ok(idx) = self.resolve_tx_index(txid)
if let Ok(index) = self.resolve_tx_index(txid)
&& let Some(rate) = self
.computer()
.transactions
.fees
.effective_fee_rate
.tx_index
.collect_one(idx)
.collect_one(index)
{
return Ok(rate);
}
@@ -91,275 +49,4 @@ impl Query {
Err(Error::UnknownTxid)
}
/// CPFP cluster for a confirmed tx: the connected component of
/// same-block parent/child edges, walked on demand. Per-tx
/// `effective_fee_rate.tx_index` provides each member's chunk rate.
fn confirmed_cpfp(&self, txid: &Txid) -> Result<CpfpInfo> {
let tx_index = self.resolve_tx_index(txid)?;
let height = self.confirmed_status_height(tx_index)?;
let WalkResult {
members,
seed_local,
} = self.walk_same_block_cluster(tx_index, height)?;
let resolved = self.resolve_members(&members)?;
let sigops = self
.indexer()
.vecs
.transactions
.total_sigop_cost
.collect_one(tx_index)
.data()?;
Ok(build_cpfp_info(&resolved, seed_local, sigops))
}
fn resolve_members(
&self,
members: &[(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)],
) -> Result<Vec<Member>> {
let indexer = self.indexer();
let computer = self.computer();
let mut base_size = indexer.vecs.transactions.base_size.cursor();
let mut total_size = indexer.vecs.transactions.total_size.cursor();
let mut fee_cursor = computer.transactions.fees.fee.tx_index.cursor();
let mut rate_cursor = computer
.transactions
.fees
.effective_fee_rate
.tx_index
.cursor();
let txid_reader = indexer.vecs.transactions.txid.reader();
members
.iter()
.map(|(tx_index, parents)| {
let i = tx_index.to_usize();
let weight =
Weight::from_sizes(*base_size.get(i).data()?, *total_size.get(i).data()?);
let vsize = VSize::from(weight);
Ok(Member {
txid: txid_reader.get(i),
fee: fee_cursor.get(i).data()?,
weight,
vsize,
rate: rate_cursor.get(i).data()?,
parents: parents.clone(),
})
})
.collect()
}
/// BFS the seed's same-block ancestors (via `outpoint`) and
/// descendants (via `spent.txin_index` -> `spending_tx`), capped
/// at `CPFP_CHAIN_LIMIT` each side to match Core/mempool.space. Returns members
/// laid out as `[ancestors..., seed, descendants...]` so the seed's
/// local index is `ancestors.len()`.
fn walk_same_block_cluster(&self, seed: TxIndex, height: Height) -> Result<WalkResult> {
let indexer = self.indexer();
let computer = self.computer();
let safe = self.safe_lengths();
let first_tx_index_vec = &indexer.vecs.transactions.first_tx_index;
let block_first = first_tx_index_vec.collect_one(height).data()?;
let next_height = height.incremented();
let block_end = if next_height < safe.height {
first_tx_index_vec.collect_one(next_height).data()?
} else {
safe.tx_index
};
let same_block = |idx: TxIndex| idx >= block_first && idx < block_end;
let mut first_txin = indexer.vecs.transactions.first_txin_index.cursor();
let mut first_txout = indexer.vecs.transactions.first_txout_index.cursor();
let mut outpoint = indexer.vecs.inputs.outpoint.cursor();
let mut spent = computer.outputs.spent.txin_index.cursor();
let mut spending_tx = indexer.vecs.inputs.tx_index.cursor();
let mut walk_inputs = |tx: TxIndex| -> SmallVec<[TxIndex; 2]> {
let mut out: SmallVec<[TxIndex; 2]> = SmallVec::new();
let Ok(start) = first_txin.get(tx.to_usize()).data() else {
return out;
};
let Ok(end) = first_txin.get(tx.to_usize() + 1).data() else {
return out;
};
for i in usize::from(start)..usize::from(end) {
let Ok(op) = outpoint.get(i).data() else {
continue;
};
if op.is_coinbase() {
continue;
}
out.push(op.tx_index());
}
out
};
let mut visited: FxHashMap<TxIndex, ()> =
FxHashMap::with_capacity_and_hasher(2 * CPFP_CHAIN_LIMIT + 1, FxBuildHasher);
visited.insert(seed, ());
// Ancestor BFS: each push records (tx_index, raw parent tx_indices)
// so we can filter against final cluster membership at the end.
let seed_inputs = walk_inputs(seed);
let mut ancestors: Vec<(TxIndex, SmallVec<[TxIndex; 2]>)> = Vec::new();
let mut stack: Vec<SmallVec<[TxIndex; 2]>> = vec![seed_inputs.clone()];
'a: while let Some(parents) = stack.pop() {
for parent in parents {
if ancestors.len() >= CPFP_CHAIN_LIMIT {
break 'a;
}
if visited.insert(parent, ()).is_some() || !same_block(parent) {
continue;
}
let parent_inputs = walk_inputs(parent);
ancestors.push((parent, parent_inputs.clone()));
stack.push(parent_inputs);
}
}
// Descendant BFS via spent outputs.
let mut descendants: Vec<(TxIndex, SmallVec<[TxIndex; 2]>)> = Vec::new();
let mut stack: Vec<TxIndex> = vec![seed];
'd: while let Some(cur) = stack.pop() {
let Ok(start) = first_txout.get(cur.to_usize()).data() else {
continue;
};
let Ok(end) = first_txout.get(cur.to_usize() + 1).data() else {
continue;
};
for i in usize::from(start)..usize::from(end) {
let Ok(txin_idx) = spent.get(i).data() else {
continue;
};
if txin_idx == TxInIndex::UNSPENT {
continue;
}
let Ok(child) = spending_tx.get(usize::from(txin_idx)).data() else {
continue;
};
if visited.insert(child, ()).is_some() || !same_block(child) {
continue;
}
descendants.push((child, walk_inputs(child)));
stack.push(child);
if descendants.len() >= CPFP_CHAIN_LIMIT {
break 'd;
}
}
}
// Lay members out as [ancestors_reverse..., seed, descendants...]
// so parents come before children when a single ancestor chain
// walks back from seed. Reversing the BFS order is good enough
// for wire output; chunk grouping doesn't depend on it.
let ancestor_count = ancestors.len();
let total = ancestor_count + 1 + descendants.len();
let mut local_of: FxHashMap<TxIndex, CpfpClusterTxIndex> =
FxHashMap::with_capacity_and_hasher(total, FxBuildHasher);
let mut members: Vec<(TxIndex, SmallVec<[TxIndex; 2]>)> = Vec::with_capacity(total);
for (tx, raw_parents) in ancestors.into_iter().rev() {
local_of.insert(tx, CpfpClusterTxIndex::from(members.len() as u32));
members.push((tx, raw_parents));
}
let seed_local = CpfpClusterTxIndex::from(members.len() as u32);
local_of.insert(seed, seed_local);
members.push((seed, seed_inputs));
for (tx, raw_parents) in descendants {
local_of.insert(tx, CpfpClusterTxIndex::from(members.len() as u32));
members.push((tx, raw_parents));
}
let resolved: Vec<(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)> = members
.into_iter()
.map(|(tx, raw_parents)| {
let parents: SmallVec<[CpfpClusterTxIndex; 2]> = raw_parents
.iter()
.filter_map(|p| local_of.get(p).copied())
.collect();
(tx, parents)
})
.collect();
Ok(WalkResult {
members: resolved,
seed_local,
})
}
}
fn build_cpfp_info(
members: &[Member],
seed_local: CpfpClusterTxIndex,
sigops: brk_types::SigOps,
) -> CpfpInfo {
let seed_pos = u32::from(seed_local) as usize;
let seed = &members[seed_pos];
let ancestors: Vec<CpfpEntry> = members[..seed_pos]
.iter()
.map(|m| CpfpEntry {
txid: m.txid,
weight: m.weight,
fee: m.fee,
})
.collect();
let descendants: Vec<CpfpEntry> = members[seed_pos + 1..]
.iter()
.map(|m| CpfpEntry {
txid: m.txid,
weight: m.weight,
fee: m.fee,
})
.collect();
let best_descendant = descendants
.iter()
.max_by_key(|e| FeeRate::from((e.fee, e.weight)))
.cloned();
let (cluster, effective_fee_per_vsize) = if members.len() <= 1 {
(None, seed.rate)
} else {
let inputs: Vec<ChunkInput<'_>> = members
.iter()
.map(|m| ChunkInput {
fee: m.fee,
vsize: m.vsize,
parents: m.parents.as_slice(),
})
.collect();
let chunks = linearize(&inputs);
let (chunk_index, seed_rate) = find_seed_chunk(&chunks, seed_local, seed.rate);
let cluster_txs: Vec<CpfpClusterTx> = members
.iter()
.map(|m| CpfpClusterTx {
txid: m.txid,
weight: m.weight,
fee: m.fee,
parents: m.parents.iter().copied().collect(),
})
.collect();
(
Some(CpfpCluster {
txs: cluster_txs,
chunks,
chunk_index,
}),
seed_rate,
)
};
CpfpInfo {
ancestors,
best_descendant,
descendants,
effective_fee_per_vsize,
sigops,
fee: seed.fee,
vsize: seed.vsize,
adjusted_vsize: sigops.adjust_vsize(seed.vsize),
cluster,
}
}
crates/brk_query/src/impl/cpfp/confirmed.rs
+336
View File
@@ -0,0 +1,336 @@
use brk_error::{OptionData, Result};
use brk_types::{
CPFP_CHAIN_LIMIT, ChunkInput, CpfpCluster, CpfpClusterTx, CpfpClusterTxIndex, CpfpEntry,
CpfpInfo, FeeRate, Height, Sats, TxInIndex, TxIndex, Txid, VSize, Weight, find_seed_chunk,
linearize,
};
use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet};
use smallvec::SmallVec;
use vecdb::{ReadableVec, VecIndex};
use crate::Query;
struct WalkResult {
members: Vec<(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)>,
seed_local: CpfpClusterTxIndex,
ancestors: Vec<TxIndex>,
descendants: Vec<TxIndex>,
}
struct Member {
txid: Txid,
fee: Sats,
weight: Weight,
vsize: VSize,
parents: SmallVec<[CpfpClusterTxIndex; 2]>,
}
impl Query {
pub(super) fn confirmed_cpfp(&self, txid: &Txid) -> Result<CpfpInfo> {
let seed = self.resolve_tx_index(txid)?;
let height = self.confirmed_status_height(seed)?;
let walk = self.walk_same_block_cluster(seed, height)?;
let members = self.resolve_members(&walk.members)?;
let ancestors = self.resolve_entries(&walk.ancestors)?;
let descendants = self.resolve_entries(&walk.descendants)?;
let sigops = self
.indexer()
.vecs
.transactions
.total_sigop_cost
.collect_one(seed)
.data()?;
Ok(build_cpfp_info(
&members,
walk.seed_local,
ancestors,
descendants,
sigops,
))
}
fn resolve_members(
&self,
members: &[(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)],
) -> Result<Vec<Member>> {
let indexer = self.indexer();
let computer = self.computer();
let mut weight = indexer.vecs.transactions.weight.cursor();
let mut fee = computer.transactions.fees.fee.tx_index.cursor();
let txid = indexer.vecs.transactions.txid.reader();
members
.iter()
.map(|(index, parents)| {
let position = index.to_usize();
let weight = weight.get(position).data()?;
Ok(Member {
txid: txid.get(position),
fee: fee.get(position).data()?,
weight,
vsize: VSize::from(weight),
parents: parents.clone(),
})
})
.collect()
}
fn resolve_entries(&self, indexes: &[TxIndex]) -> Result<Vec<CpfpEntry>> {
let indexer = self.indexer();
let computer = self.computer();
let mut weight = indexer.vecs.transactions.weight.cursor();
let mut fee = computer.transactions.fees.fee.tx_index.cursor();
let txid = indexer.vecs.transactions.txid.reader();
indexes
.iter()
.map(|index| {
let position = index.to_usize();
Ok(CpfpEntry {
txid: txid.get(position),
fee: fee.get(position).data()?,
weight: weight.get(position).data()?,
})
})
.collect()
}
fn walk_same_block_cluster(&self, seed: TxIndex, height: Height) -> Result<WalkResult> {
let indexer = self.indexer();
let computer = self.computer();
let safe = self.safe_lengths();
let first_tx = &indexer.vecs.transactions.first_tx_index;
let block_first = first_tx.collect_one(height).data()?;
let next_height = height.incremented();
let block_end = if next_height < safe.height {
first_tx.collect_one(next_height).data()?
} else {
safe.tx_index
};
let mut first_txin = indexer.vecs.transactions.first_txin_index.cursor();
let mut input_count = computer.indexes.tx_index.input_count.cursor();
let mut outpoint = indexer.vecs.inputs.outpoint.cursor();
let mut first_txout = indexer.vecs.transactions.first_txout_index.cursor();
let mut output_count = computer.indexes.tx_index.output_count.cursor();
let mut spent = computer.outputs.spent.txin_index.cursor();
let mut spending_tx = indexer.vecs.inputs.tx_index.cursor();
let mut parents_of = |tx: TxIndex| -> Result<SmallVec<[TxIndex; 2]>> {
let position = tx.to_usize();
let first = usize::from(first_txin.get(position).data()?);
let count = u64::from(input_count.get(position).data()?) as usize;
let mut parents = SmallVec::new();
for input in first..first + count {
let parent = outpoint.get(input).data()?;
if !parent.is_coinbase()
&& parent.tx_index() >= block_first
&& parent.tx_index() < block_end
&& !parents.contains(&parent.tx_index())
{
parents.push(parent.tx_index());
}
}
parents.sort_unstable();
Ok(parents)
};
let mut children_of = |tx: TxIndex| -> Result<SmallVec<[TxIndex; 2]>> {
let position = tx.to_usize();
let first = usize::from(first_txout.get(position).data()?);
let count = u64::from(output_count.get(position).data()?) as usize;
let mut children = SmallVec::new();
for output in first..first + count {
let input = spent.get(output).data()?;
if input == TxInIndex::UNSPENT {
continue;
}
let child = spending_tx.get(usize::from(input)).data()?;
if child >= block_first && child < block_end && !children.contains(&child) {
children.push(child);
}
}
children.sort_unstable();
Ok(children)
};
let block_tx_count = block_end.to_usize() - block_first.to_usize();
let mut component =
walk_component(seed, &mut parents_of, &mut children_of, block_tx_count)?;
component.sort_unstable();
let ancestors = walk_direction(seed, &mut parents_of, CPFP_CHAIN_LIMIT)?;
let descendants = walk_direction(seed, &mut children_of, CPFP_CHAIN_LIMIT)?;
let local_of: FxHashMap<TxIndex, CpfpClusterTxIndex> = component
.iter()
.enumerate()
.map(|(local, &tx)| (tx, CpfpClusterTxIndex::from(local as u32)))
.collect();
let seed_local = local_of[&seed];
let members = component
.into_iter()
.map(|tx| {
let parents = parents_of(tx)?
.iter()
.filter_map(|parent| local_of.get(parent).copied())
.collect();
Ok((tx, parents))
})
.collect::<Result<_>>()?;
Ok(WalkResult {
members,
seed_local,
ancestors,
descendants,
})
}
}
fn walk_component(
seed: TxIndex,
parents: &mut impl FnMut(TxIndex) -> Result<SmallVec<[TxIndex; 2]>>,
children: &mut impl FnMut(TxIndex) -> Result<SmallVec<[TxIndex; 2]>>,
limit: usize,
) -> Result<Vec<TxIndex>> {
let mut visited = FxHashSet::with_capacity_and_hasher(limit, FxBuildHasher);
visited.insert(seed);
let mut members = Vec::with_capacity(limit);
members.push(seed);
let mut stack = vec![seed];
while let Some(tx) = stack.pop() {
for neighbor in parents(tx)?.into_iter().chain(children(tx)?) {
if visited.insert(neighbor) {
if members.len() == limit {
return Ok(members);
}
members.push(neighbor);
stack.push(neighbor);
}
}
}
Ok(members)
}
fn walk_direction(
seed: TxIndex,
next: &mut impl FnMut(TxIndex) -> Result<SmallVec<[TxIndex; 2]>>,
limit: usize,
) -> Result<Vec<TxIndex>> {
let mut visited = FxHashSet::with_capacity_and_hasher(limit + 1, FxBuildHasher);
visited.insert(seed);
let mut members = Vec::with_capacity(limit);
let mut stack = next(seed)?.into_vec();
while let Some(tx) = stack.pop() {
if !visited.insert(tx) {
continue;
}
members.push(tx);
if members.len() == limit {
break;
}
stack.extend(next(tx)?);
}
Ok(members)
}
fn build_cpfp_info(
members: &[Member],
seed_local: CpfpClusterTxIndex,
ancestors: Vec<CpfpEntry>,
descendants: Vec<CpfpEntry>,
sigops: brk_types::SigOps,
) -> CpfpInfo {
let seed_position = u32::from(seed_local) as usize;
let seed = &members[seed_position];
let raw_rate = FeeRate::from((seed.fee, seed.vsize));
let best_descendant = descendants
.iter()
.max_by_key(|entry| FeeRate::from((entry.fee, entry.weight)))
.cloned();
let (cluster, effective_fee_per_vsize) = if members.len() == 1 {
(None, raw_rate)
} else {
let inputs: Vec<ChunkInput<'_>> = members
.iter()
.map(|member| ChunkInput {
fee: member.fee,
vsize: member.vsize,
parents: member.parents.as_slice(),
})
.collect();
let chunks = linearize(&inputs);
let (chunk_index, rate) = find_seed_chunk(&chunks, seed_local, raw_rate);
let txs = members
.iter()
.map(|member| CpfpClusterTx {
txid: member.txid,
weight: member.weight,
fee: member.fee,
parents: member.parents.iter().copied().collect(),
})
.collect();
(
Some(CpfpCluster {
txs,
chunks,
chunk_index,
}),
rate,
)
};
CpfpInfo {
ancestors,
best_descendant,
descendants,
effective_fee_per_vsize,
sigops,
fee: seed.fee,
vsize: seed.vsize,
adjusted_vsize: sigops.adjust_vsize(seed.vsize),
cluster,
}
}
#[cfg(test)]
mod tests {
use brk_types::TxIndex;
use vecdb::VecIndex;
use super::{walk_component, walk_direction};
fn adjacent(
graph: &[Vec<usize>],
tx: TxIndex,
) -> brk_error::Result<smallvec::SmallVec<[TxIndex; 2]>> {
Ok(graph[tx.to_usize()]
.iter()
.copied()
.map(TxIndex::from)
.collect())
}
#[test]
fn component_includes_siblings_but_directional_walk_does_not() {
let parents = vec![vec![], vec![0], vec![0]];
let children = vec![vec![1, 2], vec![], vec![]];
let seed = TxIndex::from(1usize);
let mut parent = |tx| adjacent(&parents, tx);
let mut child = |tx| adjacent(&children, tx);
let mut component = walk_component(seed, &mut parent, &mut child, 64).unwrap();
component.sort_unstable();
assert_eq!(component, [0usize, 1, 2].map(TxIndex::from));
assert_eq!(
walk_direction(seed, &mut parent, 25).unwrap(),
[TxIndex::from(0usize)]
);
assert!(walk_direction(seed, &mut child, 25).unwrap().is_empty());
}
}
crates/brk_query/src/impl/series.rs
+1 -6
View File
@@ -235,12 +235,7 @@ impl Query {
Index::TxInIndex => v.inputs.first_txin_index.collect_one(h).map(usize::from),
Index::TxOutIndex => v.outputs.first_txout_index.collect_one(h).map(usize::from),
Index::EmptyOutputIndex => v.scripts.empty.first_index.collect_one(h).map(usize::from),
Index::OpReturnIndex => v
.scripts
.op_return
.first_index
.collect_one(h)
.map(usize::from),
Index::OpReturnIndex => v.op_return.first_index.collect_one(h).map(usize::from),
Index::P2MSOutputIndex => v.scripts.p2ms.first_index.collect_one(h).map(usize::from),
Index::UnknownOutputIndex => v
.scripts
crates/brk_store/Cargo.toml
+1
View File
@@ -16,3 +16,4 @@ brk_types = { workspace = true }
byteview = { workspace = true }
fjall = { workspace = true }
rustc-hash = { workspace = true }
tracing = { workspace = true }
crates/brk_store/src/lib.rs
+27 -1
View File
@@ -1,12 +1,25 @@
#![doc = include_str!("../README.md")]
use std::{borrow::Cow, fmt::Debug, fs, hash::Hash, mem, ops::Range, path::Path};
use std::{
borrow::Cow,
fmt::Debug,
fs,
hash::Hash,
mem,
ops::Range,
path::Path,
sync::{
Arc,
atomic::{AtomicU64, Ordering::Relaxed},
},
};
use brk_error::Result;
use brk_types::{Height, Version};
use byteview::ByteView;
use fjall::{Database, Keyspace, KeyspaceCreateOptions, config::*};
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::info;
mod any;
mod item;
@@ -37,6 +50,7 @@ pub struct Store<K, V> {
puts: FxHashMap<K, V>,
dels: FxHashSet<K>,
caches: Vec<FxHashMap<K, V>>,
db_reads: Arc<AtomicU64>,
}
impl<K, V> Store<K, V>
@@ -103,6 +117,7 @@ where
puts: FxHashMap::default(),
dels: FxHashSet::default(),
caches,
db_reads: Arc::new(AtomicU64::new(0)),
})
}
@@ -164,6 +179,8 @@ where
}
}
self.db_reads.fetch_add(1, Relaxed);
if let Some(slice) = self.keyspace.get(ByteView::from(key))? {
Ok(Some(Cow::Owned(V::from(ByteView::from(slice)))))
} else {
@@ -212,6 +229,7 @@ where
for<'a> ByteView: From<&'a K> + From<&'a V>,
{
self.export_meta_if_needed(height)?;
self.report_db_reads();
let puts = mem::take(&mut self.puts);
let dels = mem::take(&mut self.dels);
@@ -286,6 +304,13 @@ where
Ok(())
}
fn report_db_reads(&self) {
let reads = self.db_reads.swap(0, Relaxed);
if reads != 0 {
info!(store = self.name, reads, "Store DB reads");
}
}
fn ingest<'a>(
keyspace: &Keyspace,
puts: impl Iterator<Item = (&'a K, &'a V)>,
@@ -361,6 +386,7 @@ where
fn commit(&mut self, height: Height) -> Result<()> {
self.export_meta_if_needed(height)?;
self.report_db_reads();
let puts = mem::take(&mut self.puts);
let dels = mem::take(&mut self.dels);
crates/brk_types/src/addr_bytes.rs
+43 -55
View File
@@ -38,6 +38,37 @@ impl AddrBytes {
rapidhash::v3::rapidhash_v3(self.as_slice())
}
pub(crate) fn script_payload(
script: &ScriptBuf,
output_type: OutputType,
) -> Result<&[u8], Error> {
let bytes = script.as_bytes();
match output_type {
OutputType::P2PK65 => match bytes.len() {
67 => Ok(&bytes[1..66]),
received => Err(Error::WrongLength {
expected: 67,
received,
}),
},
OutputType::P2PK33 => match bytes.len() {
35 => Ok(&bytes[1..34]),
received => Err(Error::WrongLength {
expected: 35,
received,
}),
},
OutputType::P2PKH => Ok(&bytes[3..23]),
OutputType::P2SH => Ok(&bytes[2..22]),
OutputType::P2WPKH | OutputType::P2WSH | OutputType::P2TR | OutputType::P2A => {
Ok(&bytes[2..])
}
OutputType::P2MS | OutputType::Unknown | OutputType::Empty | OutputType::OpReturn => {
Err(Error::WrongAddrType)
}
}
}
/// Reconstruct the script_pubkey from the address bytes
pub fn to_script_pubkey(&self) -> ScriptBuf {
match self {
@@ -80,64 +111,21 @@ impl TryFrom<(&ScriptBuf, OutputType)> for AddrBytes {
type Error = Error;
fn try_from(tuple: (&ScriptBuf, OutputType)) -> Result<Self, Self::Error> {
let (script, output_type) = tuple;
let bytes = Self::script_payload(script, output_type)?;
match output_type {
OutputType::P2PK65 => {
let bytes = script.as_bytes();
let bytes = match bytes.len() {
67 => &bytes[1..66],
len => {
dbg!(bytes);
return Err(Error::WrongLength {
expected: 67,
received: len,
});
}
};
Ok(Self::P2PK65(P2PK65Bytes::from(bytes)))
}
OutputType::P2PK33 => {
let bytes = script.as_bytes();
let bytes = match bytes.len() {
35 => &bytes[1..34],
len => {
dbg!(bytes);
return Err(Error::WrongLength {
expected: 35,
received: len,
});
}
};
Ok(Self::P2PK33(P2PK33Bytes::from(bytes)))
}
OutputType::P2PKH => {
let bytes = &script.as_bytes()[3..23];
Ok(Self::P2PKH(P2PKHBytes::from(bytes)))
}
OutputType::P2SH => {
let bytes = &script.as_bytes()[2..22];
Ok(Self::P2SH(P2SHBytes::from(bytes)))
}
OutputType::P2WPKH => {
let bytes = &script.as_bytes()[2..];
Ok(Self::P2WPKH(P2WPKHBytes::from(bytes)))
}
OutputType::P2WSH => {
let bytes = &script.as_bytes()[2..];
Ok(Self::P2WSH(P2WSHBytes::from(bytes)))
}
OutputType::P2TR => {
let bytes = &script.as_bytes()[2..];
Ok(Self::P2TR(P2TRBytes::from(bytes)))
}
OutputType::P2A => {
let bytes = &script.as_bytes()[2..];
Ok(Self::P2A(P2ABytes::from(bytes)))
}
Ok(match output_type {
OutputType::P2PK65 => Self::P2PK65(P2PK65Bytes::from(bytes)),
OutputType::P2PK33 => Self::P2PK33(P2PK33Bytes::from(bytes)),
OutputType::P2PKH => Self::P2PKH(P2PKHBytes::from(bytes)),
OutputType::P2SH => Self::P2SH(P2SHBytes::from(bytes)),
OutputType::P2WPKH => Self::P2WPKH(P2WPKHBytes::from(bytes)),
OutputType::P2WSH => Self::P2WSH(P2WSHBytes::from(bytes)),
OutputType::P2TR => Self::P2TR(P2TRBytes::from(bytes)),
OutputType::P2A => Self::P2A(P2ABytes::from(bytes)),
OutputType::P2MS | OutputType::Unknown | OutputType::Empty | OutputType::OpReturn => {
Err(Error::WrongAddrType)
unreachable!()
}
}
})
}
}
crates/brk_types/src/addr_hash.rs
+47 -1
View File
@@ -1,8 +1,10 @@
use bitcoin::ScriptBuf;
use brk_error::Error;
use byteview::ByteView;
use derive_more::Deref;
use vecdb::Bytes;
use super::AddrBytes;
use super::{AddrBytes, OutputType};
#[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Bytes, Hash)]
pub struct AddrHash(u64);
@@ -11,6 +13,13 @@ impl AddrHash {
pub const fn new(value: u64) -> Self {
Self(value)
}
#[inline]
pub fn from_script(script: &ScriptBuf, output_type: OutputType) -> Result<Self, Error> {
Ok(Self(rapidhash::v3::rapidhash_v3(
AddrBytes::script_payload(script, output_type)?,
)))
}
}
impl From<&AddrBytes> for AddrHash {
@@ -38,3 +47,40 @@ impl From<&AddrHash> for ByteView {
Self::new(&value.0.to_be_bytes())
}
}
#[cfg(test)]
mod tests {
use super::AddrHash;
use crate::{
AddrBytes, OutputType, P2ABytes, P2PK33Bytes, P2PK65Bytes, P2PKHBytes, P2SHBytes,
P2TRBytes, P2WPKHBytes, P2WSHBytes,
};
#[test]
fn hashes_borrowed_script_payloads_like_owned_addresses() {
let addresses = [
AddrBytes::from(P2PK65Bytes::from(&[0x04; 65][..])),
AddrBytes::from(P2PK33Bytes::from(&[0x02; 33][..])),
AddrBytes::from(P2PKHBytes::from(&[0x03; 20][..])),
AddrBytes::from(P2SHBytes::from(&[0x04; 20][..])),
AddrBytes::from(P2WPKHBytes::from(&[0x05; 20][..])),
AddrBytes::from(P2WSHBytes::from(&[0x06; 32][..])),
AddrBytes::from(P2TRBytes::from(&[0x07; 32][..])),
AddrBytes::from(P2ABytes::from(&[78, 115][..])),
];
for address in addresses {
let script = address.to_script_pubkey();
let output_type = OutputType::from(&script);
assert_eq!(
AddrHash::from_script(&script, output_type).unwrap(),
AddrHash::from(&address)
);
assert_eq!(
AddrBytes::try_from((&script, output_type)).unwrap(),
address
);
}
}
}
crates/brk_types/src/lib.rs
+2
View File
@@ -98,6 +98,7 @@ mod month6;
mod next_block_hash;
mod ohlc;
mod op_return_index;
mod op_return_kind;
mod option_ext;
mod outpoint;
mod outpoint_prefix;
@@ -296,6 +297,7 @@ pub use month6::*;
pub use next_block_hash::*;
pub use ohlc::*;
pub use op_return_index::*;
pub use op_return_kind::*;
pub use option_ext::*;
pub use outpoint::*;
pub use outpoint_prefix::*;
crates/brk_types/src/op_return_kind.rs
+98
View File
@@ -0,0 +1,98 @@
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use strum::{AsRefStr, Display};
use vecdb::{Bytes, Formattable, Pco, TransparentPco};
#[derive(
Debug,
Clone,
Copy,
AsRefStr,
Display,
PartialEq,
Eq,
PartialOrd,
Ord,
Serialize,
Deserialize,
JsonSchema,
Hash,
)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
#[repr(u8)]
pub enum OpReturnKind {
Runes,
VeriBlock,
Omni,
Stacks,
Blockstack,
Colu,
OpenAssets,
Komodo,
CoinSpark,
Poet,
Docproof,
OpenTimestamps,
Factom,
EternityWall,
Memo,
Bitproof,
Ascribe,
Stampery,
Epobc,
BareHash,
Text,
Empty,
Unknown,
}
impl OpReturnKind {
fn is_valid(value: u8) -> bool {
value <= Self::Unknown as u8
}
}
impl Formattable for OpReturnKind {
#[inline(always)]
fn write_to(&self, buf: &mut Vec<u8>) {
buf.extend_from_slice(self.as_ref().as_bytes());
}
fn fmt_json(&self, buf: &mut Vec<u8>) {
buf.push(b'"');
self.write_to(buf);
buf.push(b'"');
}
}
impl Bytes for OpReturnKind {
type Array = [u8; size_of::<Self>()];
#[inline]
fn to_bytes(&self) -> Self::Array {
[*self as u8]
}
#[inline]
fn from_bytes(bytes: &[u8]) -> vecdb::Result<Self> {
if bytes.len() != size_of::<Self>() {
return Err(vecdb::Error::WrongLength {
expected: size_of::<Self>(),
received: bytes.len(),
});
}
let value = bytes[0];
if !Self::is_valid(value) {
return Err(vecdb::Error::InvalidArgument("invalid OpReturnKind"));
}
// SAFETY: We validated that value is a valid variant.
Ok(unsafe { std::mem::transmute::<u8, Self>(value) })
}
}
impl Pco for OpReturnKind {
type NumberType = u8;
}
impl TransparentPco<u8> for OpReturnKind {}
crates/brk_types/src/output_type.rs
+43 -34
View File
@@ -3,7 +3,14 @@
reason = "P2PK65 and P2PK33 both serialize as 'p2pk'"
)]
use bitcoin::{AddressType, ScriptBuf, opcodes::all::OP_PUSHBYTES_2};
use bitcoin::{
AddressType, ScriptBuf,
opcodes::all::{
OP_CHECKSIG, OP_DUP, OP_EQUAL, OP_EQUALVERIFY, OP_HASH160, OP_PUSHBYTES_0, OP_PUSHBYTES_2,
OP_PUSHBYTES_20, OP_PUSHBYTES_32, OP_PUSHBYTES_33, OP_PUSHBYTES_65, OP_PUSHNUM_1,
OP_RETURN,
},
};
use brk_error::Error;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@@ -147,41 +154,43 @@ impl OutputType {
impl From<&ScriptBuf> for OutputType {
#[inline]
fn from(script: &ScriptBuf) -> Self {
if script.is_p2pkh() {
Self::P2PKH
} else if script.is_p2wpkh() {
Self::P2WPKH
} else if script.is_p2wsh() {
Self::P2WSH
} else if script.is_p2tr() {
Self::P2TR
} else if script.is_p2sh() {
Self::P2SH
} else if script.is_op_return() {
Self::OpReturn
} else if script.is_p2pk() {
let bytes = script.as_bytes();
let bytes = script.as_bytes();
match bytes.len() {
67 => Self::P2PK65,
35 => Self::P2PK33,
_ => {
dbg!(bytes);
unreachable!()
}
match bytes.len() {
0 => Self::Empty,
4 if bytes == [OP_PUSHNUM_1.to_u8(), OP_PUSHBYTES_2.to_u8(), 78, 115] => Self::P2A,
22 if bytes[0] == OP_PUSHBYTES_0.to_u8() && bytes[1] == OP_PUSHBYTES_20.to_u8() => {
Self::P2WPKH
}
} else if script.is_multisig() {
Self::P2MS
} else if script.witness_version() == Some(bitcoin::WitnessVersion::V1)
&& script.len() == 4
&& script.as_bytes()[1] == OP_PUSHBYTES_2.to_u8()
&& script.as_bytes()[2..4] == [78, 115]
{
Self::P2A
} else if script.is_empty() {
Self::Empty
} else {
Self::Unknown
23 if bytes[0] == OP_HASH160.to_u8()
&& bytes[1] == OP_PUSHBYTES_20.to_u8()
&& bytes[22] == OP_EQUAL.to_u8() =>
{
Self::P2SH
}
25 if bytes[0] == OP_DUP.to_u8()
&& bytes[1] == OP_HASH160.to_u8()
&& bytes[2] == OP_PUSHBYTES_20.to_u8()
&& bytes[23] == OP_EQUALVERIFY.to_u8()
&& bytes[24] == OP_CHECKSIG.to_u8() =>
{
Self::P2PKH
}
34 if bytes[0] == OP_PUSHBYTES_0.to_u8() && bytes[1] == OP_PUSHBYTES_32.to_u8() => {
Self::P2WSH
}
34 if bytes[0] == OP_PUSHNUM_1.to_u8() && bytes[1] == OP_PUSHBYTES_32.to_u8() => {
Self::P2TR
}
35 if bytes[0] == OP_PUSHBYTES_33.to_u8() && bytes[34] == OP_CHECKSIG.to_u8() => {
Self::P2PK33
}
67 if bytes[0] == OP_PUSHBYTES_65.to_u8() && bytes[66] == OP_CHECKSIG.to_u8() => {
Self::P2PK65
}
_ if bytes[0] == OP_RETURN.to_u8() => Self::OpReturn,
_ if script.is_multisig() => Self::P2MS,
_ => Self::Unknown,
}
}
}
crates/brk_types/src/tx.rs
+3 -3
View File
@@ -2,7 +2,7 @@ use crate::{
FeeRate, RawLockTime, Sats, SigOps, TxIn, TxIndex, TxOut, TxStatus, TxVersionRaw, Txid, VSize,
Weight, Witness,
};
use bitcoin::Script;
use bitcoin::{Script, constants::WITNESS_SCALE_FACTOR};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use vecdb::CheckedSub;
@@ -153,8 +153,8 @@ impl Transaction {
SigOps::from(
legacy
.saturating_mul(4)
.saturating_add(redeem.saturating_mul(4))
.saturating_mul(WITNESS_SCALE_FACTOR)
.saturating_add(redeem.saturating_mul(WITNESS_SCALE_FACTOR))
.saturating_add(witness),
)
}
crates/brk_types/src/weight.rs
+18 -5
View File
@@ -38,15 +38,17 @@ pub struct Weight(u64);
impl Weight {
/// Maximum block weight in Bitcoin (4 million weight units).
/// Note: Pre-SegWit 1MB blocks have weight = size * 4 = 4M, so this is consistent across all blocks.
pub const MAX_BLOCK: Self = Self(4_000_000);
pub const MAX_BLOCK: Self = Self(bitcoin::Weight::MAX_BLOCK.to_wu());
/// Compute weight from base size and total size.
/// Formula: weight = base_size * 3 + total_size
/// (since total_size = base_size + witness_size, this equals base_size * 4 + witness_size)
#[inline]
pub fn from_sizes(base_size: u32, total_size: u32) -> Self {
let wu = base_size as u64 * 3 + total_size as u64;
Self(wu)
let base_size = u64::from(base_size);
let witness_size = u64::from(total_size) - base_size;
Self::from(
bitcoin::Weight::from_non_witness_data_size(base_size)
+ bitcoin::Weight::from_witness_data_size(witness_size),
)
}
pub fn to_vbytes_ceil(&self) -> u64 {
@@ -183,3 +185,14 @@ impl Formattable for Weight {
buf.extend_from_slice(b.format(self.0).as_bytes());
}
}
#[cfg(test)]
mod tests {
use super::Weight;
#[test]
fn derives_weight_from_base_and_witness_sizes() {
assert_eq!(*Weight::from_sizes(100, 100), 400);
assert_eq!(*Weight::from_sizes(100, 125), 425);
}
}
rust-toolchain.toml
+1 -1
View File
@@ -1,2 +1,2 @@
[toolchain]
channel = "1.96.1"
channel = "1.97.0"