diff --git a/.gitignore b/.gitignore index c43448246..5a32aa3f6 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ _* /playground /*.txt /*.csv +/tmp # Logs *.log* diff --git a/Cargo.lock b/Cargo.lock index 22487ce53..12277c30d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -659,6 +659,7 @@ dependencies = [ "byteview", "fjall", "rustc-hash", + "tracing", ] [[package]] @@ -750,9 +751,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424" [[package]] name = "byteorder" @@ -768,9 +769,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "byteview" @@ -780,9 +781,9 @@ checksum = "1c53ba0f290bfc610084c05582d9c5d421662128fc69f4bf236707af6fd321b9" [[package]] name = "cc" -version = "1.2.65" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -1013,18 +1014,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1032,18 +1033,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] @@ -1060,9 +1061,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -1092,9 +1093,9 @@ dependencies = [ [[package]] name = "defmt" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" dependencies = [ "bitflags 1.3.2", "defmt-macros", @@ -1102,12 +1103,11 @@ dependencies = [ [[package]] name = "defmt-macros" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" dependencies = [ "defmt-parser", - "proc-macro-error2", "proc-macro2", "quote", "syn", @@ -1292,9 +1292,7 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fjall" -version = "3.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038acd422d607e0eca09e093f299f9eccf9bd097554343d93746afff81a45113" +version = "3.0.4" dependencies = [ "byteorder-lite", "byteview", @@ -1302,7 +1300,7 @@ dependencies = [ "flume", "log", "lsm-tree", - "lz4_flex", + "lz4_flex 0.11.6", "tempfile", "xxhash-rust", ] @@ -1454,18 +1452,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi 5.3.0", - "wasip2", -] - [[package]] name = "getrandom" version = "0.4.3" @@ -1474,7 +1460,7 @@ checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "r-efi 6.0.0", + "r-efi", ] [[package]] @@ -1576,9 +1562,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -1586,9 +1572,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -1876,9 +1862,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" +checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e" dependencies = [ "defmt", "jiff-static", @@ -1891,9 +1877,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" +checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" dependencies = [ "proc-macro2", "quote", @@ -1902,11 +1888,11 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] @@ -2031,9 +2017,7 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lsm-tree" -version = "3.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ef86c3c797c10eefcc73407c43ae48c19d4df686131a8334b2895a513e91df4" +version = "3.0.4" dependencies = [ "byteorder-lite", "byteview", @@ -2041,7 +2025,7 @@ dependencies = [ "enum_dispatch", "interval-heap", "log", - "lz4_flex", + "lz4_flex 0.11.6", "quick_cache", "rustc-hash", "self_cell", @@ -2053,13 +2037,19 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" dependencies = [ "twox-hash", ] +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" + [[package]] name = "matchit" version = "0.8.4" @@ -2068,9 +2058,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmap2" @@ -2114,9 +2104,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -2371,28 +2361,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" -[[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" -dependencies = [ - "proc-macro2", - "quote", -] - -[[package]] -name = "proc-macro-error2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" -dependencies = [ - "proc-macro-error-attr2", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "proc-macro2" version = "1.0.106" @@ -2430,12 +2398,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - [[package]] name = "r-efi" version = "6.0.0" @@ -2459,18 +2421,18 @@ dependencies = [ [[package]] name = "rapidhash" -version = "4.4.2" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" dependencies = [ "rustversion", ] [[package]] name = "rawdb" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b22c0f9f22e83612f89b67f341029c59d652f59f88d053037ad464ff4f1f886" +checksum = "7df8d0fa0319c87c369464ffe5ee0baa6659ed37d3426a9ae021fb463d95ad52" dependencies = [ "libc", "log", @@ -2543,9 +2505,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.4" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" dependencies = [ "aho-corasick", "memchr", @@ -2555,9 +2517,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" dependencies = [ "aho-corasick", "memchr", @@ -2595,15 +2557,15 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -2629,9 +2591,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "log", "once_cell", @@ -2664,9 +2626,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ryu" @@ -2909,9 +2871,9 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2919,9 +2881,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] @@ -3041,9 +3003,9 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] @@ -3386,14 +3348,14 @@ checksum = "bfa6c38708f6257f1ec2ca7e5a11f9bbf58a27d7060078b6b333624968183d96" [[package]] name = "vecdb" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66ff235ce524e97c0d2a8e386fe842b6939016b90ed732844cd91e0337bd5e1" +checksum = "294fcad993a26e2668dca931c610b8461e66837b5f359a0c1a89bdf357cacf4b" dependencies = [ "itoa", "libc", "log", - "lz4_flex", + "lz4_flex 0.13.1", "parking_lot", "pco", "rawdb", @@ -3409,9 +3371,9 @@ dependencies = [ [[package]] name = "vecdb_derive" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5dadc8894e9260b3ac87613a3cdf2f26c319b1957018b7f87a4e6713c9a14d" +checksum = "87e6ddc694828bd6d372573727e02ea07047f4b65f78aaf859dae2fbf2dbbcfe" dependencies = [ "quote", "syn", @@ -3439,15 +3401,6 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen", -] - [[package]] name = "wasm-bindgen" version = "0.2.126" @@ -3692,9 +3645,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" [[package]] name = "wio" @@ -3705,12 +3658,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - [[package]] name = "writeable" version = "0.6.3" @@ -3759,18 +3706,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", @@ -3839,9 +3786,9 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zstd" diff --git a/Cargo.toml b/Cargo.toml index d6a695a14..247a0308f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,15 +64,15 @@ color-eyre = "0.6.5" corepc-jsonrpc = { package = "jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false } corepc-types = { version = "0.15.0", features = ["std"], default-features = false } derive_more = { version = "2.1.1", features = ["deref", "deref_mut"] } -fjall = "3.1.5" +fjall = { version = "=3.0.4", path = "../fjall" } indexmap = { version = "2.14.0", features = ["serde"] } -jiff = { version = "0.2.31", features = ["perf-inline", "tz-system"], default-features = false } +jiff = { version = "0.2.32", features = ["perf-inline", "tz-system"], default-features = false } owo-colors = "4.3.0" parking_lot = "0.12.5" pco = "1.0.2" rayon = "1.12.0" -rapidhash = "4.4.2" -rustc-hash = "2.1.2" +rapidhash = "4.5.1" +rustc-hash = "2.1.3" schemars = { version = "1.2.1", features = ["indexmap2"] } serde = "1.0.228" serde_bytes = "0.11.19" @@ -84,9 +84,12 @@ tower-http = { version = "0.7.0", features = ["catch-panic", "compression-br", " tower-layer = "0.3" tracing = { version = "0.1", default-features = false, features = ["std"] } ureq = { version = "3.3.0", features = ["json"] } -vecdb = { version = "0.10.3", features = ["derive", "serde_json", "pco", "schemars"] } +vecdb = { version = "0.10.4", features = ["derive", "serde_json", "pco", "schemars"] } # vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] } +[patch.crates-io] +lsm-tree = { path = "../lsm-tree" } + [workspace.metadata.release] shared-version = true tag-name = "v{{version}}" diff --git a/crates/brk_computer/src/distribution/cohorts/utxo/groups.rs b/crates/brk_computer/src/distribution/cohorts/utxo/groups.rs index 158faad8d..783b4fb6a 100644 --- a/crates/brk_computer/src/distribution/cohorts/utxo/groups.rs +++ b/crates/brk_computer/src/distribution/cohorts/utxo/groups.rs @@ -16,6 +16,7 @@ use vecdb::{ use crate::{ blocks, distribution::{ + DynCohortVecs, metrics::{ AllCohortMetrics, BasicCohortMetrics, CohortMetricsBase, CoreCohortMetrics, ExtendedAdjustedCohortMetrics, ExtendedCohortMetrics, ImportConfig, @@ -23,7 +24,6 @@ use crate::{ TypeCohortMetrics, }, state::UTXOCohortState, - DynCohortVecs, }, indexes, internal::{ValuePerBlockCumulativeRolling, WindowStartVec, Windows}, diff --git a/crates/brk_computer/src/indexes/addr.rs b/crates/brk_computer/src/indexes/addr.rs index 7dc0ece1b..b1c8f7463 100644 --- a/crates/brk_computer/src/indexes/addr.rs +++ b/crates/brk_computer/src/indexes/addr.rs @@ -250,12 +250,7 @@ impl Vecs { identity: LazyVecFrom1::init( "op_return_index", version, - indexer - .vecs - .scripts - .op_return - .to_tx_index - .read_only_boxed_clone(), + indexer.vecs.op_return.to_tx_index.read_only_boxed_clone(), |index, _| index, ), }, diff --git a/crates/brk_computer/src/inputs/by_type/compute.rs b/crates/brk_computer/src/inputs/by_type/compute.rs index 621865dd5..c4dd68795 100644 --- a/crates/brk_computer/src/inputs/by_type/compute.rs +++ b/crates/brk_computer/src/inputs/by_type/compute.rs @@ -6,6 +6,8 @@ use vecdb::{AnyVec, Exit, ReadableVec, VecIndex, WritableVec}; use super::{Vecs, WithInputTypes}; use crate::internal::{CoinbasePolicy, PerBlockCumulativeRolling, walk_blocks}; +const WRITE_INTERVAL: usize = 10_000; + impl Vecs { pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { let starting_lengths = indexer.safe_lengths(); @@ -37,6 +39,7 @@ impl Vecs { let mut itype_cursor = indexer.vecs.inputs.output_type.cursor(); let mut fi_in_cursor = indexer.vecs.transactions.first_txin_index.cursor(); + let mut height = skip; walk_blocks( &fi_batch, @@ -65,7 +68,8 @@ impl Vecs { ); push_block(&mut self.tx_count, agg.txs_all, &agg.txs_per_type); - if self.input_count.all.block.batch_limit_reached() { + height += 1; + if height.is_multiple_of(WRITE_INTERVAL) { let _lock = exit.lock(); self.input_count.write()?; self.tx_count.write()?; @@ -79,12 +83,12 @@ impl Vecs { self.input_count.write()?; self.tx_count.write()?; } - - self.input_count - .compute_rest(starting_lengths.height, exit)?; - self.tx_count.compute_rest(starting_lengths.height, exit)?; } + self.input_count + .compute_rest(starting_lengths.height, exit)?; + self.tx_count.compute_rest(starting_lengths.height, exit)?; + for (otype, source) in self.input_count.by_type.iter_typed() { self.input_share.get_mut(otype).compute_count_ratio( source, diff --git a/crates/brk_computer/src/inputs/compute.rs b/crates/brk_computer/src/inputs/compute.rs index d5388e114..308c721c4 100644 --- a/crates/brk_computer/src/inputs/compute.rs +++ b/crates/brk_computer/src/inputs/compute.rs @@ -3,13 +3,12 @@ use brk_indexer::Indexer; use vecdb::Exit; use super::Vecs; -use crate::{blocks, indexes}; +use crate::blocks; impl Vecs { pub(crate) fn compute( &mut self, indexer: &Indexer, - indexes: &indexes::Vecs, blocks: &blocks::Vecs, exit: &Exit, ) -> Result<()> { @@ -18,7 +17,7 @@ impl Vecs { let starting_lengths = indexer.safe_lengths(); self.spent.compute(indexer, exit)?; - self.count.compute(indexer, indexes, blocks, exit)?; + self.count.compute(indexer, blocks, exit)?; self.per_sec.compute(&self.count, &starting_lengths, exit)?; self.by_type.compute(indexer, exit)?; diff --git a/crates/brk_computer/src/inputs/count/compute.rs b/crates/brk_computer/src/inputs/count/compute.rs index 50c78cee3..859d23b0e 100644 --- a/crates/brk_computer/src/inputs/count/compute.rs +++ b/crates/brk_computer/src/inputs/count/compute.rs @@ -3,27 +3,25 @@ use brk_indexer::Indexer; use vecdb::Exit; use super::Vecs; -use crate::{blocks, indexes}; +use crate::blocks; impl Vecs { pub(crate) fn compute( &mut self, indexer: &Indexer, - indexes: &indexes::Vecs, blocks: &blocks::Vecs, exit: &Exit, ) -> Result<()> { let starting_height = indexer.safe_lengths().height; let window_starts = blocks.lookback.window_starts(); - self.0.compute( + + self.sum.compute_count_from_indexes( starting_height, - &indexes.tx_index.input_count, - &indexer.vecs.transactions.first_tx_index, - &indexes.height.tx_index_count, - &window_starts, + &indexer.vecs.inputs.first_txin_index, + &indexer.vecs.inputs.outpoint, exit, - 0, )?; + self.compute_rest(starting_height, &window_starts, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/internal/per_block/computed/aggregated.rs b/crates/brk_computer/src/internal/per_block/computed/aggregated.rs index d001911fb..6c09e8262 100644 --- a/crates/brk_computer/src/internal/per_block/computed/aggregated.rs +++ b/crates/brk_computer/src/internal/per_block/computed/aggregated.rs @@ -2,10 +2,7 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::Height; use schemars::JsonSchema; -use vecdb::{ - AnyStoredVec, AnyVec, Database, EagerVec, Exit, ImportableVec, PcoVec, ReadableVec, Rw, - StorageMode, VecIndex, VecValue, Version, WritableVec, -}; +use vecdb::{Database, EagerVec, Exit, ImportableVec, PcoVec, Rw, StorageMode, Version}; use crate::{ indexes, @@ -52,82 +49,19 @@ where }) } - #[allow(clippy::too_many_arguments)] - pub(crate) fn compute( + pub(crate) fn compute_rest( &mut self, max_from: Height, - source: &impl ReadableVec, - first_indexes: &impl ReadableVec, - count_indexes: &impl ReadableVec, windows: &WindowStarts<'_>, exit: &Exit, - skip_count: usize, ) -> Result<()> where T: From + Default + Copy + Ord, f64: From, - A: VecIndex + VecValue + brk_types::CheckedSub, { - let combined_version = source.version() + first_indexes.version() + count_indexes.version(); - - let mut index = max_from; - index = { - self.sum - .validate_computed_version_or_reset(combined_version)?; - index.min(Height::from(self.sum.len())) - }; - index = { - self.cumulative - .height - .validate_computed_version_or_reset(combined_version)?; - index.min(Height::from(self.cumulative.height.len())) - }; - - let start = index.to_usize(); - - self.sum.truncate_if_needed_at(start)?; - self.cumulative.height.truncate_if_needed_at(start)?; - - let mut cumulative_val = index.decremented().map_or(T::from(0_usize), |idx| { - self.cumulative - .height - .collect_one_at(idx.to_usize()) - .unwrap_or(T::from(0_usize)) - }); - - let fi_len = first_indexes.len(); - let first_indexes_batch: Vec = first_indexes.collect_range_at(start, fi_len); - let count_indexes_batch: Vec = - count_indexes.collect_range_at(start, fi_len); - - first_indexes_batch - .into_iter() - .zip(count_indexes_batch) - .try_for_each(|(first_index, count_index)| -> Result<()> { - let count = u64::from(count_index) as usize; - let effective_count = count.saturating_sub(skip_count); - let effective_first_index = first_index + skip_count.min(count); - - let efi = effective_first_index.to_usize(); - let sum_val = source.fold_range_at( - efi, - efi + effective_count, - T::from(0_usize), - |acc, val| acc + val, - ); - - self.sum.push(sum_val); - cumulative_val += sum_val; - self.cumulative.height.push(cumulative_val); - - Ok(()) - })?; - - let _lock = exit.lock(); - self.sum.write()?; - self.cumulative.height.write()?; - drop(_lock); - + self.cumulative + .height + .compute_cumulative(max_from, &self.sum, exit)?; self.rolling.compute(max_from, windows, &self.sum, exit)?; Ok(()) } diff --git a/crates/brk_computer/src/internal/per_tx/lazy_distribution.rs b/crates/brk_computer/src/internal/per_tx/lazy_distribution.rs index fb55bdd79..82d5c4a0f 100644 --- a/crates/brk_computer/src/internal/per_tx/lazy_distribution.rs +++ b/crates/brk_computer/src/internal/per_tx/lazy_distribution.rs @@ -3,7 +3,7 @@ use brk_indexer::{Indexer, Lengths}; use brk_traversable::Traversable; use brk_types::TxIndex; use schemars::JsonSchema; -use vecdb::{Database, Exit, LazyVecFrom2, ReadableVec, Rw, StorageMode, Version}; +use vecdb::{Database, Exit, LazyVecFrom1, ReadableVec, Rw, StorageMode, Version}; use crate::{ indexes, @@ -11,29 +11,27 @@ use crate::{ }; #[derive(Traversable)] -pub struct LazyPerTxDistribution +pub struct LazyPerTxDistribution where T: ComputedVecValue + PartialOrd + JsonSchema, - S1: ComputedVecValue, - S2: ComputedVecValue, + S: ComputedVecValue, { - pub tx_index: LazyVecFrom2, + pub tx_index: LazyVecFrom1, #[traversable(flatten)] pub distribution: TxDerivedDistribution, } -impl LazyPerTxDistribution +impl LazyPerTxDistribution where T: NumericValue + JsonSchema, - S1: ComputedVecValue + JsonSchema, - S2: ComputedVecValue + JsonSchema, + S: ComputedVecValue + JsonSchema, { pub(crate) fn forced_import( db: &Database, name: &str, version: Version, indexes: &indexes::Vecs, - tx_index: LazyVecFrom2, + tx_index: LazyVecFrom1, ) -> Result { let distribution = TxDerivedDistribution::forced_import(db, name, version, indexes)?; Ok(Self { @@ -52,7 +50,7 @@ where where T: Copy + Ord + From + Default, f64: From, - LazyVecFrom2: ReadableVec, + LazyVecFrom1: ReadableVec, { self.distribution .derive_from(indexer, indexes, starting_lengths, &self.tx_index, exit) diff --git a/crates/brk_computer/src/internal/per_tx/lazy_distribution_transformed.rs b/crates/brk_computer/src/internal/per_tx/lazy_distribution_transformed.rs index aa175813c..78ef0bb5c 100644 --- a/crates/brk_computer/src/internal/per_tx/lazy_distribution_transformed.rs +++ b/crates/brk_computer/src/internal/per_tx/lazy_distribution_transformed.rs @@ -1,36 +1,34 @@ use brk_traversable::Traversable; use brk_types::{TxIndex, Version}; use schemars::JsonSchema; -use vecdb::{LazyVecFrom2, UnaryTransform}; +use vecdb::{LazyVecFrom1, UnaryTransform}; use crate::internal::{ComputedVecValue, LazyTxDerivedDistribution, TxDerivedDistribution}; /// Like `LazyPerTxDistribution` but with a lazy-derived distribution /// (transformed from another type's distribution rather than eagerly computed). #[derive(Clone, Traversable)] -pub struct LazyPerTxDistributionTransformed +pub struct LazyPerTxDistributionTransformed where T: ComputedVecValue + JsonSchema, - S1: ComputedVecValue, - S2: ComputedVecValue, + S: ComputedVecValue, DSource: ComputedVecValue, { - pub tx_index: LazyVecFrom2, + pub tx_index: LazyVecFrom1, #[traversable(flatten)] pub distribution: LazyTxDerivedDistribution, } -impl LazyPerTxDistributionTransformed +impl LazyPerTxDistributionTransformed where T: ComputedVecValue + JsonSchema + 'static, - S1: ComputedVecValue + JsonSchema, - S2: ComputedVecValue + JsonSchema, + S: ComputedVecValue + JsonSchema, DSource: ComputedVecValue + JsonSchema, { pub(crate) fn new>( name: &str, version: Version, - tx_index: LazyVecFrom2, + tx_index: LazyVecFrom1, source_distribution: &TxDerivedDistribution, ) -> Self { let distribution = diff --git a/crates/brk_computer/src/internal/transform/arithmetic.rs b/crates/brk_computer/src/internal/transform/arithmetic.rs index a240bbf90..508d71944 100644 --- a/crates/brk_computer/src/internal/transform/arithmetic.rs +++ b/crates/brk_computer/src/internal/transform/arithmetic.rs @@ -146,11 +146,11 @@ impl UnaryTransform for VBytesToWeight { } } -pub struct VSizeToWeight; +pub struct WeightToVSize; -impl UnaryTransform for VSizeToWeight { +impl UnaryTransform for WeightToVSize { #[inline(always)] - fn apply(vsize: VSize) -> Weight { - Weight::from(vsize) + fn apply(weight: Weight) -> VSize { + VSize::from(weight) } } diff --git a/crates/brk_computer/src/internal/transform/mod.rs b/crates/brk_computer/src/internal/transform/mod.rs index 38aacb6ad..ae2ff420f 100644 --- a/crates/brk_computer/src/internal/transform/mod.rs +++ b/crates/brk_computer/src/internal/transform/mod.rs @@ -8,7 +8,7 @@ mod specialized; pub use arithmetic::{ BlocksToDaysF32, DifficultyToHashF64, HalveCents, HalveDollars, HalveSats, HalveSatsToBitcoin, Identity, MaskSats, OneMinusBp16, OneMinusF64, ReturnF32Tenths, ReturnI8, ReturnU16, - ThsToPhsF32, VBytesToWeight, VSizeToWeight, + ThsToPhsF32, VBytesToWeight, WeightToVSize, }; pub use bps::{ Bp16ToFloat, Bp16ToPercent, Bp32ToFloat, Bp32ToPercent, Bps16ToFloat, Bps16ToPercent, diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 7f31688d4..b9b322051 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -20,6 +20,7 @@ mod internal; mod investing; mod market; mod mining; +mod op_return; mod outputs; mod pools; pub mod price; @@ -44,9 +45,10 @@ pub struct Computer { pub supply: Box>, pub inputs: Box>, pub outputs: Box>, + pub op_return: Box>, } -const VERSION: Version = Version::new(6); +const VERSION: Version = Version::new(7); impl Computer { pub fn forced_import(outputs_path: &Path, indexer: &Indexer) -> Result { @@ -87,8 +89,9 @@ impl Computer { let cached_starts = blocks.lookback.cached_window_starts(); - let (inputs, outputs, mining, transactions, pools, cointime) = - timed("Imported inputs/outputs/mining/tx/pools/cointime", || { + let (inputs, outputs, mining, transactions, pools, cointime, op_return) = timed( + "Imported inputs/outputs/mining/tx/pools/cointime/op_return", + || { thread::scope(|s| -> Result<_> { let inputs_handle = big_thread().spawn_scoped(s, || -> Result<_> { Ok(Box::new(inputs::Vecs::forced_import( @@ -143,15 +146,33 @@ impl Computer { &cached_starts, )?); + let op_return_handle = big_thread().spawn_scoped(s, || -> Result<_> { + Ok(Box::new(op_return::Vecs::forced_import( + &computed_path, + VERSION, + &indexes, + )?)) + })?; + let inputs = inputs_handle.join().unwrap()?; let outputs = outputs_handle.join().unwrap()?; let mining = mining_handle.join().unwrap()?; let transactions = transactions_handle.join().unwrap()?; let pools = pools_handle.join().unwrap()?; + let op_return = op_return_handle.join().unwrap()?; - Ok((inputs, outputs, mining, transactions, pools, cointime)) + Ok(( + inputs, + outputs, + mining, + transactions, + pools, + cointime, + op_return, + )) }) - })?; + }, + )?; // Market, indicators, and distribution are independent; import in parallel. // Supply depends on distribution so it runs after. @@ -225,6 +246,7 @@ impl Computer { inputs, price, outputs, + op_return, }; Self::retain_databases(&computed_path)?; @@ -249,6 +271,7 @@ impl Computer { supply::DB_NAME, inputs::DB_NAME, outputs::DB_NAME, + op_return::DB_NAME, ]; if !computed_path.exists() { @@ -292,8 +315,7 @@ impl Computer { let (inputs_result, prices_result) = rayon::join( || { timed("Computed inputs", || { - self.inputs - .compute(indexer, &self.indexes, &self.blocks, exit) + self.inputs.compute(indexer, &self.blocks, exit) }) }, || { @@ -337,18 +359,19 @@ impl Computer { }) }); + let op_return = scope.spawn(|| { + timed("Computed OP_RETURN", || { + self.op_return.compute(indexer, exit) + }) + }); + timed("Computed outputs", || { - self.outputs.compute( - indexer, - &self.indexes, - &self.inputs, - &self.blocks, - &self.price, - exit, - ) + self.outputs + .compute(indexer, &self.inputs, &self.blocks, &self.price, exit) })?; tx_mining.join().unwrap()?; + op_return.join().unwrap()?; market.join().unwrap()?; Ok(()) })?; @@ -502,7 +525,8 @@ impl_iter_named!( distribution, supply, inputs, - outputs + outputs, + op_return ); fn timed(label: &str, f: impl FnOnce() -> T) -> T { diff --git a/crates/brk_computer/src/op_return/by_kind.rs b/crates/brk_computer/src/op_return/by_kind.rs new file mode 100644 index 000000000..83b58341d --- /dev/null +++ b/crates/brk_computer/src/op_return/by_kind.rs @@ -0,0 +1,89 @@ +use brk_traversable::Traversable; +use brk_types::OpReturnKind; + +macro_rules! define_by_kind { + ($($field:ident => $kind:ident),+ $(,)?) => { + #[derive(Traversable)] + pub struct ByKind { + $(pub $field: T),+ + } + + impl ByKind { + pub fn try_new( + mut create: impl FnMut(OpReturnKind, &'static str) -> Result, + ) -> Result { + Ok(Self { + $($field: create(OpReturnKind::$kind, stringify!($field))?),+ + }) + } + + pub fn iter(&self) -> impl Iterator { + [$( &self.$field ),+].into_iter() + } + + pub fn iter_mut(&mut self) -> impl Iterator { + let Self { $($field),+ } = self; + [$($field),+].into_iter() + } + + pub fn iter_typed(&self) -> impl Iterator { + [$( (OpReturnKind::$kind, &self.$field) ),+].into_iter() + } + + pub fn iter_typed_mut( + &mut self, + ) -> impl Iterator { + let Self { $($field),+ } = self; + [$( (OpReturnKind::$kind, $field) ),+].into_iter() + } + } + }; +} + +define_by_kind! { + runes => Runes, + veri_block => VeriBlock, + omni => Omni, + stacks => Stacks, + blockstack => Blockstack, + colu => Colu, + open_assets => OpenAssets, + komodo => Komodo, + coin_spark => CoinSpark, + poet => Poet, + docproof => Docproof, + open_timestamps => OpenTimestamps, + factom => Factom, + eternity_wall => EternityWall, + memo => Memo, + bitproof => Bitproof, + ascribe => Ascribe, + stampery => Stampery, + epobc => Epobc, + bare_hash => BareHash, + text => Text, + empty => Empty, + unknown => Unknown, +} + +#[cfg(test)] +mod tests { + use std::convert::Infallible; + + use brk_types::OpReturnKind; + + use super::ByKind; + + #[test] + fn covers_every_kind_in_discriminant_order() { + let by_kind = + ByKind::try_new(|kind, _| Ok::<_, Infallible>(kind)).expect("infallible constructor"); + let kinds: Vec<_> = by_kind.iter_typed().collect(); + + assert_eq!(kinds.len(), OpReturnKind::Unknown as usize + 1); + for (index, (kind, value)) in kinds.into_iter().enumerate() { + assert_eq!(kind as usize, index); + assert_eq!(kind, *value); + } + } +} diff --git a/crates/brk_computer/src/op_return/compute.rs b/crates/brk_computer/src/op_return/compute.rs new file mode 100644 index 000000000..ea25690da --- /dev/null +++ b/crates/brk_computer/src/op_return/compute.rs @@ -0,0 +1,242 @@ +use brk_error::Result; +use brk_indexer::Indexer; +use brk_types::{OpReturnKind, VSize}; +use vecdb::{AnyVec, Exit, ReadableVec, VecIndex}; + +use super::{Vecs, vecs::Totals}; + +const KIND_COUNT: usize = OpReturnKind::Unknown as usize + 1; +const OLD_STANDARD_MAX_POST_OP_RETURN_BYTES: u64 = 82; +const WRITE_INTERVAL: usize = 10_000; + +#[derive(Clone, Copy, Default)] +struct PolicyTotals { + oversized: Totals, + multiple: Totals, + pre_v30_nonstandard: Totals, +} + +#[derive(Clone, Copy, Default)] +struct Carrier { + kinds: u32, + output_count: u64, + post_op_return_bytes: u64, + oversized_output_count: u64, + oversized_post_op_return_bytes: u64, + vsize: VSize, +} + +impl Carrier { + fn add_output(&mut self, kind: OpReturnKind, post_op_return_bytes: u64) { + self.kinds |= kind_bit(kind); + self.output_count += 1; + self.post_op_return_bytes += post_op_return_bytes; + if post_op_return_bytes > OLD_STANDARD_MAX_POST_OP_RETURN_BYTES { + self.oversized_output_count += 1; + self.oversized_post_op_return_bytes += post_op_return_bytes; + } + } +} + +impl Vecs { + pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { + self.db.sync_bg_tasks()?; + + let starting_lengths = indexer.safe_lengths(); + let raw = &indexer.vecs.op_return; + let txs = &indexer.vecs.transactions; + let version = raw.first_index.version() + + raw.to_tx_index.version() + + raw.kind.version() + + raw.post_op_return_bytes.version() + + txs.weight.version(); + + self.validate_and_truncate(version, starting_lengths.height)?; + + let skip = self.min_len(); + let end = raw + .first_index + .len() + .min(starting_lengths.height.to_usize()); + if skip < end { + self.truncate_if_needed_at(skip)?; + + let op_return_len = raw + .to_tx_index + .len() + .min(raw.kind.len()) + .min(raw.post_op_return_bytes.len()) + .min(starting_lengths.op_return_index.to_usize()); + let mut tx_cursor = raw.to_tx_index.cursor(); + let mut kind_cursor = raw.kind.cursor(); + let mut post_op_return_bytes = raw.post_op_return_bytes.cursor(); + let mut first_index_cursor = raw.first_index.cursor(); + let mut weight_cursor = txs.weight.cursor(); + first_index_cursor.advance(skip); + let mut start = first_index_cursor.next().unwrap().to_usize(); + + for height in skip..end { + let block_end = if height + 1 < end { + first_index_cursor.next().unwrap().to_usize() + } else { + op_return_len + }; + + tx_cursor.advance(start - tx_cursor.position()); + kind_cursor.advance(start - kind_cursor.position()); + post_op_return_bytes.advance(start - post_op_return_bytes.position()); + + let mut total = Totals::default(); + let mut by_kind = [Totals::default(); KIND_COUNT]; + let mut policy = PolicyTotals::default(); + let mut current_tx = None; + let mut carrier = Carrier::default(); + + for _ in start..block_end { + let tx_index = tx_cursor.next().unwrap(); + let kind = kind_cursor.next().unwrap(); + let bytes = u32::from(post_op_return_bytes.next().unwrap()) as u64; + let kind_index = kind as usize; + + if current_tx != Some(tx_index) { + finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier); + current_tx = Some(tx_index); + carrier = Carrier::default(); + + let tx_position = tx_index.to_usize(); + weight_cursor.advance(tx_position - weight_cursor.position()); + carrier.vsize = VSize::from(weight_cursor.next().unwrap()); + } + + total.output_count += 1; + total.post_op_return_bytes += bytes; + by_kind[kind_index].output_count += 1; + by_kind[kind_index].post_op_return_bytes += bytes; + carrier.add_output(kind, bytes); + } + + finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier); + + self.total.push(total); + for (kind, metrics) in self.by_kind.iter_typed_mut() { + metrics.push(by_kind[kind as usize]); + } + self.policy.oversized.push(policy.oversized); + self.policy.multiple.push(policy.multiple); + self.policy + .pre_v30_nonstandard + .push(policy.pre_v30_nonstandard); + + if (height + 1).is_multiple_of(WRITE_INTERVAL) { + let _lock = exit.lock(); + self.write()?; + } + start = block_end; + } + + let _lock = exit.lock(); + self.write()?; + } + + self.compute_cumulative(starting_lengths.height, exit)?; + + let exit = exit.clone(); + self.db.run_bg(move |db| { + let _lock = exit.lock(); + db.compact_deferred_default() + }); + Ok(()) + } +} + +fn finalize_transaction( + total: &mut Totals, + by_kind: &mut [Totals; KIND_COUNT], + policy: &mut PolicyTotals, + carrier: Carrier, +) { + if carrier.output_count == 0 { + return; + } + + add_carrier(total, carrier.vsize); + let mut kinds = carrier.kinds; + while kinds != 0 { + let kind_index = kinds.trailing_zeros() as usize; + add_carrier(&mut by_kind[kind_index], carrier.vsize); + kinds &= kinds - 1; + } + + if carrier.oversized_output_count > 0 { + policy.oversized.output_count += carrier.oversized_output_count; + policy.oversized.post_op_return_bytes += carrier.oversized_post_op_return_bytes; + add_carrier(&mut policy.oversized, carrier.vsize); + } + + if carrier.output_count > 1 { + policy.multiple.output_count += carrier.output_count; + policy.multiple.post_op_return_bytes += carrier.post_op_return_bytes; + add_carrier(&mut policy.multiple, carrier.vsize); + } + + if carrier.oversized_output_count > 0 || carrier.output_count > 1 { + policy.pre_v30_nonstandard.output_count += carrier.output_count; + policy.pre_v30_nonstandard.post_op_return_bytes += carrier.post_op_return_bytes; + add_carrier(&mut policy.pre_v30_nonstandard, carrier.vsize); + } +} + +fn add_carrier(metrics: &mut Totals, vsize: VSize) { + metrics.carrier_tx_count += 1; + metrics.carrier_vsize += vsize; +} + +const fn kind_bit(kind: OpReturnKind) -> u32 { + 1_u32 << kind as u8 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn multiple_kinds_count_one_total_carrier() { + let mut total = Totals::default(); + let mut by_kind = [Totals::default(); KIND_COUNT]; + let mut policy = PolicyTotals::default(); + let mut carrier = Carrier { + vsize: VSize::new(100), + ..Carrier::default() + }; + carrier.add_output(OpReturnKind::Runes, 15); + carrier.add_output(OpReturnKind::Omni, 15); + + finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier); + + assert_eq!(total.carrier_tx_count, 1); + assert_eq!(by_kind[OpReturnKind::Runes as usize].carrier_tx_count, 1); + assert_eq!(by_kind[OpReturnKind::Omni as usize].carrier_tx_count, 1); + assert_eq!(policy.multiple.output_count, 2); + assert_eq!(policy.pre_v30_nonstandard.carrier_tx_count, 1); + assert_eq!(policy.oversized.carrier_tx_count, 0); + } + + #[test] + fn oversized_output_marks_pre_v30_nonstandard_once() { + let mut total = Totals::default(); + let mut by_kind = [Totals::default(); KIND_COUNT]; + let mut policy = PolicyTotals::default(); + let mut carrier = Carrier { + vsize: VSize::new(120), + ..Carrier::default() + }; + carrier.add_output(OpReturnKind::Unknown, 83); + + finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier); + + assert_eq!(policy.oversized.output_count, 1); + assert_eq!(policy.oversized.carrier_vsize, VSize::new(120)); + assert_eq!(policy.pre_v30_nonstandard.carrier_tx_count, 1); + assert_eq!(policy.multiple.carrier_tx_count, 0); + } +} diff --git a/crates/brk_computer/src/op_return/import.rs b/crates/brk_computer/src/op_return/import.rs new file mode 100644 index 000000000..e222774e7 --- /dev/null +++ b/crates/brk_computer/src/op_return/import.rs @@ -0,0 +1,34 @@ +use std::path::Path; + +use brk_error::Result; +use brk_types::Version; + +use super::{ByKind, Metrics, Policy, Vecs}; +use crate::{ + indexes, + internal::db_utils::{finalize_db, open_db}, +}; + +impl Vecs { + pub(crate) fn forced_import( + parent_path: &Path, + version: Version, + indexes: &indexes::Vecs, + ) -> Result { + let db = open_db(parent_path, super::DB_NAME, 1_000_000)?; + let total = Metrics::forced_import(&db, "op_return", version, indexes)?; + let by_kind = ByKind::try_new(|_, name| { + Metrics::forced_import(&db, &format!("op_return_{name}"), version, indexes) + })?; + let policy = Policy::forced_import(&db, version, indexes)?; + + let this = Self { + db, + total, + by_kind, + policy, + }; + finalize_db(&this.db, &this)?; + Ok(this) + } +} diff --git a/crates/brk_computer/src/op_return/mod.rs b/crates/brk_computer/src/op_return/mod.rs new file mode 100644 index 000000000..c979eb4f8 --- /dev/null +++ b/crates/brk_computer/src/op_return/mod.rs @@ -0,0 +1,9 @@ +mod by_kind; +mod compute; +mod import; +mod vecs; + +pub use by_kind::ByKind; +pub use vecs::{Metrics, Policy, Vecs}; + +pub const DB_NAME: &str = "op_return"; diff --git a/crates/brk_computer/src/op_return/vecs.rs b/crates/brk_computer/src/op_return/vecs.rs new file mode 100644 index 000000000..a3dc0b119 --- /dev/null +++ b/crates/brk_computer/src/op_return/vecs.rs @@ -0,0 +1,274 @@ +use brk_error::Result; +use brk_traversable::Traversable; +use brk_types::{Height, StoredU64, VSize, Version}; +use schemars::JsonSchema; +use vecdb::{ + AnyStoredVec, AnyVec, Database, EagerVec, Exit, ImportableVec, PcoVec, Rw, StorageMode, + WritableVec, +}; + +use super::ByKind; +use crate::{ + indexes, + internal::{NumericValue, PerBlock}, +}; + +#[derive(Traversable)] +pub struct Series +where + T: NumericValue + JsonSchema, +{ + pub block: M::Stored>>, + pub cumulative: PerBlock, +} + +impl Series +where + T: NumericValue + JsonSchema, +{ + fn forced_import( + db: &Database, + name: &str, + version: Version, + indexes: &indexes::Vecs, + ) -> Result { + let block = EagerVec::forced_import(db, name, version)?; + let cumulative = + PerBlock::forced_import(db, &format!("{name}_cumulative"), version, indexes)?; + Ok(Self { block, cumulative }) + } + + fn len(&self) -> usize { + self.block.len() + } + + fn validate_and_truncate(&mut self, version: Version, height: Height) -> Result<()> { + self.block.validate_and_truncate(version, height)?; + Ok(()) + } + + fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> { + self.block.truncate_if_needed_at(len)?; + Ok(()) + } + + fn push(&mut self, block: T) { + self.block.push(block); + } + + fn write(&mut self) -> Result<()> { + self.block.write()?; + Ok(()) + } + + fn compute_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> { + self.cumulative + .height + .compute_cumulative(max_from, &self.block, exit)?; + Ok(()) + } +} + +#[derive(Clone, Copy, Default)] +pub(super) struct Totals { + pub output_count: u64, + pub post_op_return_bytes: u64, + pub carrier_tx_count: u64, + pub carrier_vsize: VSize, +} + +#[derive(Traversable)] +pub struct Metrics { + pub output_count: Series, + pub post_op_return_bytes: Series, + pub carrier_tx_count: Series, + pub carrier_vsize: Series, +} + +impl Metrics { + pub(crate) fn forced_import( + db: &Database, + prefix: &str, + version: Version, + indexes: &indexes::Vecs, + ) -> Result { + Ok(Self { + output_count: Series::forced_import( + db, + &format!("{prefix}_output_count"), + version, + indexes, + )?, + post_op_return_bytes: Series::forced_import( + db, + &format!("{prefix}_post_op_return_bytes"), + version, + indexes, + )?, + carrier_tx_count: Series::forced_import( + db, + &format!("{prefix}_carrier_tx_count"), + version, + indexes, + )?, + carrier_vsize: Series::forced_import( + db, + &format!("{prefix}_carrier_vsize"), + version, + indexes, + )?, + }) + } + + fn len(&self) -> usize { + self.output_count + .len() + .min(self.post_op_return_bytes.len()) + .min(self.carrier_tx_count.len()) + .min(self.carrier_vsize.len()) + } + + pub(super) fn push(&mut self, block: Totals) { + self.output_count.push(block.output_count.into()); + self.post_op_return_bytes + .push(block.post_op_return_bytes.into()); + self.carrier_tx_count.push(block.carrier_tx_count.into()); + self.carrier_vsize.push(block.carrier_vsize); + } + + fn validate_and_truncate(&mut self, version: Version, height: Height) -> Result<()> { + self.output_count.validate_and_truncate(version, height)?; + self.post_op_return_bytes + .validate_and_truncate(version, height)?; + self.carrier_tx_count + .validate_and_truncate(version, height)?; + self.carrier_vsize.validate_and_truncate(version, height)?; + Ok(()) + } + + fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> { + self.output_count.truncate_if_needed_at(len)?; + self.post_op_return_bytes.truncate_if_needed_at(len)?; + self.carrier_tx_count.truncate_if_needed_at(len)?; + self.carrier_vsize.truncate_if_needed_at(len)?; + Ok(()) + } + + fn write(&mut self) -> Result<()> { + self.output_count.write()?; + self.post_op_return_bytes.write()?; + self.carrier_tx_count.write()?; + self.carrier_vsize.write()?; + Ok(()) + } + + fn compute_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> { + self.output_count.compute_cumulative(max_from, exit)?; + self.post_op_return_bytes + .compute_cumulative(max_from, exit)?; + self.carrier_tx_count.compute_cumulative(max_from, exit)?; + self.carrier_vsize.compute_cumulative(max_from, exit)?; + Ok(()) + } +} + +#[derive(Traversable)] +pub struct Policy { + pub oversized: Metrics, + pub multiple: Metrics, + pub pre_v30_nonstandard: Metrics, +} + +impl Policy { + pub(crate) fn forced_import( + db: &Database, + version: Version, + indexes: &indexes::Vecs, + ) -> Result { + let import = |name| { + Metrics::forced_import(db, &format!("op_return_policy_{name}"), version, indexes) + }; + + Ok(Self { + oversized: import("oversized")?, + multiple: import("multiple")?, + pre_v30_nonstandard: import("pre_v30_nonstandard")?, + }) + } + + fn iter(&self) -> impl Iterator { + [&self.oversized, &self.multiple, &self.pre_v30_nonstandard].into_iter() + } + + fn iter_mut(&mut self) -> impl Iterator { + [ + &mut self.oversized, + &mut self.multiple, + &mut self.pre_v30_nonstandard, + ] + .into_iter() + } +} + +#[derive(Traversable)] +pub struct Vecs { + #[traversable(skip)] + pub(crate) db: Database, + pub total: Metrics, + pub by_kind: ByKind>, + pub policy: Policy, +} + +impl Vecs { + pub(crate) fn min_len(&self) -> usize { + self.by_kind + .iter() + .chain(self.policy.iter()) + .map(Metrics::len) + .fold(self.total.len(), usize::min) + } + + pub(crate) fn validate_and_truncate(&mut self, version: Version, height: Height) -> Result<()> { + self.total.validate_and_truncate(version, height)?; + for metrics in self.by_kind.iter_mut() { + metrics.validate_and_truncate(version, height)?; + } + for metrics in self.policy.iter_mut() { + metrics.validate_and_truncate(version, height)?; + } + Ok(()) + } + + pub(crate) fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> { + self.total.truncate_if_needed_at(len)?; + for metrics in self.by_kind.iter_mut() { + metrics.truncate_if_needed_at(len)?; + } + for metrics in self.policy.iter_mut() { + metrics.truncate_if_needed_at(len)?; + } + Ok(()) + } + + pub(crate) fn write(&mut self) -> Result<()> { + self.total.write()?; + for metrics in self.by_kind.iter_mut() { + metrics.write()?; + } + for metrics in self.policy.iter_mut() { + metrics.write()?; + } + Ok(()) + } + + pub(crate) fn compute_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> { + self.total.compute_cumulative(max_from, exit)?; + for metrics in self.by_kind.iter_mut() { + metrics.compute_cumulative(max_from, exit)?; + } + for metrics in self.policy.iter_mut() { + metrics.compute_cumulative(max_from, exit)?; + } + Ok(()) + } +} diff --git a/crates/brk_computer/src/outputs/by_type/compute.rs b/crates/brk_computer/src/outputs/by_type/compute.rs index 55692336c..722073763 100644 --- a/crates/brk_computer/src/outputs/by_type/compute.rs +++ b/crates/brk_computer/src/outputs/by_type/compute.rs @@ -6,6 +6,8 @@ use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec}; use super::{Vecs, WithOutputTypes}; use crate::internal::{CoinbasePolicy, PerBlockCumulativeRolling, walk_blocks}; +const WRITE_INTERVAL: usize = 10_000; + impl Vecs { pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { let starting_lengths = indexer.safe_lengths(); @@ -44,6 +46,7 @@ impl Vecs { let mut otype_cursor = indexer.vecs.outputs.output_type.cursor(); let mut fo_cursor = indexer.vecs.transactions.first_txout_index.cursor(); + let mut height = skip; walk_blocks( &fi_batch, @@ -77,7 +80,8 @@ impl Vecs { .block .push(StoredU64::from(spendable_total)); - if self.output_count.all.block.batch_limit_reached() { + height += 1; + if height.is_multiple_of(WRITE_INTERVAL) { let _lock = exit.lock(); self.output_count.write()?; self.spendable_output_count.block.write()?; @@ -93,14 +97,14 @@ impl Vecs { self.spendable_output_count.block.write()?; self.tx_count.write()?; } - - self.output_count - .compute_rest(starting_lengths.height, exit)?; - self.spendable_output_count - .compute_rest(starting_lengths.height, exit)?; - self.tx_count.compute_rest(starting_lengths.height, exit)?; } + self.output_count + .compute_rest(starting_lengths.height, exit)?; + self.spendable_output_count + .compute_rest(starting_lengths.height, exit)?; + self.tx_count.compute_rest(starting_lengths.height, exit)?; + for (otype, source) in self.output_count.by_type.iter_typed() { self.output_share.get_mut(otype).compute_count_ratio( source, diff --git a/crates/brk_computer/src/outputs/compute.rs b/crates/brk_computer/src/outputs/compute.rs index 99d327eb5..f567dd57b 100644 --- a/crates/brk_computer/src/outputs/compute.rs +++ b/crates/brk_computer/src/outputs/compute.rs @@ -3,14 +3,13 @@ use brk_indexer::Indexer; use vecdb::Exit; use super::Vecs; -use crate::{blocks, indexes, inputs, price}; +use crate::{blocks, inputs, price}; impl Vecs { #[allow(clippy::too_many_arguments)] pub(crate) fn compute( &mut self, indexer: &Indexer, - indexes: &indexes::Vecs, inputs: &inputs::Vecs, blocks: &blocks::Vecs, prices: &price::Vecs, @@ -20,7 +19,7 @@ impl Vecs { let starting_lengths = indexer.safe_lengths(); - self.count.compute(indexer, indexes, blocks, exit)?; + self.count.compute(indexer, blocks, exit)?; self.per_sec.compute(&self.count, &starting_lengths, exit)?; self.value.compute(indexer, prices, exit)?; self.by_type.compute(indexer, exit)?; diff --git a/crates/brk_computer/src/outputs/count/compute.rs b/crates/brk_computer/src/outputs/count/compute.rs index 35adbf9d5..2608745f3 100644 --- a/crates/brk_computer/src/outputs/count/compute.rs +++ b/crates/brk_computer/src/outputs/count/compute.rs @@ -3,27 +3,26 @@ use brk_indexer::Indexer; use vecdb::Exit; use super::Vecs; -use crate::{blocks, indexes}; +use crate::blocks; impl Vecs { pub(crate) fn compute( &mut self, indexer: &Indexer, - indexes: &indexes::Vecs, blocks: &blocks::Vecs, exit: &Exit, ) -> Result<()> { let starting_height = indexer.safe_lengths().height; let window_starts = blocks.lookback.window_starts(); - self.total.compute( + + self.total.sum.compute_count_from_indexes( starting_height, - &indexes.tx_index.output_count, - &indexer.vecs.transactions.first_tx_index, - &indexes.height.tx_index_count, - &window_starts, + &indexer.vecs.outputs.first_txout_index, + &indexer.vecs.outputs.value, exit, - 0, )?; + self.total + .compute_rest(starting_height, &window_starts, exit)?; Ok(()) } } diff --git a/crates/brk_computer/src/price/compute.rs b/crates/brk_computer/src/price/compute.rs index a188b26b0..e7ef5c37b 100644 --- a/crates/brk_computer/src/price/compute.rs +++ b/crates/brk_computer/src/price/compute.rs @@ -3,7 +3,7 @@ use std::ops::Range; use brk_error::Result; use brk_indexer::{Indexer, Lengths}; use brk_oracle::{ - bin_to_cents, cents_to_bin, Config, Oracle, PaymentFilter, START_HEIGHT_FAST, START_HEIGHT_SLOW, + Config, Oracle, PaymentFilter, START_HEIGHT_FAST, START_HEIGHT_SLOW, bin_to_cents, cents_to_bin, }; use brk_types::{Cents, OutputType, Sats, TxIndex, TxOutIndex}; use tracing::info; diff --git a/crates/brk_computer/src/transactions/compute.rs b/crates/brk_computer/src/transactions/compute.rs index 6f67a3dbf..e64dd3072 100644 --- a/crates/brk_computer/src/transactions/compute.rs +++ b/crates/brk_computer/src/transactions/compute.rs @@ -19,7 +19,7 @@ impl Vecs { self.db.sync_bg_tasks()?; let (r1, (r2, r3)) = rayon::join( - || self.count.compute(indexer, &blocks.lookback, exit), + || self.count.compute(indexer, indexes, &blocks.lookback, exit), || { rayon::join( || self.versions.compute(indexer, exit), @@ -34,6 +34,11 @@ impl Vecs { self.fees .compute(indexer, indexes, &inputs.spent, &self.size, exit)?; + self.patterns + .compute(indexer, indexes, &inputs.spent, exit)?; + + self.policy.compute(indexer, indexes, &self.fees, exit)?; + self.volume .compute(indexer, indexes, prices, &self.count, &self.fees, exit)?; diff --git a/crates/brk_computer/src/transactions/count/compute.rs b/crates/brk_computer/src/transactions/count/compute.rs index 491d26a97..778acdaef 100644 --- a/crates/brk_computer/src/transactions/count/compute.rs +++ b/crates/brk_computer/src/transactions/count/compute.rs @@ -3,12 +3,13 @@ use brk_indexer::Indexer; use vecdb::Exit; use super::Vecs; -use crate::blocks; +use crate::{blocks, indexes}; impl Vecs { pub(crate) fn compute( &mut self, indexer: &Indexer, + indexes: &indexes::Vecs, lookback: &blocks::LookbackVecs, exit: &Exit, ) -> Result<()> { @@ -17,10 +18,10 @@ impl Vecs { let window_starts = lookback.window_starts(); self.total .compute(starting_height, &window_starts, exit, |height| { - Ok(height.compute_count_from_indexes( + Ok(height.compute_transform( starting_height, - &indexer.vecs.transactions.first_tx_index, - &indexer.vecs.transactions.txid, + &indexes.height.tx_index_count, + |(height, count, ..)| (height, count), exit, )?) })?; diff --git a/crates/brk_computer/src/transactions/fees/compute.rs b/crates/brk_computer/src/transactions/fees/compute.rs index 7c15f23f0..8239758a0 100644 --- a/crates/brk_computer/src/transactions/fees/compute.rs +++ b/crates/brk_computer/src/transactions/fees/compute.rs @@ -1,6 +1,10 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{FeeRate, OutPoint, Sats, TxInIndex, TxIndex, VSize}; +use brk_types::{ + ChunkInput, CpfpClusterTxIndex, FeeRate, OutPoint, Sats, StoredBool, StoredU64, TxInIndex, + TxIndex, VSize, linearize, +}; +use smallvec::SmallVec; use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec, unlikely}; use super::super::size; @@ -70,7 +74,11 @@ impl Vecs { let dep_version = self.input_value.version() + self.output_value.version() - + size_vecs.vsize.tx_index.version(); + + size_vecs.vsize.tx_index.version() + + indexer.vecs.inputs.outpoint.version() + + indexer.vecs.transactions.first_tx_index.version() + + indexer.vecs.transactions.first_txin_index.version() + + indexes.height.tx_index_count.version(); self.fee .tx_index @@ -80,92 +88,135 @@ impl Vecs { self.effective_fee_rate .tx_index .validate_computed_version_or_reset(dep_version)?; + self.is_cpfp_parent + .validate_computed_version_or_reset(dep_version)?; + self.is_cpfp_child + .validate_computed_version_or_reset(dep_version)?; + self.count + .cpfp_parent + .validate_computed_version_or_reset(dep_version)?; + self.count + .cpfp_child + .validate_computed_version_or_reset(dep_version)?; let target = self .input_value .len() .min(self.output_value.len()) - .min(size_vecs.vsize.tx_index.len()); - let min = self + .min(size_vecs.vsize.tx_index.len()) + .min(starting_lengths.tx_index.to_usize()); + let tx_len = self .fee .tx_index .len() .min(self.fee_rate.len()) .min(self.effective_fee_rate.tx_index.len()) + .min(self.is_cpfp_parent.len()) + .min(self.is_cpfp_child.len()) .min(starting_lengths.tx_index.to_usize()); - - if min >= target { - return Ok(()); - } - - self.fee - .tx_index - .truncate_if_needed(starting_lengths.tx_index)?; - self.fee_rate - .truncate_if_needed(starting_lengths.tx_index)?; - self.effective_fee_rate - .tx_index - .truncate_if_needed(starting_lengths.tx_index)?; - - let start_tx = self.fee.tx_index.len(); - let max_height = indexer.vecs.transactions.first_tx_index.len(); - - let start_height = if start_tx == 0 { - 0 + let max_height = indexer + .vecs + .transactions + .first_tx_index + .len() + .min(starting_lengths.height.to_usize()) + .min(indexes.height.tx_index_count.len()); + let next_height = if tx_len >= target { + max_height } else { indexes .tx_heights - .get_shared(TxIndex::from(start_tx)) + .get_shared(TxIndex::from(tx_len)) .unwrap() .to_usize() }; + let count_len = self + .count + .cpfp_parent + .len() + .min(self.count.cpfp_child.len()) + .min(max_height); + let start_height = count_len.min(next_height); + if start_height >= max_height { + return Ok(()); + } + + let start_tx = indexer + .vecs + .transactions + .first_tx_index + .collect_one_at(start_height) + .unwrap() + .to_usize(); + self.fee + .tx_index + .truncate_if_needed(TxIndex::from(start_tx))?; + self.fee_rate.truncate_if_needed(TxIndex::from(start_tx))?; + self.effective_fee_rate + .tx_index + .truncate_if_needed(TxIndex::from(start_tx))?; + self.is_cpfp_parent + .truncate_if_needed(TxIndex::from(start_tx))?; + self.is_cpfp_child + .truncate_if_needed(TxIndex::from(start_tx))?; + self.count.cpfp_parent.truncate_if_needed_at(start_height)?; + self.count.cpfp_child.truncate_if_needed_at(start_height)?; + + let mut tx_count = indexes.height.tx_index_count.cursor(); + let mut next_block_input = indexer.vecs.inputs.first_txin_index.cursor(); + tx_count.advance(start_height); + next_block_input.advance(start_height + 1); + + let mut input_values = Vec::new(); + let mut output_values = Vec::new(); + let mut vsizes = Vec::new(); + let mut txin_starts = Vec::new(); + let mut outpoints = Vec::new(); + let mut fees = Vec::new(); + let mut cluster = Cluster::default(); + let mut first_tx = start_tx; for h in start_height..max_height { - let first_tx: usize = indexer - .vecs - .transactions - .first_tx_index - .collect_one_at(h) - .unwrap() - .to_usize(); - let n = *indexes.height.tx_index_count.collect_one_at(h).unwrap() as usize; + let n = u64::from(tx_count.next().unwrap()) as usize; if first_tx + n > target { break; } // Batch read all per-tx data for this block - let input_values = self.input_value.collect_range_at(first_tx, first_tx + n); - let output_values = self.output_value.collect_range_at(first_tx, first_tx + n); - let vsizes: Vec = size_vecs + self.input_value + .collect_range_into_at(first_tx, first_tx + n, &mut input_values); + self.output_value + .collect_range_into_at(first_tx, first_tx + n, &mut output_values); + size_vecs .vsize .tx_index - .collect_range_at(first_tx, first_tx + n); - let txin_starts: Vec = indexer + .collect_range_into_at(first_tx, first_tx + n, &mut vsizes); + indexer .vecs .transactions .first_txin_index - .collect_range_at(first_tx, first_tx + n); + .collect_range_into_at(first_tx, first_tx + n, &mut txin_starts); let input_begin = txin_starts[0].to_usize(); let input_end = if h + 1 < max_height { + next_block_input.next().unwrap().to_usize() + } else { indexer .vecs .inputs - .first_txin_index - .collect_one_at(h + 1) - .unwrap() - .to_usize() - } else { - indexer.vecs.inputs.outpoint.len() + .outpoint + .len() + .min(starting_lengths.txin_index.to_usize()) }; - let outpoints: Vec = indexer - .vecs - .inputs - .outpoint - .collect_range_at(input_begin, input_end); + indexer.vecs.inputs.outpoint.collect_range_into_at( + input_begin, + input_end, + &mut outpoints, + ); // Compute fee + fee_rate per tx - let mut fees = Vec::with_capacity(n); + fees.clear(); + fees.reserve(n); for j in 0..n { let fee = if unlikely(input_values[j].is_max()) { Sats::ZERO @@ -178,36 +229,56 @@ impl Vecs { } // Effective fee rate via same-block CPFP clustering - let effective = cluster_fee_rates( + cluster_fee_rates( &txin_starts, &outpoints, input_begin, first_tx, &fees, &vsizes, + &mut cluster, ); - for rate in effective { - self.effective_fee_rate.tx_index.push(rate); + let mut parent_count = 0; + let mut child_count = 0; + for ((&effective, &fee), &vsize) in cluster.rates.iter().zip(&fees).zip(&vsizes) { + let (is_parent, is_child) = cpfp_roles(effective, FeeRate::from((fee, vsize))); + parent_count += is_parent as u64; + child_count += is_child as u64; + self.effective_fee_rate.tx_index.push(effective); + self.is_cpfp_parent.push(StoredBool::from(is_parent)); + self.is_cpfp_child.push(StoredBool::from(is_child)); } + self.count.cpfp_parent.push(StoredU64::from(parent_count)); + self.count.cpfp_child.push(StoredU64::from(child_count)); if h % 1_000 == 0 { let _lock = exit.lock(); self.fee.tx_index.write()?; self.fee_rate.write()?; self.effective_fee_rate.tx_index.write()?; + self.is_cpfp_parent.write()?; + self.is_cpfp_child.write()?; + self.count.cpfp_parent.write()?; + self.count.cpfp_child.write()?; } + + first_tx += n; } let _lock = exit.lock(); self.fee.tx_index.write()?; self.fee_rate.write()?; self.effective_fee_rate.tx_index.write()?; + self.is_cpfp_parent.write()?; + self.is_cpfp_child.write()?; + self.count.cpfp_parent.write()?; + self.count.cpfp_child.write()?; Ok(()) } } -/// Clusters same-block parent-child txs and computes effective fee rate per cluster. +/// Computes SFL chunk rates for each same-block dependency component. fn cluster_fee_rates( txin_starts: &[TxInIndex], outpoints: &[OutPoint], @@ -215,57 +286,251 @@ fn cluster_fee_rates( first_tx: usize, fees: &[Sats], vsizes: &[VSize], -) -> Vec { + cluster: &mut Cluster, +) { let n = fees.len(); - let mut parent: Vec = (0..n).collect(); + cluster.rates.clear(); + cluster.rates.extend( + fees.iter() + .zip(vsizes) + .map(|(&fee, &vsize)| FeeRate::from((fee, vsize))), + ); + cluster.parents.clear(); + cluster.parents.resize_with(n, SmallVec::new); + cluster.roots.clear(); + cluster.roots.extend(0..n); + cluster.members.clear(); + cluster.local_index.clear(); + cluster.local_index.resize(n, usize::MAX); - for j in 1..n { - let start = txin_starts[j].to_usize() - outpoint_base; - let end = if j + 1 < txin_starts.len() { - txin_starts[j + 1].to_usize() - outpoint_base - } else { - outpoints.len() - }; + for child in 0..n { + let mut parents: SmallVec<[usize; 2]> = + same_block_parents(child, txin_starts, outpoints, outpoint_base, first_tx, n).collect(); + parents.sort_unstable(); + parents.dedup(); + for &parent in &parents { + union(&mut cluster.roots, child, parent); + } + cluster.parents[child] = parents; + } - for op in &outpoints[start..end] { - if op.is_coinbase() { - continue; - } - let parent_tx = op.tx_index().to_usize(); - if parent_tx >= first_tx && parent_tx < first_tx + n { - union(&mut parent, j, parent_tx - first_tx); - } + for tx in 0..n { + cluster.members.push((root(&mut cluster.roots, tx), tx)); + } + cluster.members.sort_unstable(); + + let mut start = 0; + while start < n { + let component_root = cluster.members[start].0; + let end = cluster.members[start..] + .partition_point(|&(candidate, _)| candidate == component_root) + + start; + if end - start > 1 { + linearize_component( + &cluster.members[start..end], + &cluster.parents, + fees, + vsizes, + &mut cluster.rates, + &mut cluster.local_index, + &mut cluster.local_parents, + ); + } + start = end; + } +} + +#[allow(clippy::too_many_arguments)] +fn linearize_component( + members: &[(usize, usize)], + parents: &[SmallVec<[usize; 2]>], + fees: &[Sats], + vsizes: &[VSize], + rates: &mut [FeeRate], + local_index: &mut [usize], + local_parents: &mut Vec>, +) { + for (local, &(_, tx)) in members.iter().enumerate() { + local_index[tx] = local; + } + + local_parents.clear(); + local_parents.extend(members.iter().map(|&(_, tx)| { + parents[tx] + .iter() + .map(|&parent| CpfpClusterTxIndex::from(local_index[parent] as u32)) + .collect() + })); + + let inputs: Vec> = members + .iter() + .enumerate() + .map(|(local, &(_, tx))| ChunkInput { + fee: fees[tx], + vsize: vsizes[tx], + parents: local_parents[local].as_slice(), + }) + .collect(); + + for chunk in linearize(&inputs) { + for local in chunk.txs { + rates[members[u32::from(local) as usize].1] = chunk.feerate; } } - - let mut cluster_fee = vec![Sats::ZERO; n]; - let mut cluster_vsize = vec![VSize::from(0u64); n]; - for j in 0..n { - let root = find(&mut parent, j); - cluster_fee[root] += fees[j]; - cluster_vsize[root] += vsizes[j]; - } - - (0..n) - .map(|j| { - let root = find(&mut parent, j); - FeeRate::from((cluster_fee[root], cluster_vsize[root])) - }) - .collect() } -fn find(parent: &mut [usize], mut i: usize) -> usize { - while parent[i] != i { - parent[i] = parent[parent[i]]; - i = parent[i]; +fn union(roots: &mut [usize], left: usize, right: usize) { + let left = root(roots, left); + let right = root(roots, right); + if left != right { + roots[right] = left; } - i } -fn union(parent: &mut [usize], a: usize, b: usize) { - let ra = find(parent, a); - let rb = find(parent, b); - if ra != rb { - parent[ra] = rb; +fn root(roots: &mut [usize], node: usize) -> usize { + let mut root = node; + while roots[root] != root { + root = roots[root]; + } + + let mut current = node; + while roots[current] != current { + let next = roots[current]; + roots[current] = root; + current = next; + } + root +} + +fn same_block_parents<'a>( + tx: usize, + txin_starts: &'a [TxInIndex], + outpoints: &'a [OutPoint], + outpoint_base: usize, + first_tx: usize, + tx_count: usize, +) -> impl Iterator + 'a { + let start = txin_starts[tx].to_usize() - outpoint_base; + let end = txin_starts + .get(tx + 1) + .map_or(outpoints.len(), |index| index.to_usize() - outpoint_base); + + outpoints[start..end].iter().filter_map(move |outpoint| { + let parent = outpoint.tx_index().to_usize(); + (parent >= first_tx && parent < first_tx + tx_count).then_some(parent - first_tx) + }) +} + +fn cpfp_roles(effective: FeeRate, raw: FeeRate) -> (bool, bool) { + (effective > raw, effective < raw) +} + +#[derive(Default)] +struct Cluster { + rates: Vec, + parents: Vec>, + roots: Vec, + members: Vec<(usize, usize)>, + local_index: Vec, + local_parents: Vec>, +} + +#[cfg(test)] +mod tests { + use brk_types::{FeeRate, OutPoint, Sats, TxInIndex, TxIndex, VSize, Vout}; + + use super::{Cluster, cluster_fee_rates, cpfp_roles}; + + #[test] + fn marks_actual_cpfp_roles() { + let mut cluster = Cluster::default(); + cluster_fee_rates( + &[TxInIndex::from(0usize), TxInIndex::from(1usize)], + &[ + OutPoint::COINBASE, + OutPoint::new(TxIndex::from(10usize), Vout::ZERO), + ], + 0, + 10, + &[Sats::new(100), Sats::new(200)], + &[VSize::new(100), VSize::new(100)], + &mut cluster, + ); + + assert_eq!(cluster.rates, [FeeRate::new(1.5), FeeRate::new(1.5)]); + assert_eq!( + [ + cpfp_roles(cluster.rates[0], FeeRate::new(1.0)), + cpfp_roles(cluster.rates[1], FeeRate::new(2.0)), + ], + [(true, false), (false, true)] + ); + } + + #[test] + fn keeps_independent_transaction_rates_separate() { + let mut cluster = Cluster::default(); + cluster_fee_rates( + &[TxInIndex::from(0usize), TxInIndex::from(1usize)], + &[OutPoint::COINBASE, OutPoint::COINBASE], + 0, + 10, + &[Sats::new(100), Sats::new(300)], + &[VSize::new(100), VSize::new(100)], + &mut cluster, + ); + + assert_eq!(cluster.rates, [FeeRate::new(1.0), FeeRate::new(3.0)]); + assert_eq!( + [ + cpfp_roles(cluster.rates[0], FeeRate::new(1.0)), + cpfp_roles(cluster.rates[1], FeeRate::new(3.0)), + ], + [(false, false), (false, false)] + ); + } + + #[test] + fn linearizes_shared_parent_branches_independently_of_sibling_order() { + let txin_starts = [ + TxInIndex::from(0usize), + TxInIndex::from(1usize), + TxInIndex::from(2usize), + ]; + let outpoints = [ + OutPoint::COINBASE, + OutPoint::new(TxIndex::from(10usize), Vout::ZERO), + OutPoint::new(TxIndex::from(10usize), Vout::ZERO), + ]; + let vsizes = [VSize::new(100); 3]; + let mut cluster = Cluster::default(); + + cluster_fee_rates( + &txin_starts, + &outpoints, + 0, + 10, + &[Sats::ZERO, Sats::ZERO, Sats::new(3_000)], + &vsizes, + &mut cluster, + ); + assert_eq!( + cluster.rates, + [FeeRate::new(15.0), FeeRate::new(0.0), FeeRate::new(15.0)] + ); + + cluster_fee_rates( + &txin_starts, + &outpoints, + 0, + 10, + &[Sats::ZERO, Sats::new(3_000), Sats::ZERO], + &vsizes, + &mut cluster, + ); + assert_eq!( + cluster.rates, + [FeeRate::new(15.0), FeeRate::new(15.0), FeeRate::new(0.0)] + ); } } diff --git a/crates/brk_computer/src/transactions/fees/import.rs b/crates/brk_computer/src/transactions/fees/import.rs index 7f1d86a4b..e18bbc1d8 100644 --- a/crates/brk_computer/src/transactions/fees/import.rs +++ b/crates/brk_computer/src/transactions/fees/import.rs @@ -2,7 +2,7 @@ use brk_error::Result; use brk_types::Version; use vecdb::{Database, EagerVec, ImportableVec}; -use super::Vecs; +use super::{CountVecs, Vecs}; use crate::{indexes, internal::PerTxDistribution}; /// Bump this when fee/feerate aggregation logic changes (e.g., skip coinbase, skip zero-fee). @@ -16,6 +16,10 @@ impl Vecs { ) -> Result { let v = version + VERSION; Ok(Self { + count: CountVecs { + cpfp_parent: EagerVec::forced_import(db, "cpfp_parent_count", version)?, + cpfp_child: EagerVec::forced_import(db, "cpfp_child_count", version)?, + }, input_value: EagerVec::forced_import(db, "input_value", version)?, output_value: EagerVec::forced_import(db, "output_value", version)?, fee: PerTxDistribution::forced_import(db, "fee", v, indexes)?, @@ -26,6 +30,8 @@ impl Vecs { v, indexes, )?, + is_cpfp_parent: EagerVec::forced_import(db, "is_cpfp_parent", version)?, + is_cpfp_child: EagerVec::forced_import(db, "is_cpfp_child", version)?, }) } } diff --git a/crates/brk_computer/src/transactions/fees/mod.rs b/crates/brk_computer/src/transactions/fees/mod.rs index 1136f9ebd..a3b2299b5 100644 --- a/crates/brk_computer/src/transactions/fees/mod.rs +++ b/crates/brk_computer/src/transactions/fees/mod.rs @@ -2,4 +2,4 @@ mod compute; mod import; mod vecs; -pub use vecs::Vecs; +pub use vecs::{CountVecs, Vecs}; diff --git a/crates/brk_computer/src/transactions/fees/vecs.rs b/crates/brk_computer/src/transactions/fees/vecs.rs index 5403f77a1..cd4b71ae0 100644 --- a/crates/brk_computer/src/transactions/fees/vecs.rs +++ b/crates/brk_computer/src/transactions/fees/vecs.rs @@ -1,14 +1,23 @@ use brk_traversable::Traversable; -use brk_types::{FeeRate, Sats, TxIndex}; +use brk_types::{FeeRate, Height, Sats, StoredBool, StoredU64, TxIndex}; use vecdb::{EagerVec, PcoVec, Rw, StorageMode}; use crate::internal::PerTxDistribution; +#[derive(Traversable)] +pub struct CountVecs { + pub cpfp_parent: M::Stored>>, + pub cpfp_child: M::Stored>>, +} + #[derive(Traversable)] pub struct Vecs { + pub count: CountVecs, pub input_value: M::Stored>>, pub output_value: M::Stored>>, pub fee: PerTxDistribution, pub fee_rate: M::Stored>>, pub effective_fee_rate: PerTxDistribution, + pub is_cpfp_parent: M::Stored>>, + pub is_cpfp_child: M::Stored>>, } diff --git a/crates/brk_computer/src/transactions/import.rs b/crates/brk_computer/src/transactions/import.rs index 2e6643a21..30a3ca389 100644 --- a/crates/brk_computer/src/transactions/import.rs +++ b/crates/brk_computer/src/transactions/import.rs @@ -12,7 +12,9 @@ use crate::{ }, }; -use super::{CountVecs, FeesVecs, SizeVecs, Vecs, VersionsVecs, VolumeVecs}; +use super::{ + CountVecs, FeesVecs, PatternsVecs, PolicyVecs, SizeVecs, Vecs, VersionsVecs, VolumeVecs, +}; impl Vecs { pub(crate) fn forced_import( @@ -28,6 +30,8 @@ impl Vecs { let count = CountVecs::forced_import(&db, version, indexes, cached_starts)?; let size = SizeVecs::forced_import(&db, version, indexer, indexes)?; let fees = FeesVecs::forced_import(&db, version, indexes)?; + let patterns = PatternsVecs::forced_import(&db, version)?; + let policy = PolicyVecs::forced_import(&db, version)?; let versions = VersionsVecs::forced_import(&db, version, indexes, cached_starts)?; let volume = VolumeVecs::forced_import(&db, version, indexes, cached_starts)?; @@ -36,6 +40,8 @@ impl Vecs { count, size, fees, + patterns, + policy, versions, volume, }; diff --git a/crates/brk_computer/src/transactions/mod.rs b/crates/brk_computer/src/transactions/mod.rs index c926ecc5f..57ecf3f93 100644 --- a/crates/brk_computer/src/transactions/mod.rs +++ b/crates/brk_computer/src/transactions/mod.rs @@ -1,5 +1,7 @@ pub mod count; pub mod fees; +pub mod patterns; +pub mod policy; pub mod size; pub mod versions; pub mod volume; @@ -12,6 +14,8 @@ use vecdb::{Database, Rw, StorageMode}; pub use count::Vecs as CountVecs; pub use fees::Vecs as FeesVecs; +pub use patterns::Vecs as PatternsVecs; +pub use policy::Vecs as PolicyVecs; pub use size::Vecs as SizeVecs; pub use versions::Vecs as VersionsVecs; pub use volume::Vecs as VolumeVecs; @@ -26,6 +30,8 @@ pub struct Vecs { pub count: CountVecs, pub size: SizeVecs, pub fees: FeesVecs, + pub patterns: PatternsVecs, + pub policy: PolicyVecs, pub versions: VersionsVecs, pub volume: VolumeVecs, } diff --git a/crates/brk_computer/src/transactions/patterns/coinjoin.rs b/crates/brk_computer/src/transactions/patterns/coinjoin.rs new file mode 100644 index 000000000..0d96b0c8d --- /dev/null +++ b/crates/brk_computer/src/transactions/patterns/coinjoin.rs @@ -0,0 +1,127 @@ +use brk_types::{OutputType, Sats, TypeIndex}; +use rustc_hash::FxHashSet; + +type Address = (OutputType, TypeIndex); + +#[derive(Default)] +pub(super) struct Candidate { + values: FxHashSet, + input_addresses: FxHashSet
, + output_addresses: FxHashSet
, + zero_values: usize, + address_reuse: bool, +} + +impl Candidate { + pub(super) fn clear(&mut self) { + self.values.clear(); + self.input_addresses.clear(); + self.output_addresses.clear(); + self.zero_values = 0; + self.address_reuse = false; + } + + pub(super) fn add_input( + &mut self, + value: Sats, + output_type: OutputType, + type_index: TypeIndex, + ) { + self.add_value(value); + if has_script_address(output_type) { + self.address_reuse |= !self.input_addresses.insert((output_type, type_index)); + } + } + + pub(super) fn add_output( + &mut self, + value: Sats, + output_type: OutputType, + type_index: TypeIndex, + ) { + self.add_value(value); + if has_script_address(output_type) { + let address = (output_type, type_index); + self.address_reuse |= + self.input_addresses.contains(&address) || !self.output_addresses.insert(address); + } + } + + pub(super) fn is_match(&self, input_count: usize, output_count: usize) -> bool { + !self.address_reuse + && self.values.len() + self.zero_values <= (input_count + output_count) / 2 + } + + fn add_value(&mut self, value: Sats) { + if value.is_zero() { + self.zero_values += 1; + } else { + self.values.insert(value); + } + } +} + +fn has_script_address(output_type: OutputType) -> bool { + matches!( + output_type, + OutputType::P2PK65 + | OutputType::P2PK33 + | OutputType::P2PKH + | OutputType::P2SH + | OutputType::P2WPKH + | OutputType::P2WSH + | OutputType::P2TR + | OutputType::P2A + ) +} + +#[cfg(test)] +mod tests { + use brk_types::{OutputType, Sats, TypeIndex}; + + use super::Candidate; + + #[test] + fn repeated_values_match_without_address_reuse() { + let mut candidate = Candidate::default(); + for index in 0usize..5 { + candidate.add_input( + Sats::new(10_000), + OutputType::P2WPKH, + TypeIndex::from(index), + ); + candidate.add_output( + Sats::new(9_000), + OutputType::P2WPKH, + TypeIndex::from(index + 10), + ); + } + assert!(candidate.is_match(5, 5)); + } + + #[test] + fn reused_output_address_does_not_match() { + let mut candidate = Candidate::default(); + for _ in 0..5 { + candidate.add_output( + Sats::new(9_000), + OutputType::P2WPKH, + TypeIndex::from(1usize), + ); + } + assert!(!candidate.is_match(5, 5)); + } + + #[test] + fn reused_input_address_does_not_match() { + let mut candidate = Candidate::default(); + for _ in 0..5 { + candidate.add_input( + Sats::new(10_000), + OutputType::P2PK33, + TypeIndex::from(1usize), + ); + } + assert!(!candidate.is_match(5, 5)); + } +} diff --git a/crates/brk_computer/src/transactions/patterns/compute.rs b/crates/brk_computer/src/transactions/patterns/compute.rs new file mode 100644 index 000000000..d98628198 --- /dev/null +++ b/crates/brk_computer/src/transactions/patterns/compute.rs @@ -0,0 +1,251 @@ +use brk_error::Result; +use brk_indexer::Indexer; +use brk_types::{StoredBool, StoredU64, TxIndex}; +use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec}; + +use super::{Vecs, coinjoin::Candidate}; +use crate::{indexes, inputs}; + +const WRITE_INTERVAL: usize = 10_000; + +impl Vecs { + pub(crate) fn compute( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + spent: &inputs::SpentVecs, + exit: &Exit, + ) -> Result<()> { + let features = &indexer.vecs.transaction_features; + let version = indexes.tx_index.input_count.version() + + indexes.tx_index.output_count.version() + + indexer.vecs.transactions.first_tx_index.version() + + indexer.vecs.transactions.first_txin_index.version() + + indexer.vecs.transactions.first_txout_index.version() + + spent.value.version() + + indexer.vecs.inputs.output_type.version() + + indexer.vecs.inputs.type_index.version() + + indexer.vecs.outputs.value.version() + + indexer.vecs.outputs.output_type.version() + + indexer.vecs.outputs.type_index.version() + + features.has_op_return.version() + + features.has_inscription.version() + + indexes.height.tx_index_count.version(); + + for vec in [ + &mut self.is_coinjoin, + &mut self.is_consolidation, + &mut self.is_batch_payout, + ] { + vec.validate_computed_version_or_reset(version)?; + } + for vec in [ + &mut self.count.coinjoin, + &mut self.count.consolidation, + &mut self.count.batch_payout, + ] { + vec.validate_computed_version_or_reset(version)?; + } + + let lengths = indexer.safe_lengths(); + let target_tx = lengths.tx_index.to_usize(); + let target_height = lengths.height.to_usize(); + let tx_len = self + .is_coinjoin + .len() + .min(self.is_consolidation.len()) + .min(self.is_batch_payout.len()) + .min(target_tx); + let count_len = self + .count + .coinjoin + .len() + .min(self.count.consolidation.len()) + .min(self.count.batch_payout.len()) + .min(target_height); + let start_height = count_len.min(next_height(indexes, tx_len, target_tx, target_height)); + if start_height >= target_height { + return Ok(()); + } + + let first_tx = &indexer.vecs.transactions.first_tx_index; + let start_tx = first_tx.collect_one_at(start_height).unwrap().to_usize(); + self.is_coinjoin.truncate_if_needed_at(start_tx)?; + self.is_consolidation.truncate_if_needed_at(start_tx)?; + self.is_batch_payout.truncate_if_needed_at(start_tx)?; + self.count.coinjoin.truncate_if_needed_at(start_height)?; + self.count + .consolidation + .truncate_if_needed_at(start_height)?; + self.count + .batch_payout + .truncate_if_needed_at(start_height)?; + + let first_txin = indexer + .vecs + .transactions + .first_txin_index + .collect_one_at(start_tx) + .unwrap() + .to_usize(); + let first_txout = indexer + .vecs + .transactions + .first_txout_index + .collect_one_at(start_tx) + .unwrap() + .to_usize(); + + let mut input_count = indexes.tx_index.input_count.cursor(); + let mut output_count = indexes.tx_index.output_count.cursor(); + let mut input_value = spent.value.cursor(); + let mut input_type = indexer.vecs.inputs.output_type.cursor(); + let mut input_type_index = indexer.vecs.inputs.type_index.cursor(); + let mut output_value = indexer.vecs.outputs.value.cursor(); + let mut output_type = indexer.vecs.outputs.output_type.cursor(); + let mut output_type_index = indexer.vecs.outputs.type_index.cursor(); + let mut has_op_return = features.has_op_return.cursor(); + let mut has_inscription = features.has_inscription.cursor(); + let mut tx_count = indexes.height.tx_index_count.cursor(); + + input_count.advance(start_tx); + output_count.advance(start_tx); + input_value.advance(first_txin); + input_type.advance(first_txin); + input_type_index.advance(first_txin); + output_value.advance(first_txout); + output_type.advance(first_txout); + output_type_index.advance(first_txout); + has_op_return.advance(start_tx); + has_inscription.advance(start_tx); + tx_count.advance(start_height); + + let mut candidate = Candidate::default(); + let mut block_start = start_tx; + for height in start_height..target_height { + let block_end = block_start + u64::from(tx_count.next().unwrap()) as usize; + let mut coinjoin_count = 0; + let mut consolidation_count = 0; + let mut batch_payout_count = 0; + + for tx_index in block_start..block_end { + let inputs = usize::from(input_count.next().unwrap()); + let outputs = usize::from(output_count.next().unwrap()); + let op_return = has_op_return.next().unwrap().is_true(); + let inscription = has_inscription.next().unwrap().is_true(); + let token_related = op_return || inscription; + + let consolidation = is_consolidation(inputs, outputs); + let batch_payout = is_batch_payout(inputs, outputs, tx_index == block_start); + let coinjoin_candidate = is_coinjoin_candidate(inputs, outputs, token_related); + + let coinjoin = if coinjoin_candidate { + candidate.clear(); + for _ in 0..inputs { + candidate.add_input( + input_value.next().unwrap(), + input_type.next().unwrap(), + input_type_index.next().unwrap(), + ); + } + for _ in 0..outputs { + candidate.add_output( + output_value.next().unwrap(), + output_type.next().unwrap(), + output_type_index.next().unwrap(), + ); + } + candidate.is_match(inputs, outputs) + } else { + input_value.advance(inputs); + input_type.advance(inputs); + input_type_index.advance(inputs); + output_value.advance(outputs); + output_type.advance(outputs); + output_type_index.advance(outputs); + false + }; + + coinjoin_count += coinjoin as u64; + consolidation_count += consolidation as u64; + batch_payout_count += batch_payout as u64; + self.is_coinjoin.push(StoredBool::from(coinjoin)); + self.is_consolidation.push(StoredBool::from(consolidation)); + self.is_batch_payout.push(StoredBool::from(batch_payout)); + } + + self.count.coinjoin.push(StoredU64::from(coinjoin_count)); + self.count + .consolidation + .push(StoredU64::from(consolidation_count)); + self.count + .batch_payout + .push(StoredU64::from(batch_payout_count)); + + if (height + 1).is_multiple_of(WRITE_INTERVAL) { + let _lock = exit.lock(); + self.write()?; + } + + block_start = block_end; + } + + let _lock = exit.lock(); + self.write() + } + + fn write(&mut self) -> Result<()> { + self.is_coinjoin.write()?; + self.is_consolidation.write()?; + self.is_batch_payout.write()?; + self.count.coinjoin.write()?; + self.count.consolidation.write()?; + self.count.batch_payout.write()?; + Ok(()) + } +} + +fn next_height( + indexes: &indexes::Vecs, + tx_len: usize, + target_tx: usize, + target_height: usize, +) -> usize { + if tx_len >= target_tx { + target_height + } else { + indexes + .tx_heights + .get_shared(TxIndex::from(tx_len)) + .unwrap() + .to_usize() + } +} + +fn is_consolidation(inputs: usize, outputs: usize) -> bool { + inputs >= outputs * 5 +} + +fn is_batch_payout(inputs: usize, outputs: usize, is_coinbase: bool) -> bool { + !is_coinbase && outputs >= inputs * 5 +} + +fn is_coinjoin_candidate(inputs: usize, outputs: usize, token_related: bool) -> bool { + inputs >= 5 && outputs >= 5 && inputs < outputs * 5 && outputs < inputs * 5 && !token_related +} + +#[cfg(test)] +mod tests { + use super::{is_batch_payout, is_coinjoin_candidate, is_consolidation}; + + #[test] + fn ratio_boundaries_match_filter_semantics() { + assert!(is_consolidation(25, 5)); + assert!(is_batch_payout(5, 25, false)); + assert!(!is_batch_payout(1, 5, true)); + assert!(!is_coinjoin_candidate(25, 5, false)); + assert!(!is_coinjoin_candidate(5, 25, false)); + assert!(is_coinjoin_candidate(5, 5, false)); + assert!(!is_coinjoin_candidate(5, 5, true)); + } +} diff --git a/crates/brk_computer/src/transactions/patterns/import.rs b/crates/brk_computer/src/transactions/patterns/import.rs new file mode 100644 index 000000000..859b4ff8d --- /dev/null +++ b/crates/brk_computer/src/transactions/patterns/import.rs @@ -0,0 +1,20 @@ +use brk_error::Result; +use brk_types::Version; +use vecdb::{Database, EagerVec, ImportableVec}; + +use super::{CountVecs, Vecs}; + +impl Vecs { + pub(crate) fn forced_import(db: &Database, version: Version) -> Result { + Ok(Self { + count: CountVecs { + coinjoin: EagerVec::forced_import(db, "coinjoin_count", version)?, + consolidation: EagerVec::forced_import(db, "consolidation_count", version)?, + batch_payout: EagerVec::forced_import(db, "batch_payout_count", version)?, + }, + is_coinjoin: EagerVec::forced_import(db, "is_coinjoin", version)?, + is_consolidation: EagerVec::forced_import(db, "is_consolidation", version)?, + is_batch_payout: EagerVec::forced_import(db, "is_batch_payout", version)?, + }) + } +} diff --git a/crates/brk_computer/src/transactions/patterns/mod.rs b/crates/brk_computer/src/transactions/patterns/mod.rs new file mode 100644 index 000000000..b34029e5a --- /dev/null +++ b/crates/brk_computer/src/transactions/patterns/mod.rs @@ -0,0 +1,6 @@ +mod coinjoin; +mod compute; +mod import; +mod vecs; + +pub use vecs::{CountVecs, Vecs}; diff --git a/crates/brk_computer/src/transactions/patterns/vecs.rs b/crates/brk_computer/src/transactions/patterns/vecs.rs new file mode 100644 index 000000000..a484f4c8b --- /dev/null +++ b/crates/brk_computer/src/transactions/patterns/vecs.rs @@ -0,0 +1,18 @@ +use brk_traversable::Traversable; +use brk_types::{Height, StoredBool, StoredU64, TxIndex}; +use vecdb::{EagerVec, PcoVec, Rw, StorageMode}; + +#[derive(Traversable)] +pub struct CountVecs { + pub coinjoin: M::Stored>>, + pub consolidation: M::Stored>>, + pub batch_payout: M::Stored>>, +} + +#[derive(Traversable)] +pub struct Vecs { + pub count: CountVecs, + pub is_coinjoin: M::Stored>>, + pub is_consolidation: M::Stored>>, + pub is_batch_payout: M::Stored>>, +} diff --git a/crates/brk_computer/src/transactions/policy/compute.rs b/crates/brk_computer/src/transactions/policy/compute.rs new file mode 100644 index 000000000..146d04b14 --- /dev/null +++ b/crates/brk_computer/src/transactions/policy/compute.rs @@ -0,0 +1,127 @@ +use brk_error::Result; +use brk_indexer::Indexer; +use brk_types::{Sats, StoredBool, StoredU64, TxIndex}; +use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec}; + +use super::Vecs; +use crate::{indexes, transactions::fees}; + +const FIRST_EPHEMERAL_DUST_HEIGHT: usize = 905_000; +const WRITE_INTERVAL: usize = 10_000; + +impl Vecs { + pub(crate) fn compute( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + fees: &fees::Vecs, + exit: &Exit, + ) -> Result<()> { + let features = &indexer.vecs.transaction_features; + let version = features.is_unconditionally_nonstandard.version() + + features.has_dust_output.version() + + fees.fee.tx_index.version() + + indexer.vecs.transactions.first_tx_index.version() + + indexes.height.tx_index_count.version(); + self.is_nonstandard + .validate_computed_version_or_reset(version)?; + self.count.validate_computed_version_or_reset(version)?; + + let lengths = indexer.safe_lengths(); + let target_tx = lengths.tx_index.to_usize().min(fees.fee.tx_index.len()); + let target_height = lengths.height.to_usize(); + let tx_len = self.is_nonstandard.len().min(target_tx); + let count_len = self.count.len().min(target_height); + let next_height = if tx_len >= target_tx { + target_height + } else { + indexes + .tx_heights + .get_shared(TxIndex::from(tx_len)) + .unwrap() + .to_usize() + }; + let start_height = count_len.min(next_height); + if start_height >= target_height { + return Ok(()); + } + + let first_tx = &indexer.vecs.transactions.first_tx_index; + let start_tx = first_tx.collect_one_at(start_height).unwrap().to_usize(); + self.is_nonstandard.truncate_if_needed_at(start_tx)?; + self.count.truncate_if_needed_at(start_height)?; + + let mut unconditional = features.is_unconditionally_nonstandard.cursor(); + let mut has_dust = features.has_dust_output.cursor(); + let mut fee = fees.fee.tx_index.cursor(); + let mut tx_count = indexes.height.tx_index_count.cursor(); + unconditional.advance(start_tx); + has_dust.advance(start_tx); + fee.advance(start_tx); + tx_count.advance(start_height); + + let mut block_start = start_tx; + for height in start_height..target_height { + let block_end = + (block_start + u64::from(tx_count.next().unwrap()) as usize).min(target_tx); + let mut count = 0; + + for _ in block_start..block_end { + let raw = unconditional.next().unwrap().is_true(); + let dust = has_dust.next().unwrap().is_true(); + let nonstandard = if raw { + fee.advance(1); + true + } else if dust { + dust_is_nonstandard(height, fee.next().unwrap()) + } else { + fee.advance(1); + false + }; + count += nonstandard as u64; + self.is_nonstandard.push(StoredBool::from(nonstandard)); + } + self.count.push(StoredU64::from(count)); + + if (height + 1).is_multiple_of(WRITE_INTERVAL) { + let _lock = exit.lock(); + self.is_nonstandard.write()?; + self.count.write()?; + } + + block_start = block_end; + } + + let _lock = exit.lock(); + self.is_nonstandard.write()?; + self.count.write()?; + Ok(()) + } +} + +fn dust_is_nonstandard(height: usize, fee: Sats) -> bool { + height < FIRST_EPHEMERAL_DUST_HEIGHT || fee != Sats::ZERO +} + +#[cfg(test)] +mod tests { + use brk_types::Sats; + + use super::{FIRST_EPHEMERAL_DUST_HEIGHT, dust_is_nonstandard}; + + #[test] + fn zero_fee_ephemeral_dust_starts_at_activation() { + assert!(dust_is_nonstandard( + FIRST_EPHEMERAL_DUST_HEIGHT - 1, + Sats::ZERO + )); + assert!(!dust_is_nonstandard( + FIRST_EPHEMERAL_DUST_HEIGHT, + Sats::ZERO + )); + assert!(dust_is_nonstandard( + FIRST_EPHEMERAL_DUST_HEIGHT, + Sats::new(1) + )); + } +} diff --git a/crates/brk_computer/src/transactions/policy/import.rs b/crates/brk_computer/src/transactions/policy/import.rs new file mode 100644 index 000000000..55c8df60e --- /dev/null +++ b/crates/brk_computer/src/transactions/policy/import.rs @@ -0,0 +1,14 @@ +use brk_error::Result; +use brk_types::Version; +use vecdb::{Database, EagerVec, ImportableVec}; + +use super::Vecs; + +impl Vecs { + pub(crate) fn forced_import(db: &Database, version: Version) -> Result { + Ok(Self { + count: EagerVec::forced_import(db, "nonstandard_count", version)?, + is_nonstandard: EagerVec::forced_import(db, "is_nonstandard", version)?, + }) + } +} diff --git a/crates/brk_computer/src/transactions/policy/mod.rs b/crates/brk_computer/src/transactions/policy/mod.rs new file mode 100644 index 000000000..1136f9ebd --- /dev/null +++ b/crates/brk_computer/src/transactions/policy/mod.rs @@ -0,0 +1,5 @@ +mod compute; +mod import; +mod vecs; + +pub use vecs::Vecs; diff --git a/crates/brk_computer/src/transactions/policy/vecs.rs b/crates/brk_computer/src/transactions/policy/vecs.rs new file mode 100644 index 000000000..7529ccf8e --- /dev/null +++ b/crates/brk_computer/src/transactions/policy/vecs.rs @@ -0,0 +1,9 @@ +use brk_traversable::Traversable; +use brk_types::{Height, StoredBool, StoredU64, TxIndex}; +use vecdb::{EagerVec, PcoVec, Rw, StorageMode}; + +#[derive(Traversable)] +pub struct Vecs { + pub count: M::Stored>>, + pub is_nonstandard: M::Stored>>, +} diff --git a/crates/brk_computer/src/transactions/size/compute.rs b/crates/brk_computer/src/transactions/size/compute.rs index a4bc55b45..1af8fce8a 100644 --- a/crates/brk_computer/src/transactions/size/compute.rs +++ b/crates/brk_computer/src/transactions/size/compute.rs @@ -14,7 +14,7 @@ impl Vecs { ) -> Result<()> { let starting_lengths = indexer.safe_lengths(); - self.vsize + self.weight .derive_from(indexer, indexes, &starting_lengths, exit)?; Ok(()) diff --git a/crates/brk_computer/src/transactions/size/import.rs b/crates/brk_computer/src/transactions/size/import.rs index c0fcb2558..a63a9d1f4 100644 --- a/crates/brk_computer/src/transactions/size/import.rs +++ b/crates/brk_computer/src/transactions/size/import.rs @@ -1,12 +1,12 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{TxIndex, VSize, Version, Weight}; -use vecdb::{Database, LazyVecFrom2, ReadableCloneableVec}; +use brk_types::{Version, Weight}; +use vecdb::{Database, LazyVecFrom1, ReadableCloneableVec}; use super::Vecs; use crate::{ indexes, - internal::{LazyPerTxDistribution, LazyPerTxDistributionTransformed, VSizeToWeight}, + internal::{Identity, LazyPerTxDistribution, LazyPerTxDistributionTransformed, WeightToVSize}, }; impl Vecs { @@ -16,37 +16,31 @@ impl Vecs { indexer: &Indexer, indexes: &indexes::Vecs, ) -> Result { - let tx_index_to_vsize = LazyVecFrom2::init( - "tx_vsize", + let tx_index_to_weight = LazyVecFrom1::transformed::>( + "tx_weight", version, - indexer.vecs.transactions.base_size.read_only_boxed_clone(), - indexer.vecs.transactions.total_size.read_only_boxed_clone(), - |_index: TxIndex, base_size, total_size| { - VSize::from(Weight::from_sizes(*base_size, *total_size)) - }, + indexer.vecs.transactions.weight.read_only_boxed_clone(), ); - let vsize = LazyPerTxDistribution::forced_import( + let weight = LazyPerTxDistribution::forced_import( db, - "tx_vsize", + "tx_weight", version, indexes, - tx_index_to_vsize, + tx_index_to_weight, )?; - let tx_index_to_weight = LazyVecFrom2::init( - "tx_weight", + let tx_index_to_vsize = LazyVecFrom1::transformed::( + "tx_vsize", version, - indexer.vecs.transactions.base_size.read_only_boxed_clone(), - indexer.vecs.transactions.total_size.read_only_boxed_clone(), - |_index: TxIndex, base_size, total_size| Weight::from_sizes(*base_size, *total_size), + indexer.vecs.transactions.weight.read_only_boxed_clone(), ); - let weight = LazyPerTxDistributionTransformed::new::( - "tx_weight", + let vsize = LazyPerTxDistributionTransformed::new::( + "tx_vsize", version, - tx_index_to_weight, - &vsize.distribution, + tx_index_to_vsize, + &weight.distribution, ); Ok(Self { vsize, weight }) diff --git a/crates/brk_computer/src/transactions/size/vecs.rs b/crates/brk_computer/src/transactions/size/vecs.rs index 292e3202d..3003a6751 100644 --- a/crates/brk_computer/src/transactions/size/vecs.rs +++ b/crates/brk_computer/src/transactions/size/vecs.rs @@ -1,11 +1,11 @@ use brk_traversable::Traversable; -use brk_types::{StoredU32, VSize, Weight}; +use brk_types::{VSize, Weight}; use vecdb::{Rw, StorageMode}; use crate::internal::{LazyPerTxDistribution, LazyPerTxDistributionTransformed}; #[derive(Traversable)] pub struct Vecs { - pub vsize: LazyPerTxDistribution, - pub weight: LazyPerTxDistributionTransformed, + pub vsize: LazyPerTxDistributionTransformed, + pub weight: LazyPerTxDistribution, } diff --git a/crates/brk_computer/src/transactions/versions/compute.rs b/crates/brk_computer/src/transactions/versions/compute.rs index 885e879c5..5d47a258c 100644 --- a/crates/brk_computer/src/transactions/versions/compute.rs +++ b/crates/brk_computer/src/transactions/versions/compute.rs @@ -1,90 +1,31 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{StoredU64, TxVersion}; -use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec}; +use vecdb::Exit; use super::Vecs; impl Vecs { pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> { - let starting_height = indexer.safe_lengths().height; + let lengths = indexer.safe_lengths(); + let starting_height = lengths.height; + let counts = &indexer.vecs.transaction_features.count; - let dep_version = indexer.vecs.transactions.tx_version.version() - + indexer.vecs.transactions.first_tx_index.version() - + indexer.vecs.transactions.txid.version(); - - for vec in [&mut self.v1.block, &mut self.v2.block, &mut self.v3.block] { - vec.validate_and_truncate(dep_version, starting_height)?; + for (metrics, source) in [ + (&mut self.v1, &counts.v1), + (&mut self.v2, &counts.v2), + (&mut self.v3, &counts.v3), + (&mut self.other, &counts.other_version), + ] { + metrics.compute(starting_height, exit, |block| { + Ok(block.compute_transform( + starting_height, + source, + |(height, count, ..)| (height, count), + exit, + )?) + })?; } - let skip = self - .v1 - .block - .len() - .min(self.v2.block.len()) - .min(self.v3.block.len()); - - let first_tx_index = &indexer.vecs.transactions.first_tx_index; - let end = first_tx_index.len(); - if skip >= end { - return Ok(()); - } - - // Truncate all 3 to skip, then push (no per-element bounds checks). - self.v1.block.truncate_if_needed_at(skip)?; - self.v2.block.truncate_if_needed_at(skip)?; - self.v3.block.truncate_if_needed_at(skip)?; - - // Single cursor over tx_version — scanned once for all 3 version counts. - let mut cursor = indexer.vecs.transactions.tx_version.cursor(); - let fi_batch = first_tx_index.collect_range_at(skip, end); - let txid_len = indexer.vecs.transactions.txid.len(); - - for (j, first_index) in fi_batch.iter().enumerate() { - let next_first = fi_batch - .get(j + 1) - .map(|fi| fi.to_usize()) - .unwrap_or(txid_len); - - let mut c1: usize = 0; - let mut c2: usize = 0; - let mut c3: usize = 0; - - let fi = first_index.to_usize(); - cursor.advance(fi - cursor.position()); - for _ in fi..next_first { - match cursor.next().unwrap() { - TxVersion::ONE => c1 += 1, - TxVersion::TWO => c2 += 1, - TxVersion::THREE => c3 += 1, - _ => {} - } - } - - self.v1.block.push(StoredU64::from(c1 as u64)); - self.v2.block.push(StoredU64::from(c2 as u64)); - self.v3.block.push(StoredU64::from(c3 as u64)); - - if self.v1.block.batch_limit_reached() { - let _lock = exit.lock(); - self.v1.block.write()?; - self.v2.block.write()?; - self.v3.block.write()?; - } - } - - { - let _lock = exit.lock(); - self.v1.block.write()?; - self.v2.block.write()?; - self.v3.block.write()?; - } - - // Derive cumulative + sums from base - self.v1.compute_rest(starting_height, exit)?; - self.v2.compute_rest(starting_height, exit)?; - self.v3.compute_rest(starting_height, exit)?; - Ok(()) } } diff --git a/crates/brk_computer/src/transactions/versions/import.rs b/crates/brk_computer/src/transactions/versions/import.rs index ec66cb51a..52351411a 100644 --- a/crates/brk_computer/src/transactions/versions/import.rs +++ b/crates/brk_computer/src/transactions/versions/import.rs @@ -37,6 +37,13 @@ impl Vecs { indexes, cached_starts, )?, + other: PerBlockCumulativeRolling::forced_import( + db, + "tx_other_version", + version, + indexes, + cached_starts, + )?, }) } } diff --git a/crates/brk_computer/src/transactions/versions/vecs.rs b/crates/brk_computer/src/transactions/versions/vecs.rs index 8aaed73e0..c80b00f01 100644 --- a/crates/brk_computer/src/transactions/versions/vecs.rs +++ b/crates/brk_computer/src/transactions/versions/vecs.rs @@ -9,4 +9,5 @@ pub struct Vecs { pub v1: PerBlockCumulativeRolling, pub v2: PerBlockCumulativeRolling, pub v3: PerBlockCumulativeRolling, + pub other: PerBlockCumulativeRolling, } diff --git a/crates/brk_indexer/src/constants.rs b/crates/brk_indexer/src/constants.rs index f2247c6f3..a77f64c20 100644 --- a/crates/brk_indexer/src/constants.rs +++ b/crates/brk_indexer/src/constants.rs @@ -4,7 +4,7 @@ use brk_types::{TxIndex, Txid, TxidPrefix, Version}; // One version for all data sources // Increment on **change _OR_ addition** -pub const VERSION: Version = Version::new(26); +pub const VERSION: Version = Version::new(27); pub const SNAPSHOT_BLOCK_RANGE: usize = 1_000; /// Known duplicate Bitcoin transactions (BIP30) diff --git a/crates/brk_indexer/src/lengths.rs b/crates/brk_indexer/src/lengths.rs index 0f621d426..4c113d23b 100644 --- a/crates/brk_indexer/src/lengths.rs +++ b/crates/brk_indexer/src/lengths.rs @@ -93,8 +93,7 @@ impl Lengths { .p2ms .first_index .checked_push(height, self.p2ms_output_index)?; - vecs.scripts - .op_return + vecs.op_return .first_index .checked_push(height, self.op_return_index)?; vecs.addrs @@ -177,8 +176,8 @@ impl Lengths { height, )?, op_return_index: next_index( - &vecs.scripts.op_return.first_index, - &vecs.scripts.op_return.to_tx_index, + &vecs.op_return.first_index, + &vecs.op_return.to_tx_index, height, )?, p2pk33_addr_index: next_index( diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 73a689fac..043cc6584 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -37,6 +37,7 @@ pub struct Indexer { path: PathBuf, pub vecs: Vecs, pub stores: Stores, + buffers: BlockBuffers, safe_lengths: SafeLengths, } @@ -53,7 +54,12 @@ impl Indexer { /// actually queryable. pub fn tip_blockhash(&self) -> BlockHash { match self.safe_lengths().height.decremented() { - Some(h) => self.vecs.blocks.blockhash.collect_one(h).unwrap_or_default(), + Some(h) => self + .vecs + .blocks + .blockhash + .collect_one(h) + .unwrap_or_default(), None => BlockHash::default(), } } @@ -102,6 +108,7 @@ impl Indexer { path: indexed_path.clone(), vecs, stores, + buffers: BlockBuffers::default(), safe_lengths, }) }; @@ -128,6 +135,7 @@ impl Indexer { /// record that gets replayed on every recovery), this cleanly recreates. fn full_reset(&mut self) -> Result<()> { info!("Full reset..."); + self.buffers.reset(); self.safe_lengths.reset(); self.vecs.reset()?; let stores_path = self.path.join("stores"); @@ -196,6 +204,8 @@ impl Indexer { debug!("Rollback vecs done."); drop(lock); + self.buffers.continue_from(prev_hash); + let mut lengths = starting_lengths; let is_export_height = @@ -227,10 +237,10 @@ impl Indexer { }; let mut readers = Readers::new(&self.vecs); - let mut buffers = BlockBuffers::default(); let vecs = &mut self.vecs; let stores = &mut self.stores; + let buffers = &mut self.buffers; for block in reader.after(prev_hash)?.iter() { let block = match block { @@ -277,38 +287,26 @@ impl Indexer { processor.push_block_size_and_weight(&txs)?; let (txins_result, txouts_result) = rayon::join( - || processor.process_inputs(&txs, &mut buffers.txid_prefix_map), - || processor.process_outputs(), + || processor.process_inputs(&txs, &mut buffers.inputs), + || processor.process_outputs(&mut buffers.addresses), ); let txins = txins_result?; let txouts = txouts_result?; - let tx_count = block.txdata.len(); let input_count = txins.len(); let output_count = txouts.len(); - BlockProcessor::collect_same_block_spent_outpoints( - &txins, - &mut buffers.same_block_spent, - ); - - processor.check_txid_collisions(&txs)?; - - let sigops = processor.compute_sigops(&txins, &txouts); - - processor.finalize_and_store_metadata( + processor.analyze_and_finalize_transactions( txs, txouts, txins, - sigops, - &buffers.same_block_spent, - &mut buffers.already_added_addrs, - &mut buffers.same_block_output_info, + &mut buffers.addresses, )?; processor .lengths .add_block(tx_count, input_count, output_count); + buffers.finish_block(*block.hash()); if is_export_height(height) { drop(readers); @@ -389,6 +387,7 @@ impl ReadOnlyClone for Indexer { path: self.path.clone(), vecs: self.vecs.read_only_clone(), stores: self.stores.clone(), + buffers: BlockBuffers::default(), safe_lengths: self.safe_lengths.clone(), } } diff --git a/crates/brk_indexer/src/processor/metadata.rs b/crates/brk_indexer/src/processor/block/mod.rs similarity index 78% rename from crates/brk_indexer/src/processor/metadata.rs rename to crates/brk_indexer/src/processor/block/mod.rs index 41146b41b..f99c7169c 100644 --- a/crates/brk_indexer/src/processor/metadata.rs +++ b/crates/brk_indexer/src/processor/block/mod.rs @@ -1,21 +1,22 @@ use brk_error::{Error, Result}; use brk_types::{BlockHashPrefix, Timestamp}; use tracing::error; -use vecdb::WritableVec; +use vecdb::{WritableVec, unlikely}; -use super::{BlockProcessor, ComputedTx}; +use super::{BlockProcessor, transaction::ComputedTx}; impl BlockProcessor<'_> { - pub fn process_block_metadata(&mut self) -> Result<()> { + pub(crate) fn process_block_metadata(&mut self) -> Result<()> { let height = self.height; let blockhash = self.block.hash(); let blockhash_prefix = BlockHashPrefix::from(blockhash); - if self - .stores - .blockhash_prefix_to_height - .get(&blockhash_prefix)? - .is_some_and(|prev_height| *prev_height != height) + if unlikely(self.check_collisions) + && self + .stores + .blockhash_prefix_to_height + .get(&blockhash_prefix)? + .is_some_and(|prev_height| *prev_height != height) { error!("BlockHash: {blockhash}"); return Err(Error::Internal("BlockHash prefix collision")); @@ -51,13 +52,13 @@ impl BlockProcessor<'_> { /// Push block total_size and weight, reusing per-tx sizes already computed in ComputedTx. /// This avoids redundant tx serialization (base_size + total_size were already computed). - pub fn push_block_size_and_weight(&mut self, txs: &[ComputedTx]) -> Result<()> { + pub(crate) fn push_block_size_and_weight(&mut self, txs: &[ComputedTx]) -> Result<()> { let overhead = bitcoin::block::Header::SIZE + bitcoin::VarInt::from(txs.len()).size(); let mut total_size = overhead; - let mut weight = overhead * 4; + let mut weight = bitcoin::Weight::from_non_witness_data_size(overhead as u64); let mut sw_txs = 0u32; let mut sw_size = 0usize; - let mut sw_weight = 0usize; + let mut sw_weight = bitcoin::Weight::ZERO; for (i, tx) in txs.iter().enumerate() { total_size += tx.total_size as usize; diff --git a/crates/brk_indexer/src/processor/buffer.rs b/crates/brk_indexer/src/processor/buffer.rs new file mode 100644 index 000000000..1bde47c1a --- /dev/null +++ b/crates/brk_indexer/src/processor/buffer.rs @@ -0,0 +1,29 @@ +use brk_types::BlockHash; + +use super::{txin::InputResolver, txout::BlockAddresses}; + +/// Reusable buffers cleared and refilled each block to avoid allocation churn. +#[derive(Default)] +pub(crate) struct BlockBuffers { + pub(crate) inputs: InputResolver, + pub(crate) addresses: BlockAddresses, + tip: Option, +} + +impl BlockBuffers { + pub(crate) fn continue_from(&mut self, parent: Option) { + if self.tip != parent { + self.addresses.clear_cache(); + } + self.tip = parent; + } + + pub(crate) fn finish_block(&mut self, blockhash: BlockHash) { + self.tip = Some(blockhash); + } + + pub(crate) fn reset(&mut self) { + self.addresses.clear_cache(); + self.tip = None; + } +} diff --git a/crates/brk_indexer/src/processor/mod.rs b/crates/brk_indexer/src/processor/mod.rs index bf9013f22..f3714c9a2 100644 --- a/crates/brk_indexer/src/processor/mod.rs +++ b/crates/brk_indexer/src/processor/mod.rs @@ -1,89 +1,22 @@ -mod metadata; -mod sigops; -mod tx; +mod block; +mod buffer; +mod transaction; mod txin; mod txout; -mod types; -pub use types::*; +pub(crate) use buffer::BlockBuffers; -use brk_cohort::ByAddrType; -use brk_error::Result; -use brk_types::{AddrHash, Block, Height, OutPoint, SigOps, TxInIndex, TypeIndex}; -use rustc_hash::{FxHashMap, FxHashSet}; +use brk_types::{Block, Height}; use crate::{Lengths, Readers, Stores, Vecs}; /// Processes a single block, extracting and storing all indexed data. -pub struct BlockProcessor<'a> { - pub block: &'a Block, - pub height: Height, - pub check_collisions: bool, - pub lengths: &'a mut Lengths, - pub vecs: &'a mut Vecs, - pub stores: &'a mut Stores, - pub readers: &'a Readers, -} - -impl BlockProcessor<'_> { - /// Finalizes outputs/inputs in parallel with storing tx metadata. - #[allow(clippy::too_many_arguments)] - pub fn finalize_and_store_metadata( - &mut self, - txs: Vec, - txouts: Vec, - txins: Vec<(TxInIndex, InputSource)>, - sigops: Vec, - same_block_spent_outpoints: &FxHashSet, - already_added: &mut ByAddrType>, - same_block_info: &mut FxHashMap, - ) -> Result<()> { - let lengths = &mut *self.lengths; - - // Split transactions vecs: finalize needs first_txout_index/first_txin_index, metadata needs the rest - let (first_txout_index, first_txin_index, mut tx_metadata) = - self.vecs.transactions.split_for_finalize(); - - let outputs = &mut self.vecs.outputs; - let inputs = &mut self.vecs.inputs; - let addrs = &mut self.vecs.addrs; - let scripts = &mut self.vecs.scripts; - - let addr_hash_stores = &mut self.stores.addr_type_to_addr_hash_to_addr_index; - let addr_tx_index_stores = &mut self.stores.addr_type_to_addr_index_and_tx_index; - let addr_outpoint_stores = &mut self.stores.addr_type_to_addr_index_and_unspent_outpoint; - let txid_prefix_store = &mut self.stores.txid_prefix_to_tx_index; - - let (finalize_result, metadata_result) = rayon::join( - || -> Result<()> { - txout::finalize_outputs( - lengths, - first_txout_index, - outputs, - addrs, - scripts, - addr_hash_stores, - addr_tx_index_stores, - addr_outpoint_stores, - txouts, - same_block_spent_outpoints, - already_added, - same_block_info, - )?; - txin::finalize_inputs( - first_txin_index, - inputs, - addr_tx_index_stores, - addr_outpoint_stores, - txins, - same_block_info, - ) - }, - || tx::store_tx_metadata(txs, sigops, txid_prefix_store, &mut tx_metadata), - ); - - finalize_result?; - metadata_result?; - Ok(()) - } +pub(crate) struct BlockProcessor<'a> { + pub(crate) block: &'a Block, + pub(crate) height: Height, + pub(crate) check_collisions: bool, + pub(crate) lengths: &'a mut Lengths, + pub(crate) vecs: &'a mut Vecs, + pub(crate) stores: &'a mut Stores, + pub(crate) readers: &'a Readers, } diff --git a/crates/brk_indexer/src/processor/sigops.rs b/crates/brk_indexer/src/processor/sigops.rs deleted file mode 100644 index 19e74c9be..000000000 --- a/crates/brk_indexer/src/processor/sigops.rs +++ /dev/null @@ -1,182 +0,0 @@ -use bitcoin::{Script, script::Instruction}; -use brk_types::{OutputType, SigOps, TxInIndex}; -use rayon::prelude::*; - -use super::{BlockProcessor, InputSource, ProcessedOutput}; - -impl BlockProcessor<'_> { - /// BIP-141 sigop cost per tx in the block. Mirrors - /// `bitcoin::Transaction::total_sigop_cost` but dispatches on each - /// input's prevout `OutputType` and each output's `OutputType` - /// (already resolved by `process_inputs`/`process_outputs`) instead - /// of round-tripping through bitcoin's closure API with - /// synthetic-prevout `ScriptBuf` allocations. The legacy-sigop walk - /// is short-circuited by `OutputType` for every script with a - /// canonical shape (~99% of outputs and ~95% of inputs on mainnet); - /// only `OpReturn`/`Unknown` outputs and non-segwit/non-P2SH inputs - /// fall back to a real script walk. - pub fn compute_sigops( - &self, - txins: &[(TxInIndex, InputSource)], - txouts: &[ProcessedOutput<'_>], - ) -> Vec { - let txdata = &self.block.txdata; - let base_tx_index = u32::from(self.lengths.tx_index); - - let mut tx_input_offsets = Vec::with_capacity(txdata.len()); - let mut tx_output_offsets = Vec::with_capacity(txdata.len()); - let mut input_offset = 0usize; - let mut output_offset = 0usize; - for tx in txdata { - tx_input_offsets.push(input_offset); - input_offset += tx.input.len(); - tx_output_offsets.push(output_offset); - output_offset += tx.output.len(); - } - - txdata - .par_iter() - .enumerate() - .map(|(i, tx)| { - if tx.is_coinbase() { - return SigOps::ZERO; - } - let in_start = tx_input_offsets[i]; - let tx_inputs = &txins[in_start..in_start + tx.input.len()]; - let out_start = tx_output_offsets[i]; - let tx_outputs = &txouts[out_start..out_start + tx.output.len()]; - - let mut legacy: usize = 0; - let mut redeem: usize = 0; - let mut witness: usize = 0; - - for (input, (_, source)) in tx.input.iter().zip(tx_inputs.iter()) { - let prev_kind = match source { - InputSource::PreviousBlock { output_type, .. } => *output_type, - InputSource::SameBlock { outpoint, .. } => { - let local = (u32::from(outpoint.tx_index()) - base_tx_index) as usize; - let vout = u32::from(outpoint.vout()) as usize; - txouts[tx_output_offsets[local] + vout].output_type - } - }; - - // Single match per input: legacy script_sig sigops AND the - // redeem/witness contribution. Consensus enforces a - // push-only or empty script_sig in the four cases below - // (BIP-16 for P2SH from block 173805 onwards; BIP-141 / - // BIP-341 for segwit/taproot from activation), so legacy - // sigops are guaranteed 0 there. Everything else falls - // through to a real `count_sigops_legacy` walk. - match prev_kind { - OutputType::P2SH => { - // Faithful to bitcoin's count_p2sh_sigops + the - // nested-segwit branch of count_witness_sigops in - // a single script walk: redeem sigops use - // last_pushdata (no push-only check), wrapped - // witness sigops require both push-only and - // last_pushdata. - let (last_push, is_push_only) = - last_push_and_push_only(&input.script_sig); - let Some(redeem_bytes) = last_push else { - continue; - }; - let rs = Script::from_bytes(redeem_bytes); - redeem = redeem.saturating_add(rs.count_sigops()); - if !is_push_only { - continue; - } - if rs.is_p2wpkh() { - witness = witness.saturating_add(1); - } else if rs.is_p2wsh() - && let Some(last) = input.witness.last() - { - witness = - witness.saturating_add(Script::from_bytes(last).count_sigops()); - } - } - OutputType::P2WPKH => { - witness = witness.saturating_add(1); - } - OutputType::P2WSH => { - if let Some(last) = input.witness.last() { - witness = - witness.saturating_add(Script::from_bytes(last).count_sigops()); - } - } - OutputType::P2TR => {} - _ => { - legacy = legacy.saturating_add(input.script_sig.count_sigops_legacy()); - } - } - } - - for processed in tx_outputs { - legacy = legacy.saturating_add(legacy_sigops_for_output( - processed.output_type, - &processed.txout.script_pubkey, - )); - } - - SigOps::from( - legacy - .saturating_mul(4) - .saturating_add(redeem.saturating_mul(4)) - .saturating_add(witness), - ) - }) - .collect() - } -} - -/// Legacy sigop count of a script_pubkey, dispatched on `OutputType`. -/// Every variant except `OpReturn` and `Unknown` has a canonical shape -/// recognised by `OutputType::from`'s exact byte-pattern matchers, so -/// the legacy sigop count is fixed: P2PKH and P2PK both end in a -/// single OP_CHECKSIG (1), P2MS contains one OP_CHECKMULTISIG counted -/// as 20 in legacy mode, and P2SH/P2WPKH/P2WSH/P2TR/P2A/Empty contain -/// no CHECKSIG-class opcodes outside their pushdata. `OpReturn` -/// payloads can include 0xac/0xae bytes outside a push, and `Unknown` -/// can be anything, so both fall back to a real script walk. -#[inline] -fn legacy_sigops_for_output(output_type: OutputType, script_pubkey: &Script) -> usize { - match output_type { - OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65 => 1, - OutputType::P2MS => 20, - OutputType::P2SH - | OutputType::P2WPKH - | OutputType::P2WSH - | OutputType::P2TR - | OutputType::P2A - | OutputType::Empty => 0, - OutputType::OpReturn | OutputType::Unknown => script_pubkey.count_sigops_legacy(), - } -} - -/// Single-pass equivalent of bitcoin's private `last_pushdata()` plus the -/// public `is_push_only()`: returns the bytes of the script's last -/// `Instruction::PushBytes` (only when it is the *last* instruction) -/// alongside whether every instruction was a push (per Core, -/// `OP_RESERVED` and `OP_PUSHNUM_1..16` count as pushes too). -fn last_push_and_push_only(script: &Script) -> (Option<&[u8]>, bool) { - let mut last: Option<&[u8]> = None; - let mut push_only = true; - for inst in script.instructions() { - match inst { - Ok(Instruction::PushBytes(b)) => { - last = Some(b.as_bytes()); - } - Ok(Instruction::Op(op)) => { - last = None; - if op.to_u8() > 0x60 { - push_only = false; - } - } - Err(_) => { - last = None; - push_only = false; - break; - } - } - } - (last, push_only) -} diff --git a/crates/brk_indexer/src/processor/transaction/analysis/features/mod.rs b/crates/brk_indexer/src/processor/transaction/analysis/features/mod.rs new file mode 100644 index 000000000..320a5a848 --- /dev/null +++ b/crates/brk_indexer/src/processor/transaction/analysis/features/mod.rs @@ -0,0 +1,19 @@ +mod sighash; + +use crate::TxFeatureFlags; + +pub(super) fn scan_ecdsa_signature(bytes: &[u8], flags: &mut TxFeatureFlags) { + sighash::scan_ecdsa_signature(bytes, flags); +} + +pub(super) fn scan_taproot_signature(bytes: &[u8], flags: &mut TxFeatureFlags) { + sighash::scan_taproot_signature(bytes, flags); +} + +pub(super) fn record_validated_ecdsa_sighash(bytes: &[u8], flags: &mut TxFeatureFlags) { + sighash::record_validated_ecdsa_sighash(bytes, flags); +} + +pub(super) fn record_validated_taproot_sighash(bytes: &[u8], flags: &mut TxFeatureFlags) { + sighash::record_validated_taproot_sighash(bytes, flags); +} diff --git a/crates/brk_indexer/src/processor/transaction/analysis/features/sighash.rs b/crates/brk_indexer/src/processor/transaction/analysis/features/sighash.rs new file mode 100644 index 000000000..4d84a606f --- /dev/null +++ b/crates/brk_indexer/src/processor/transaction/analysis/features/sighash.rs @@ -0,0 +1,131 @@ +use bitcoin::{ + secp256k1::{ecdsa::Signature as EcdsaSignature, schnorr::Signature as SchnorrSignature}, + sighash::{EcdsaSighashType, TapSighashType}, +}; + +use crate::TxFeatureFlags; + +const MIN_ECDSA_SIGNATURE_BYTES: usize = 9; +const MAX_ECDSA_SIGNATURE_BYTES: usize = 73; + +pub(super) fn scan_ecdsa_signature(bytes: &[u8], flags: &mut TxFeatureFlags) { + if !is_ecdsa_signature_candidate(bytes) { + return; + } + + let (sighash_byte, der) = bytes.split_last().unwrap(); + let Ok(sighash_type) = EcdsaSighashType::from_standard(u32::from(*sighash_byte)) else { + return; + }; + let candidate_flags = ecdsa_feature_flags(sighash_type); + if flags.contains_all(candidate_flags) { + return; + } + + if EcdsaSignature::from_der(der).is_err() { + return; + } + flags.insert(candidate_flags); +} + +pub(super) fn scan_taproot_signature(bytes: &[u8], flags: &mut TxFeatureFlags) { + let (sighash_type, signature) = match bytes.len() { + 64 => (TapSighashType::Default, bytes), + 65 => { + let Ok(sighash_type) = TapSighashType::from_consensus_u8(bytes[64]) else { + return; + }; + (sighash_type, &bytes[..64]) + } + _ => return, + }; + let candidate_flags = taproot_feature_flags(sighash_type); + if flags.contains_all(candidate_flags) { + return; + } + + if SchnorrSignature::from_slice(signature).is_err() { + return; + } + flags.insert(candidate_flags); +} + +pub(super) fn record_validated_ecdsa_sighash(bytes: &[u8], flags: &mut TxFeatureFlags) { + let Ok(sighash_type) = EcdsaSighashType::from_standard(u32::from(*bytes.last().unwrap())) + else { + return; + }; + flags.insert(ecdsa_feature_flags(sighash_type)); +} + +pub(super) fn record_validated_taproot_sighash(bytes: &[u8], flags: &mut TxFeatureFlags) { + let sighash_type = match bytes.len() { + 64 => TapSighashType::Default, + 65 => { + let Ok(sighash_type) = TapSighashType::from_consensus_u8(bytes[64]) else { + return; + }; + sighash_type + } + _ => return, + }; + flags.insert(taproot_feature_flags(sighash_type)); +} + +fn ecdsa_feature_flags(sighash_type: EcdsaSighashType) -> u32 { + let mut flags = match sighash_type { + EcdsaSighashType::All | EcdsaSighashType::AllPlusAnyoneCanPay => { + TxFeatureFlags::SIGHASH_ALL + } + EcdsaSighashType::None | EcdsaSighashType::NonePlusAnyoneCanPay => { + TxFeatureFlags::SIGHASH_NONE + } + EcdsaSighashType::Single | EcdsaSighashType::SinglePlusAnyoneCanPay => { + TxFeatureFlags::SIGHASH_SINGLE + } + }; + if sighash_type.to_u32() & 0x80 != 0 { + flags |= TxFeatureFlags::SIGHASH_ANYONE_CAN_PAY; + } + flags +} + +#[inline] +fn is_ecdsa_signature_candidate(bytes: &[u8]) -> bool { + (MIN_ECDSA_SIGNATURE_BYTES..=MAX_ECDSA_SIGNATURE_BYTES).contains(&bytes.len()) + && bytes.first() == Some(&0x30) +} + +fn taproot_feature_flags(sighash_type: TapSighashType) -> u32 { + let mut flags = match sighash_type { + TapSighashType::Default => TxFeatureFlags::SIGHASH_DEFAULT, + TapSighashType::All | TapSighashType::AllPlusAnyoneCanPay => TxFeatureFlags::SIGHASH_ALL, + TapSighashType::None | TapSighashType::NonePlusAnyoneCanPay => TxFeatureFlags::SIGHASH_NONE, + TapSighashType::Single | TapSighashType::SinglePlusAnyoneCanPay => { + TxFeatureFlags::SIGHASH_SINGLE + } + }; + if matches!( + sighash_type, + TapSighashType::AllPlusAnyoneCanPay + | TapSighashType::NonePlusAnyoneCanPay + | TapSighashType::SinglePlusAnyoneCanPay + ) { + flags |= TxFeatureFlags::SIGHASH_ANYONE_CAN_PAY; + } + flags +} + +#[cfg(test)] +mod tests { + use super::is_ecdsa_signature_candidate; + + #[test] + fn rejects_impossible_ecdsa_signature_encodings() { + assert!(!is_ecdsa_signature_candidate(&[0x30; 8])); + assert!(is_ecdsa_signature_candidate(&[0x30; 9])); + assert!(is_ecdsa_signature_candidate(&[0x30; 73])); + assert!(!is_ecdsa_signature_candidate(&[0x30; 74])); + assert!(!is_ecdsa_signature_candidate(&[0x02; 33])); + } +} diff --git a/crates/brk_indexer/src/processor/transaction/analysis/input/mod.rs b/crates/brk_indexer/src/processor/transaction/analysis/input/mod.rs new file mode 100644 index 000000000..50d029d74 --- /dev/null +++ b/crates/brk_indexer/src/processor/transaction/analysis/input/mod.rs @@ -0,0 +1,111 @@ +mod redeem; +mod script_sig; +mod witness; + +use bitcoin::{TxIn, taproot::LeafVersion}; +use brk_types::OutputType; + +use crate::TxFeatureFlags; + +pub(super) struct Facts<'a> { + pub(super) script_sig: ScriptSigFacts<'a>, + redeem: redeem::Facts, + pub(super) witness: WitnessFacts<'a>, +} + +impl Facts<'_> { + #[inline] + pub(super) fn redeem_sigops(&self) -> Option { + self.redeem.sigops() + } + + #[inline] + pub(super) fn redeem_is_p2wpkh(&self) -> bool { + self.redeem.is_p2wpkh() + } + + #[inline] + pub(super) fn redeem_is_p2wsh(&self) -> bool { + self.redeem.is_p2wsh() + } + + #[inline] + pub(super) fn redeem_is_witness_program(&self) -> bool { + self.redeem.is_witness_program() + } +} + +pub(super) struct ScriptSigFacts<'a> { + pub(super) accurate_sigops: usize, + pub(super) last_push: Option<&'a [u8]>, + pub(super) legacy_sigops: usize, + pub(super) push_only: bool, +} + +impl ScriptSigFacts<'_> { + const EMPTY: Self = Self { + accurate_sigops: 0, + last_push: None, + legacy_sigops: 0, + push_only: true, + }; +} + +pub(super) struct WitnessFacts<'a> { + pub(super) has_annex: bool, + pub(super) last: Option<&'a [u8]>, + pub(super) leaf_version: Option, + pub(super) max_argument_bytes: usize, + pub(super) stack_items: usize, +} + +pub(super) fn analyze<'a>( + input: &'a TxIn, + output_type: OutputType, + flags: &mut TxFeatureFlags, +) -> Facts<'a> { + flags.insert_type(output_type); + + let script_sig = if input.script_sig.is_empty() { + ScriptSigFacts::EMPTY + } else { + script_sig::analyze( + &input.script_sig, + output_type, + input.witness.is_empty(), + flags, + ) + }; + let redeem = redeem::Facts::analyze(script_sig.last_push, output_type); + let witness = witness::analyze( + &input.witness, + redeem.effective_output_type(output_type, input.witness.len()), + flags, + ); + + Facts { + script_sig, + redeem, + witness, + } +} + +#[cfg(test)] +mod tests { + use bitcoin::TxIn; + use brk_types::OutputType; + + use super::analyze; + use crate::TxFeatureFlags; + + #[test] + fn empty_script_sig_has_empty_facts() { + let input = TxIn::default(); + let facts = analyze(&input, OutputType::Unknown, &mut TxFeatureFlags::default()); + + assert_eq!(facts.script_sig.accurate_sigops, 0); + assert_eq!(facts.script_sig.last_push, None); + assert_eq!(facts.script_sig.legacy_sigops, 0); + assert!(facts.script_sig.push_only); + } +} diff --git a/crates/brk_indexer/src/processor/transaction/analysis/input/redeem.rs b/crates/brk_indexer/src/processor/transaction/analysis/input/redeem.rs new file mode 100644 index 000000000..d88f49bf7 --- /dev/null +++ b/crates/brk_indexer/src/processor/transaction/analysis/input/redeem.rs @@ -0,0 +1,113 @@ +use bitcoin::{Script, WitnessVersion}; +use brk_types::OutputType; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Kind { + None, + P2WPKH, + P2WSH, + Witness, + Other, +} + +#[derive(Clone, Copy)] +pub(super) struct Facts { + kind: Kind, + sigops: usize, +} + +impl Facts { + pub(super) fn analyze(last_push: Option<&[u8]>, output_type: OutputType) -> Self { + if output_type != OutputType::P2SH { + return Self::NONE; + } + + let Some(bytes) = last_push else { + return Self::NONE; + }; + let script = Script::from_bytes(bytes); + let kind = match script.witness_version() { + Some(WitnessVersion::V0) if script.len() == 22 => Kind::P2WPKH, + Some(WitnessVersion::V0) if script.len() == 34 => Kind::P2WSH, + Some(_) => Kind::Witness, + None => Kind::Other, + }; + + Self { + kind, + sigops: script.count_sigops(), + } + } + + pub(super) fn effective_output_type( + self, + output_type: OutputType, + witness_items: usize, + ) -> OutputType { + match self.kind { + Kind::P2WPKH if witness_items == 2 => OutputType::P2WPKH, + _ => output_type, + } + } + + pub(super) fn is_present(self) -> bool { + self.kind != Kind::None + } + + pub(super) fn is_p2wpkh(self) -> bool { + self.kind == Kind::P2WPKH + } + + pub(super) fn is_p2wsh(self) -> bool { + self.kind == Kind::P2WSH + } + + pub(super) fn is_witness_program(self) -> bool { + matches!(self.kind, Kind::P2WPKH | Kind::P2WSH | Kind::Witness) + } + + pub(super) fn sigops(self) -> Option { + self.is_present().then_some(self.sigops) + } + + const NONE: Self = Self { + kind: Kind::None, + sigops: 0, + }; +} + +#[cfg(test)] +mod tests { + use bitcoin::ScriptBuf; + use brk_types::OutputType; + + use super::{Facts, Kind}; + + fn classify(hex: &str) -> Facts { + let script = ScriptBuf::from_hex(hex).unwrap(); + Facts::analyze(Some(script.as_bytes()), OutputType::P2SH) + } + + #[test] + fn classifies_redeem_scripts_once() { + assert_eq!( + classify("00140000000000000000000000000000000000000000").kind, + Kind::P2WPKH + ); + assert_eq!( + classify("00200000000000000000000000000000000000000000000000000000000000000000").kind, + Kind::P2WSH + ); + assert_eq!( + classify("51200000000000000000000000000000000000000000000000000000000000000000").kind, + Kind::Witness + ); + assert_eq!(classify("ac").kind, Kind::Other); + assert_eq!(classify("ac").sigops(), Some(1)); + assert_eq!(Facts::analyze(None, OutputType::P2SH).kind, Kind::None); + assert_eq!( + Facts::analyze(Some(&[0xac]), OutputType::P2PKH).kind, + Kind::None + ); + } +} diff --git a/crates/brk_indexer/src/processor/transaction/analysis/input/script_sig.rs b/crates/brk_indexer/src/processor/transaction/analysis/input/script_sig.rs new file mode 100644 index 000000000..8766c1bcc --- /dev/null +++ b/crates/brk_indexer/src/processor/transaction/analysis/input/script_sig.rs @@ -0,0 +1,240 @@ +use bitcoin::{ + Script, + opcodes::all::{OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY, OP_CHECKSIG, OP_CHECKSIGVERIFY}, + script::Instruction, +}; +use brk_types::OutputType; + +use super::{super::features, ScriptSigFacts}; +use crate::TxFeatureFlags; + +pub(super) fn analyze<'a>( + script: &'a Script, + output_type: OutputType, + scan_signatures: bool, + flags: &mut TxFeatureFlags, +) -> ScriptSigFacts<'a> { + if scan_signatures && let Some((signature, last_push)) = direct_push_spend(script, output_type) + { + features::record_validated_ecdsa_sighash(signature, flags); + return ScriptSigFacts { + accurate_sigops: 0, + last_push: Some(last_push), + legacy_sigops: 0, + push_only: true, + }; + } + + let mut accurate_sigops = 0; + let mut first_push = None; + let mut last_push = None; + let mut legacy_sigops = 0; + let mut only_push_bytes = true; + let mut push_count = 0; + let mut push_only = true; + let mut pushnum = None; + let validated_shape = scan_signatures + && matches!( + output_type, + OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65 + ); + + for instruction in script.instructions() { + match instruction { + Ok(Instruction::PushBytes(bytes)) => { + let bytes = bytes.as_bytes(); + first_push.get_or_insert(bytes); + last_push = Some(bytes); + push_count += 1; + pushnum = None; + if scan_signatures && !validated_shape { + features::scan_ecdsa_signature(bytes, flags); + } + } + Ok(Instruction::Op(opcode)) => { + only_push_bytes = false; + last_push = None; + if opcode.to_u8() > 0x60 { + push_only = false; + } + match opcode { + OP_CHECKSIG | OP_CHECKSIGVERIFY => { + accurate_sigops += 1; + legacy_sigops += 1; + } + OP_CHECKMULTISIG | OP_CHECKMULTISIGVERIFY => { + accurate_sigops += pushnum.unwrap_or(20); + legacy_sigops += 20; + } + _ => pushnum = decode_pushnum(opcode.to_u8()), + } + } + Err(_) => { + only_push_bytes = false; + last_push = None; + push_only = false; + break; + } + } + } + + if validated_shape { + let expected_pushes = match output_type { + OutputType::P2PKH => 2, + OutputType::P2PK33 | OutputType::P2PK65 => 1, + _ => unreachable!(), + }; + + if only_push_bytes && push_count == expected_pushes { + features::record_validated_ecdsa_sighash(first_push.unwrap(), flags); + } else { + scan_ecdsa_signatures(script, flags); + } + } + + ScriptSigFacts { + accurate_sigops, + last_push, + legacy_sigops, + push_only, + } +} + +fn direct_push_spend(script: &Script, output_type: OutputType) -> Option<(&[u8], &[u8])> { + let bytes = script.as_bytes(); + let signature_len = usize::from(*bytes.first()?); + if !(1..=75).contains(&signature_len) { + return None; + } + + let signature_end = 1 + signature_len; + let signature = bytes.get(1..signature_end)?; + + match output_type { + OutputType::P2PK33 | OutputType::P2PK65 => { + (signature_end == bytes.len()).then_some((signature, signature)) + } + OutputType::P2PKH => { + let public_key_len = usize::from(*bytes.get(signature_end)?); + if !matches!(public_key_len, 33 | 65) { + return None; + } + + let public_key_start = signature_end + 1; + let public_key_end = public_key_start + public_key_len; + (public_key_end == bytes.len()) + .then(|| (signature, &bytes[public_key_start..public_key_end])) + } + _ => None, + } +} + +fn scan_ecdsa_signatures(script: &Script, flags: &mut TxFeatureFlags) { + for instruction in script.instructions() { + if let Ok(Instruction::PushBytes(bytes)) = instruction { + features::scan_ecdsa_signature(bytes.as_bytes(), flags); + } + } +} + +#[inline] +fn decode_pushnum(opcode: u8) -> Option { + (0x51..=0x60) + .contains(&opcode) + .then(|| usize::from(opcode - 0x50)) +} + +#[cfg(test)] +mod tests { + use bitcoin::ScriptBuf; + use brk_types::OutputType; + + use super::analyze; + use crate::TxFeatureFlags; + + fn analyze_script(script: &ScriptBuf) -> super::ScriptSigFacts<'_> { + analyze( + script, + OutputType::Unknown, + false, + &mut TxFeatureFlags::default(), + ) + } + + #[test] + fn analyzes_last_push_and_push_only_together() { + let pushed = ScriptBuf::from_hex("03616263").unwrap(); + let facts = analyze_script(&pushed); + assert_eq!(facts.last_push, Some(b"abc".as_slice())); + assert!(facts.push_only); + + let followed_by_checksig = ScriptBuf::from_hex("03616263ac").unwrap(); + let facts = analyze_script(&followed_by_checksig); + assert_eq!(facts.last_push, None); + assert!(!facts.push_only); + + let push_number = ScriptBuf::from_hex("51").unwrap(); + let facts = analyze_script(&push_number); + assert_eq!(facts.last_push, None); + assert!(facts.push_only); + } + + #[test] + fn counts_legacy_and_accurate_sigops_together() { + let script = ScriptBuf::from_hex("52aeac").unwrap(); + let facts = analyze_script(&script); + + assert_eq!(facts.legacy_sigops, 21); + assert_eq!(facts.accurate_sigops, 3); + } + + #[test] + fn records_validated_p2pkh_and_p2pk_sighashes() { + let signature = [0x30, 0x06, 0x02, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01]; + let compressed_key = [0x02; 33]; + let uncompressed_key = [0x04; 65]; + let mut scripts = Vec::new(); + + for public_key in [compressed_key.as_slice(), uncompressed_key.as_slice()] { + let mut bytes = Vec::with_capacity(signature.len() + public_key.len() + 2); + bytes.push(signature.len() as u8); + bytes.extend(signature); + bytes.push(public_key.len() as u8); + bytes.extend(public_key); + scripts.push((ScriptBuf::from_bytes(bytes), OutputType::P2PKH, public_key)); + } + + let mut p2pk = Vec::with_capacity(signature.len() + 1); + p2pk.push(signature.len() as u8); + p2pk.extend(signature); + scripts.push(( + ScriptBuf::from_bytes(p2pk), + OutputType::P2PK33, + signature.as_slice(), + )); + + for (script, output_type, last_push) in scripts { + let mut flags = TxFeatureFlags::default(); + let facts = analyze(&script, output_type, true, &mut flags); + + assert_eq!(facts.accurate_sigops, 0); + assert_eq!(facts.last_push, Some(last_push)); + assert_eq!(facts.legacy_sigops, 0); + assert!(facts.push_only); + assert!(flags.is_set(TxFeatureFlags::SIGHASH_ALL)); + } + } + + #[test] + fn falls_back_for_noncanonical_p2pkh_shape() { + let script = ScriptBuf::from_hex( + "000930060201010201010121020000000000000000000000000000000000000000000000000000000000000000", + ) + .unwrap(); + let mut flags = TxFeatureFlags::default(); + + analyze(&script, OutputType::P2PKH, true, &mut flags); + + assert!(flags.is_set(TxFeatureFlags::SIGHASH_ALL)); + } +} diff --git a/crates/brk_indexer/src/processor/transaction/analysis/input/witness.rs b/crates/brk_indexer/src/processor/transaction/analysis/input/witness.rs new file mode 100644 index 000000000..d8424d92c --- /dev/null +++ b/crates/brk_indexer/src/processor/transaction/analysis/input/witness.rs @@ -0,0 +1,202 @@ +use bitcoin::{ + Script, Witness, + opcodes::all::OP_IF, + script::Instruction, + taproot::{LeafVersion, TAPROOT_ANNEX_PREFIX, TAPROOT_CONTROL_BASE_SIZE, TAPROOT_LEAF_MASK}, +}; +use brk_types::OutputType; + +use super::{super::features, WitnessFacts}; +use crate::TxFeatureFlags; + +pub(super) fn analyze<'a>( + witness: &'a Witness, + output_type: OutputType, + flags: &mut TxFeatureFlags, +) -> WitnessFacts<'a> { + match output_type { + OutputType::P2TR => analyze_taproot(witness, flags), + OutputType::P2WPKH => analyze_p2wpkh(witness, flags), + _ => analyze_ecdsa(witness, flags), + } +} + +fn analyze_p2wpkh<'a>(witness: &'a Witness, flags: &mut TxFeatureFlags) -> WitnessFacts<'a> { + let mut stack = witness.iter(); + let signature = stack.next().unwrap(); + let public_key = stack.next().unwrap(); + debug_assert!(stack.next().is_none()); + features::record_validated_ecdsa_sighash(signature, flags); + + WitnessFacts { + has_annex: false, + last: Some(public_key), + leaf_version: None, + max_argument_bytes: signature.len(), + stack_items: 2, + } +} + +fn analyze_ecdsa<'a>(witness: &'a Witness, flags: &mut TxFeatureFlags) -> WitnessFacts<'a> { + let stack_items = witness.len(); + let mut last = None; + let mut max_argument_bytes = 0; + + for (index, item) in witness.iter().enumerate() { + features::scan_ecdsa_signature(item, flags); + if index + 1 < stack_items { + max_argument_bytes = max_argument_bytes.max(item.len()); + } + last = Some(item); + } + + WitnessFacts { + has_annex: false, + last, + leaf_version: None, + max_argument_bytes, + stack_items, + } +} + +fn analyze_taproot<'a>(witness: &'a Witness, flags: &mut TxFeatureFlags) -> WitnessFacts<'a> { + let len = witness.len(); + if len == 1 { + let signature = witness.last().unwrap(); + features::record_validated_taproot_sighash(signature, flags); + return WitnessFacts { + has_annex: false, + last: Some(signature), + leaf_version: None, + max_argument_bytes: signature.len(), + stack_items: 1, + }; + } + + let last = witness.last(); + let has_annex = len > 1 && last.is_some_and(|item| item.first() == Some(&TAPROOT_ANNEX_PREFIX)); + if has_annex { + flags.insert(TxFeatureFlags::ANNEX); + } + + let stack_items = len - usize::from(has_annex); + let argument_count = if stack_items == 1 { + 1 + } else { + stack_items.saturating_sub(2) + }; + let script_index = (stack_items >= 2).then(|| stack_items - 2); + let control_index = (stack_items >= 2).then(|| stack_items - 1); + let is_key_path = stack_items == 1; + let mut leaf_version = None; + let mut max_argument_bytes = 0; + + for (index, item) in witness.iter().enumerate() { + if index < argument_count { + if is_key_path { + features::record_validated_taproot_sighash(item, flags); + } else { + features::scan_taproot_signature(item, flags); + } + max_argument_bytes = max_argument_bytes.max(item.len()); + } else if Some(index) == script_index { + if has_inscription_envelope(Script::from_bytes(item)) { + flags.insert(TxFeatureFlags::INSCRIPTION); + } + } else if Some(index) == control_index && item.len() >= TAPROOT_CONTROL_BASE_SIZE { + leaf_version = LeafVersion::from_consensus(item[0] & TAPROOT_LEAF_MASK).ok(); + } + } + + WitnessFacts { + has_annex, + last, + leaf_version, + max_argument_bytes, + stack_items, + } +} + +fn has_inscription_envelope(script: &Script) -> bool { + let mut state = 0; + for instruction in script.instructions() { + state = match (state, instruction) { + (0, Ok(Instruction::PushBytes(bytes))) if bytes.is_empty() => 1, + (1, Ok(Instruction::Op(OP_IF))) => 2, + (2, Ok(Instruction::PushBytes(bytes))) if bytes.as_bytes() == b"ord" => return true, + (_, Ok(Instruction::PushBytes(bytes))) if bytes.is_empty() => 1, + _ => 0, + }; + } + false +} + +#[cfg(test)] +mod tests { + use bitcoin::{ScriptBuf, TxIn, Witness, taproot::TAPROOT_ANNEX_PREFIX}; + use brk_types::OutputType; + + use super::{analyze, has_inscription_envelope}; + use crate::TxFeatureFlags; + + #[test] + fn recognizes_only_ord_envelopes() { + let inscription = ScriptBuf::from_hex("0063036f726468").unwrap(); + let generic_envelope = ScriptBuf::from_hex("006303666f6f68").unwrap(); + + assert!(has_inscription_envelope(&inscription)); + assert!(!has_inscription_envelope(&generic_envelope)); + } + + #[test] + fn reads_tapscript_before_control_block_and_annex() { + let script = ScriptBuf::from_hex("0063036f726468").unwrap(); + let control_block = [0xc0; 33]; + let annex = [TAPROOT_ANNEX_PREFIX, 0x01]; + let input = TxIn { + witness: Witness::from_slice(&[ + [0x01].as_slice(), + script.as_bytes(), + control_block.as_slice(), + annex.as_slice(), + ]), + ..TxIn::default() + }; + let mut flags = TxFeatureFlags::default(); + + let facts = analyze(&input.witness, OutputType::P2TR, &mut flags); + + assert!(facts.has_annex); + assert!(flags.is_set(TxFeatureFlags::ANNEX)); + assert!(flags.is_set(TxFeatureFlags::INSCRIPTION)); + } + + #[test] + fn reads_fixed_shape_p2wpkh_once() { + let signature = [0x01; 71]; + let public_key = [0x02; 33]; + let witness = Witness::from_slice(&[signature.as_slice(), public_key.as_slice()]); + let mut flags = TxFeatureFlags::default(); + + let facts = analyze(&witness, OutputType::P2WPKH, &mut flags); + + assert_eq!(facts.last, Some(public_key.as_slice())); + assert_eq!(facts.max_argument_bytes, signature.len()); + assert_eq!(facts.stack_items, 2); + assert!(flags.is_set(TxFeatureFlags::SIGHASH_ALL)); + } + + #[test] + fn reads_taproot_key_path_once() { + let signature = [0x00; 64]; + let witness = Witness::from_slice(&[signature]); + let mut flags = TxFeatureFlags::default(); + + let facts = analyze(&witness, OutputType::P2TR, &mut flags); + + assert_eq!(facts.last, Some(signature.as_slice())); + assert_eq!(facts.max_argument_bytes, signature.len()); + assert_eq!(facts.stack_items, 1); + assert!(flags.is_set(TxFeatureFlags::SIGHASH_DEFAULT)); + } +} diff --git a/crates/brk_indexer/src/processor/transaction/analysis/mod.rs b/crates/brk_indexer/src/processor/transaction/analysis/mod.rs new file mode 100644 index 000000000..31989e048 --- /dev/null +++ b/crates/brk_indexer/src/processor/transaction/analysis/mod.rs @@ -0,0 +1,98 @@ +mod features; +mod input; +mod output; +mod policy; +mod sigops; + +use brk_types::{OutputType, SigOps}; +use rayon::prelude::*; +use vecdb::{likely, unlikely}; + +use super::ComputedTx; +use crate::TxFeatureFlags; +use crate::processor::{BlockProcessor, txin::InputSource, txout::ProcessedOutput}; + +pub(super) struct TransactionAnalysis { + pub(super) total_sigop_cost: SigOps, + pub(super) explicitly_rbf: bool, + pub(super) features: TxFeatureFlags, +} + +impl BlockProcessor<'_> { + #[inline] + pub(crate) fn tracks_executed_legacy_sigops(&self) -> bool { + policy::tracks_executed_legacy_sigops(self.height) + } + + pub(super) fn analyze_transactions( + &self, + txs: &[ComputedTx<'_>], + txins: &[InputSource], + txouts: &[ProcessedOutput], + ) -> Vec { + let block_first_tx_index = self.lengths.tx_index; + let track_executed_legacy = self.tracks_executed_legacy_sigops(); + + txs.par_iter() + .map(|tx| { + let tx_inputs = tx.inputs(txins); + let tx_outputs = tx.outputs(txouts); + let is_coinbase = tx.is_coinbase(block_first_tx_index); + + let mut sigops = sigops::Accumulator::new(track_executed_legacy); + let mut flags = TxFeatureFlags::default(); + let mut explicitly_rbf = false; + let mut output_scanner = output::Scanner::default(); + let mut policy = policy::Accumulator::new(self.height); + + if unlikely(is_coinbase) { + let input = &tx.tx.input[0]; + explicitly_rbf = input.sequence.is_rbf(); + sigops.scan_coinbase_input(input); + } else { + for (input, source) in tx.tx.input.iter().zip(tx_inputs) { + explicitly_rbf |= input.sequence.is_rbf(); + let (output_type, legacy_sigops) = resolved_output_facts(source, txouts); + let facts = input::analyze(input, output_type, &mut flags); + sigops.scan_input(output_type, legacy_sigops, &facts); + policy.scan_input(input, output_type, &facts); + } + } + + for (txout, output) in tx.tx.output.iter().zip(tx_outputs) { + output_scanner.scan(txout, output, &mut flags); + sigops.scan_output(txout, output); + if likely(!is_coinbase) { + policy.scan_output(txout, output); + } + } + + let sigops = sigops.finish(); + if likely(!is_coinbase) { + policy.finish(tx, sigops, &mut flags); + } + + TransactionAnalysis { + total_sigop_cost: sigops.total, + explicitly_rbf, + features: flags, + } + }) + .collect() + } +} + +fn resolved_output_facts(source: &InputSource, txouts: &[ProcessedOutput]) -> (OutputType, SigOps) { + match source { + InputSource::PreviousBlock { + output_type, + legacy_sigops, + .. + } => (*output_type, *legacy_sigops), + InputSource::SameBlock { txout_offset, .. } => { + let output = &txouts[*txout_offset]; + (output.output_type, output.legacy_sigops) + } + InputSource::Coinbase => unreachable!(), + } +} diff --git a/crates/brk_indexer/src/processor/transaction/analysis/output.rs b/crates/brk_indexer/src/processor/transaction/analysis/output.rs new file mode 100644 index 000000000..6dc6c4171 --- /dev/null +++ b/crates/brk_indexer/src/processor/transaction/analysis/output.rs @@ -0,0 +1,79 @@ +use crate::TxFeatureFlags; +use crate::processor::txout::ProcessedOutput; +use bitcoin::{PublicKey, TxOut, script::Instruction}; +use brk_types::OutputType; + +#[derive(Default)] +pub(super) struct Scanner { + p2wsh_count: usize, + payload_size: usize, +} + +impl Scanner { + pub(super) fn scan( + &mut self, + txout: &TxOut, + output: &ProcessedOutput, + flags: &mut TxFeatureFlags, + ) { + flags.insert_type(output.output_type); + + match output.output_type { + OutputType::P2PK33 | OutputType::P2PK65 => { + if txout.script_pubkey.p2pk_public_key().is_none() { + flags.insert(TxFeatureFlags::FAKE_PUBKEY); + } + self.p2wsh_count = 0; + } + OutputType::P2MS => { + if has_fake_multisig_key(&txout.script_pubkey) { + flags.insert(TxFeatureFlags::FAKE_PUBKEY); + } + self.p2wsh_count = 0; + } + OutputType::P2WSH => { + let program = &txout.script_pubkey.as_bytes()[2..]; + if self.p2wsh_count == 0 { + self.payload_size = u16::from_be_bytes([program[0], program[1]]) as usize; + } + self.p2wsh_count += 1; + + let output_count = (self.payload_size + 33) / 32; + if self.p2wsh_count == output_count { + let padding = output_count * 32 - self.payload_size - 2; + if program[32 - padding..].iter().all(|byte| *byte == 0) { + flags.insert(TxFeatureFlags::FAKE_SCRIPTHASH); + } + } + } + _ => self.p2wsh_count = 0, + } + } +} + +fn has_fake_multisig_key(script: &bitcoin::Script) -> bool { + script.instructions().any(|instruction| { + let Ok(Instruction::PushBytes(bytes)) = instruction else { + return false; + }; + let bytes = bytes.as_bytes(); + is_burn_key(bytes) || PublicKey::from_slice(bytes).is_err() + }) +} + +fn is_burn_key(bytes: &[u8]) -> bool { + bytes.len() == 33 + && matches!( + bytes, + [0x02, rest @ ..] + if rest.iter().all(|byte| *byte == 0x02) + || rest.iter().all(|byte| *byte == 0x22) + ) + || bytes.len() == 33 + && matches!( + bytes, + [0x03, rest @ ..] + if rest.iter().all(|byte| *byte == 0x03) + || rest.iter().all(|byte| *byte == 0x33) + ) +} diff --git a/crates/brk_indexer/src/processor/transaction/analysis/policy.rs b/crates/brk_indexer/src/processor/transaction/analysis/policy.rs new file mode 100644 index 000000000..43a513f80 --- /dev/null +++ b/crates/brk_indexer/src/processor/transaction/analysis/policy.rs @@ -0,0 +1,384 @@ +use bitcoin::{ + Amount, Script, Transaction, TxIn, TxOut, WitnessVersion, policy::MAX_STANDARD_TX_SIGOPS_COST, + taproot::LeafVersion, +}; +use brk_types::{Height, OutputType, SigOps}; + +use super::super::ComputedTx; +use super::{input, sigops::ComputedSigOps}; +use crate::TxFeatureFlags; +use crate::processor::txout::ProcessedOutput; + +// Deterministic policy snapshots, not consensus activation heights. +const LAST_V2_POLICY_HEIGHT: u32 = 863_500; +const FIRST_V29_POLICY_HEIGHT: u32 = 892_500; +const FIRST_V30_POLICY_HEIGHT: u32 = 921_000; +// rust-bitcoin 0.32 still exposes Core's previous 82-byte policy value. +const MIN_STANDARD_TX_NONWITNESS_SIZE: u32 = 65; +const MAX_EXECUTED_LEGACY_SIGOP_COST: u32 = 10_000; +const MAX_SCRIPT_SIG_SIZE: usize = 1_650; +const MAX_P2SH_SIGOPS: usize = 15; +const MAX_V29_OP_RETURN_SCRIPT_BYTES: usize = 83; +const MAX_V30_OP_RETURN_SCRIPT_BYTES: usize = 100_000; +const MAX_P2WSH_SCRIPT_BYTES: usize = 3_600; +const MAX_P2WSH_STACK_ITEMS: usize = 100; +const MAX_WITNESS_STACK_ITEM_BYTES: usize = 80; +const MAX_STANDARD_BARE_MULTISIG_SIGOP_COST: u32 = + 3 * bitcoin::constants::WITNESS_SCALE_FACTOR as u32; + +#[derive(Default)] +pub(super) struct Accumulator { + height: u32, + nonstandard: bool, + dust_output_count: usize, + op_return_count: usize, + op_return_script_bytes: usize, +} + +impl Accumulator { + pub(super) fn new(height: Height) -> Self { + Self { + height: height.into(), + ..Self::default() + } + } + + pub(super) fn scan_input( + &mut self, + input: &TxIn, + output_type: OutputType, + facts: &input::Facts<'_>, + ) { + if input.script_sig.len() > MAX_SCRIPT_SIG_SIZE || !facts.script_sig.push_only { + self.nonstandard = true; + return; + } + + self.nonstandard |= match output_type { + OutputType::P2SH => facts + .redeem_sigops() + .is_some_and(|count| count > MAX_P2SH_SIGOPS), + OutputType::P2A => self.height <= LAST_V2_POLICY_HEIGHT, + OutputType::P2TR => facts.witness.has_annex, + OutputType::OpReturn | OutputType::Empty | OutputType::Unknown => true, + _ => false, + } || has_nonstandard_witness(output_type, facts); + } + + pub(super) fn scan_output(&mut self, txout: &TxOut, output: &ProcessedOutput) { + let script = &txout.script_pubkey; + self.nonstandard |= match output.output_type { + OutputType::Empty => true, + OutputType::Unknown => !is_standard_unknown_witness(script), + OutputType::P2MS => has_too_many_bare_multisig_keys(output.legacy_sigops), + OutputType::OpReturn => { + self.op_return_count += 1; + self.op_return_script_bytes += script.len(); + false + } + _ => false, + }; + self.dust_output_count += is_dust(txout.value, output.output_type, script) as usize; + } + + pub(super) fn finish( + mut self, + tx: &ComputedTx<'_>, + sigops: ComputedSigOps, + flags: &mut TxFeatureFlags, + ) { + self.nonstandard |= if self.height < FIRST_V30_POLICY_HEIGHT { + self.op_return_count > 1 || self.op_return_script_bytes > MAX_V29_OP_RETURN_SCRIPT_BYTES + } else { + self.op_return_script_bytes > MAX_V30_OP_RETURN_SCRIPT_BYTES + }; + self.nonstandard |= + has_unconditionally_nonstandard_dust(self.height, self.dust_output_count); + self.nonstandard |= has_nonstandard_header(tx, sigops, self.height); + + if self.nonstandard { + flags.insert(TxFeatureFlags::UNCONDITIONALLY_NONSTANDARD); + } + if self.dust_output_count > 0 { + flags.insert(TxFeatureFlags::DUST_OUTPUT); + } + } +} + +pub(super) fn tracks_executed_legacy_sigops(height: Height) -> bool { + u32::from(height) >= FIRST_V30_POLICY_HEIGHT +} + +fn has_nonstandard_header(tx: &ComputedTx, sigops: ComputedSigOps, height: u32) -> bool { + has_nonstandard_version(tx.tx.version.0, height) + || tx.weight() > Transaction::MAX_STANDARD_WEIGHT + || tx.base_size < MIN_STANDARD_TX_NONWITNESS_SIZE + || u32::from(sigops.total) > MAX_STANDARD_TX_SIGOPS_COST + || height >= FIRST_V30_POLICY_HEIGHT + && u32::from(sigops.executed_legacy) > MAX_EXECUTED_LEGACY_SIGOP_COST +} + +fn has_unconditionally_nonstandard_dust(height: u32, dust_output_count: usize) -> bool { + if height < FIRST_V29_POLICY_HEIGHT { + dust_output_count > 0 + } else { + dust_output_count > 1 + } +} + +fn is_dust(value: Amount, output_type: OutputType, script: &Script) -> bool { + let threshold = match output_type { + OutputType::P2PK65 => 672, + OutputType::P2PK33 => 576, + OutputType::P2PKH => 546, + OutputType::P2MS | OutputType::Unknown => { + return value < script.minimal_non_dust(); + } + OutputType::P2SH => 540, + OutputType::OpReturn => return false, + OutputType::P2WPKH => 294, + OutputType::P2WSH | OutputType::P2TR => 330, + OutputType::P2A => 240, + OutputType::Empty => 471, + }; + + value.to_sat() < threshold +} + +fn has_nonstandard_witness(output_type: OutputType, facts: &input::Facts<'_>) -> bool { + if facts.witness.stack_items == 0 { + return false; + } + + match output_type { + OutputType::P2A => true, + OutputType::P2WPKH => false, + OutputType::P2WSH => has_nonstandard_p2wsh_witness(&facts.witness), + OutputType::P2TR => { + facts.witness.has_annex || has_nonstandard_taproot_witness(&facts.witness) + } + OutputType::P2SH => { + if facts.redeem_sigops().is_none() { + return true; + } + if facts.redeem_is_p2wsh() { + has_nonstandard_p2wsh_witness(&facts.witness) + } else { + !facts.redeem_is_witness_program() + } + } + _ => true, + } +} + +fn has_nonstandard_p2wsh_witness(witness: &input::WitnessFacts<'_>) -> bool { + let stack_items = witness.stack_items - 1; + witness.last.unwrap().len() > MAX_P2WSH_SCRIPT_BYTES + || stack_items > MAX_P2WSH_STACK_ITEMS + || witness.max_argument_bytes > MAX_WITNESS_STACK_ITEM_BYTES +} + +fn has_nonstandard_taproot_witness(witness: &input::WitnessFacts<'_>) -> bool { + if witness.stack_items < 2 { + return false; + } + + witness.leaf_version.is_none() + || witness.leaf_version == Some(LeafVersion::TapScript) + && witness.max_argument_bytes > MAX_WITNESS_STACK_ITEM_BYTES +} + +fn has_nonstandard_version(version: i32, height: u32) -> bool { + let max = if height <= LAST_V2_POLICY_HEIGHT { + 2 + } else { + 3 + }; + !(1..=max).contains(&version) +} + +fn is_standard_unknown_witness(script: &Script) -> bool { + script + .witness_version() + .is_some_and(|version| version != WitnessVersion::V0) +} + +fn has_too_many_bare_multisig_keys(sigops: SigOps) -> bool { + u32::from(sigops) > MAX_STANDARD_BARE_MULTISIG_SIGOP_COST +} + +#[cfg(test)] +mod tests { + use bitcoin::{Amount, ScriptBuf, TxIn, Witness}; + use brk_types::{ + AddrBytes, Height, OutputType, P2ABytes, P2PK33Bytes, P2PK65Bytes, P2PKHBytes, P2SHBytes, + P2TRBytes, P2WPKHBytes, P2WSHBytes, SigOps, + }; + + use super::super::input; + use super::{ + FIRST_V29_POLICY_HEIGHT, FIRST_V30_POLICY_HEIGHT, LAST_V2_POLICY_HEIGHT, + has_nonstandard_p2wsh_witness, has_nonstandard_taproot_witness, has_nonstandard_version, + has_nonstandard_witness, has_too_many_bare_multisig_keys, + has_unconditionally_nonstandard_dust, is_dust, is_standard_unknown_witness, + tracks_executed_legacy_sigops, + }; + + #[test] + fn tracks_executed_sigops_only_when_policy_uses_them() { + assert!(!tracks_executed_legacy_sigops(Height::from( + FIRST_V30_POLICY_HEIGHT - 1 + ))); + assert!(tracks_executed_legacy_sigops(Height::from( + FIRST_V30_POLICY_HEIGHT + ))); + } + + #[test] + fn accepts_only_policy_transaction_versions() { + assert!(has_nonstandard_version(0, LAST_V2_POLICY_HEIGHT)); + assert!(has_nonstandard_version(-1, LAST_V2_POLICY_HEIGHT)); + assert!(!has_nonstandard_version(1, LAST_V2_POLICY_HEIGHT)); + assert!(!has_nonstandard_version(2, LAST_V2_POLICY_HEIGHT)); + assert!(has_nonstandard_version(3, LAST_V2_POLICY_HEIGHT)); + assert!(!has_nonstandard_version(3, LAST_V2_POLICY_HEIGHT + 1)); + } + + #[test] + fn applies_ephemeral_dust_policy_from_v29() { + assert!(!has_unconditionally_nonstandard_dust( + FIRST_V29_POLICY_HEIGHT - 1, + 0 + )); + assert!(has_unconditionally_nonstandard_dust( + FIRST_V29_POLICY_HEIGHT - 1, + 1 + )); + assert!(!has_unconditionally_nonstandard_dust( + FIRST_V29_POLICY_HEIGHT, + 1 + )); + assert!(has_unconditionally_nonstandard_dust( + FIRST_V29_POLICY_HEIGHT, + 2 + )); + } + + #[test] + fn uses_exact_dust_thresholds_for_fixed_scripts() { + let scripts = [ + AddrBytes::from(P2PK65Bytes::from(&[0; 65][..])).to_script_pubkey(), + AddrBytes::from(P2PK33Bytes::from(&[0; 33][..])).to_script_pubkey(), + AddrBytes::from(P2PKHBytes::from(&[0; 20][..])).to_script_pubkey(), + AddrBytes::from(P2SHBytes::from(&[0; 20][..])).to_script_pubkey(), + AddrBytes::from(P2WPKHBytes::from(&[0; 20][..])).to_script_pubkey(), + AddrBytes::from(P2WSHBytes::from(&[0; 32][..])).to_script_pubkey(), + AddrBytes::from(P2TRBytes::from(&[0; 32][..])).to_script_pubkey(), + AddrBytes::from(P2ABytes::from(&[0; 2][..])).to_script_pubkey(), + ]; + let thresholds = [672, 576, 546, 540, 294, 330, 330, 240]; + + for ((output_type, script), threshold) in OutputType::ADDR_TYPES + .into_iter() + .zip(&scripts) + .zip(thresholds) + { + assert_eq!(script.minimal_non_dust().to_sat(), threshold); + assert!(is_dust( + Amount::from_sat(threshold - 1), + output_type, + script + )); + assert!(!is_dust(Amount::from_sat(threshold), output_type, script)); + } + + let empty = ScriptBuf::new(); + assert_eq!(empty.minimal_non_dust().to_sat(), 471); + assert!(is_dust(Amount::from_sat(470), OutputType::Empty, &empty)); + assert!(!is_dust(Amount::from_sat(471), OutputType::Empty, &empty)); + } + + #[test] + fn keeps_exact_dust_calculation_for_unknown_scripts() { + let script = ScriptBuf::from_bytes(vec![0x61; 1_000]); + let threshold = script.minimal_non_dust().to_sat(); + + assert!(is_dust( + Amount::from_sat(threshold - 1), + OutputType::Unknown, + &script + )); + assert!(!is_dust( + Amount::from_sat(threshold), + OutputType::Unknown, + &script + )); + } + + #[test] + fn recognizes_future_witness_programs() { + assert!(is_standard_unknown_witness( + &ScriptBuf::from_hex( + "52200000000000000000000000000000000000000000000000000000000000000000" + ) + .unwrap() + )); + assert!(!is_standard_unknown_witness( + &ScriptBuf::from_hex( + "00200000000000000000000000000000000000000000000000000000000000000000" + ) + .unwrap() + )); + } + + #[test] + fn limits_standard_bare_multisig_to_three_keys() { + assert!(!has_too_many_bare_multisig_keys(SigOps::new(12))); + assert!(has_too_many_bare_multisig_keys(SigOps::new(16))); + } + + #[test] + fn rejects_p2a_witness_stuffing() { + let input = TxIn { + witness: Witness::from_slice(&[b"stuffing"]), + ..TxIn::default() + }; + let facts = input::analyze( + &input, + OutputType::P2A, + &mut crate::TxFeatureFlags::default(), + ); + assert!(has_nonstandard_witness(OutputType::P2A, &facts,)); + } + + #[test] + fn enforces_witness_stack_item_limits() { + let oversized = [0_u8; 81]; + let input = TxIn { + witness: Witness::from_slice(&[oversized.as_slice(), [0x51].as_slice()]), + ..TxIn::default() + }; + let facts = input::analyze( + &input, + OutputType::P2WSH, + &mut crate::TxFeatureFlags::default(), + ); + assert!(has_nonstandard_p2wsh_witness(&facts.witness)); + + let control_block = [0xc0_u8; 33]; + let input = TxIn { + witness: Witness::from_slice(&[ + oversized.as_slice(), + [0x51].as_slice(), + control_block.as_slice(), + ]), + ..TxIn::default() + }; + let facts = input::analyze( + &input, + OutputType::P2TR, + &mut crate::TxFeatureFlags::default(), + ); + assert!(has_nonstandard_taproot_witness(&facts.witness)); + } +} diff --git a/crates/brk_indexer/src/processor/transaction/analysis/sigops.rs b/crates/brk_indexer/src/processor/transaction/analysis/sigops.rs new file mode 100644 index 000000000..0e23cdfb0 --- /dev/null +++ b/crates/brk_indexer/src/processor/transaction/analysis/sigops.rs @@ -0,0 +1,192 @@ +use bitcoin::{Script, TxIn, TxOut, constants::WITNESS_SCALE_FACTOR}; +use brk_types::{OutputType, SigOps}; + +use crate::processor::txout::ProcessedOutput; + +use super::input; + +#[derive(Default)] +pub(super) struct Accumulator { + legacy: usize, + redeem: usize, + witness: usize, + executed_legacy: usize, + track_executed_legacy: bool, +} + +impl Accumulator { + pub(super) fn new(track_executed_legacy: bool) -> Self { + Self { + track_executed_legacy, + ..Self::default() + } + } + + pub(super) fn scan_input( + &mut self, + prev_kind: OutputType, + prev_legacy_sigops: SigOps, + facts: &input::Facts<'_>, + ) { + if !facts.script_sig.push_only { + self.legacy = self.legacy.saturating_add(facts.script_sig.legacy_sigops); + } + + if self.track_executed_legacy { + if !facts.script_sig.push_only { + self.executed_legacy = self.executed_legacy.saturating_add( + facts + .script_sig + .accurate_sigops + .saturating_mul(WITNESS_SCALE_FACTOR), + ); + } + if prev_kind != OutputType::P2SH { + self.executed_legacy = self + .executed_legacy + .saturating_add(u32::from(prev_legacy_sigops) as usize); + } + } + + match prev_kind { + OutputType::P2SH => { + let Some(redeem_sigops) = facts.redeem_sigops() else { + return; + }; + self.redeem = self.redeem.saturating_add(redeem_sigops); + if self.track_executed_legacy { + self.executed_legacy = self + .executed_legacy + .saturating_add(redeem_sigops.saturating_mul(WITNESS_SCALE_FACTOR)); + } + if !facts.script_sig.push_only { + return; + } + if facts.redeem_is_p2wpkh() { + self.witness = self.witness.saturating_add(1); + } else if facts.redeem_is_p2wsh() + && let Some(last) = facts.witness.last + { + self.witness = self + .witness + .saturating_add(Script::from_bytes(last).count_sigops()); + } + } + OutputType::P2WPKH => self.witness = self.witness.saturating_add(1), + OutputType::P2WSH => { + if let Some(last) = facts.witness.last { + self.witness = self + .witness + .saturating_add(Script::from_bytes(last).count_sigops()); + } + } + OutputType::P2TR => {} + _ => {} + } + } + + pub(super) fn scan_coinbase_input(&mut self, input: &TxIn) { + self.legacy = self + .legacy + .saturating_add(input.script_sig.count_sigops_legacy()); + } + + pub(super) fn scan_output(&mut self, txout: &TxOut, output: &ProcessedOutput) { + self.legacy = self + .legacy + .saturating_add(legacy_sigops_for_output(output, &txout.script_pubkey)); + } + + pub(super) fn finish(self) -> ComputedSigOps { + ComputedSigOps { + total: SigOps::from( + self.legacy + .saturating_mul(WITNESS_SCALE_FACTOR) + .saturating_add(self.redeem.saturating_mul(WITNESS_SCALE_FACTOR)) + .saturating_add(self.witness), + ), + executed_legacy: SigOps::from(self.executed_legacy), + } + } +} + +#[derive(Clone, Copy, Default)] +pub(super) struct ComputedSigOps { + pub(super) total: SigOps, + pub(super) executed_legacy: SigOps, +} + +/// Legacy sigop count of a script_pubkey, dispatched on `OutputType`. +/// Every variant except `OpReturn` and `Unknown` has a canonical shape +/// recognised by `OutputType::from`'s exact byte-pattern matchers, so +/// the legacy sigop count is fixed: P2PKH and P2PK both end in a +/// single OP_CHECKSIG (1), P2MS contains one OP_CHECKMULTISIG counted +/// as 20 in legacy mode, and P2SH/P2WPKH/P2WSH/P2TR/P2A/Empty contain +/// no CHECKSIG-class opcodes outside their pushdata. `OpReturn` +/// payloads can include 0xac/0xae bytes outside a push, and `Unknown` +/// can be anything, so both fall back to a real script walk. +#[inline] +fn legacy_sigops_for_output(output: &ProcessedOutput, script_pubkey: &Script) -> usize { + match output.output_type { + OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65 => 1, + OutputType::P2MS => 20, + OutputType::P2SH + | OutputType::P2WPKH + | OutputType::P2WSH + | OutputType::P2TR + | OutputType::P2A + | OutputType::Empty => 0, + OutputType::OpReturn => output.op_return_legacy_sigops(), + OutputType::Unknown => script_pubkey.count_sigops_legacy(), + } +} + +#[cfg(test)] +mod tests { + use bitcoin::{ScriptBuf, TxIn}; + use brk_types::{OutputType, SigOps}; + + use super::{Accumulator, input}; + use crate::TxFeatureFlags; + + #[test] + fn push_only_script_sig_keeps_prevout_executed_sigops() { + let input = TxIn { + script_sig: ScriptBuf::from_hex("00").unwrap(), + ..TxIn::default() + }; + let facts = input::analyze(&input, OutputType::P2PKH, &mut TxFeatureFlags::default()); + let mut accumulator = Accumulator::new(true); + + accumulator.scan_input(OutputType::P2PKH, SigOps::new(4), &facts); + + assert_eq!(u32::from(accumulator.finish().executed_legacy), 4); + } + + #[test] + fn counts_coinbase_legacy_script_sigops() { + let input = TxIn { + script_sig: ScriptBuf::from_hex("ac").unwrap(), + ..TxIn::default() + }; + let mut accumulator = Accumulator::new(false); + + accumulator.scan_coinbase_input(&input); + + assert_eq!(u32::from(accumulator.finish().total), 4); + } + + #[test] + fn counts_legacy_script_sigops_for_known_prevout_types() { + let input = TxIn { + script_sig: ScriptBuf::from_hex("ac").unwrap(), + ..TxIn::default() + }; + let facts = input::analyze(&input, OutputType::P2TR, &mut TxFeatureFlags::default()); + let mut accumulator = Accumulator::new(false); + + accumulator.scan_input(OutputType::P2TR, SigOps::ZERO, &facts); + + assert_eq!(u32::from(accumulator.finish().total), 4); + } +} diff --git a/crates/brk_indexer/src/processor/transaction/computed.rs b/crates/brk_indexer/src/processor/transaction/computed.rs new file mode 100644 index 000000000..a30786ba9 --- /dev/null +++ b/crates/brk_indexer/src/processor/transaction/computed.rs @@ -0,0 +1,82 @@ +use bitcoin::Transaction; +use brk_types::{TxIndex, Txid, TxidPrefix, Vout}; + +pub(crate) struct ComputedTx<'a> { + pub(crate) tx_index: TxIndex, + pub(crate) tx: &'a Transaction, + pub(crate) txid: Txid, + pub(crate) insert_txid_prefix: bool, + pub(crate) base_size: u32, + pub(crate) total_size: u32, + input_offset: usize, + output_offset: usize, +} + +impl ComputedTx<'_> { + pub(super) fn new( + tx_index: TxIndex, + tx: &Transaction, + txid: Txid, + insert_txid_prefix: bool, + base_size: u32, + total_size: u32, + ) -> ComputedTx<'_> { + ComputedTx { + tx_index, + tx, + txid, + insert_txid_prefix, + base_size, + total_size, + input_offset: 0, + output_offset: 0, + } + } + + pub(super) fn set_block_offsets(txs: &mut [Self]) { + let mut input_offset = 0; + let mut output_offset = 0; + + for tx in txs { + tx.input_offset = input_offset; + tx.output_offset = output_offset; + input_offset += tx.tx.input.len(); + output_offset += tx.tx.output.len(); + } + } + + #[inline] + pub(super) fn inputs<'a, T>(&self, inputs: &'a [T]) -> &'a [T] { + &inputs[self.input_offset..self.input_offset + self.tx.input.len()] + } + + #[inline] + pub(super) fn outputs<'a, T>(&self, outputs: &'a [T]) -> &'a [T] { + &outputs[self.output_offset..self.output_offset + self.tx.output.len()] + } + + #[inline] + pub(crate) fn txout_offset(&self, vout: Vout) -> usize { + self.output_offset + usize::from(vout) + } + + #[inline] + pub(crate) fn is_coinbase(&self, block_first_tx_index: TxIndex) -> bool { + self.tx_index == block_first_tx_index + } + + #[inline] + pub(crate) fn txid_prefix(&self) -> TxidPrefix { + TxidPrefix::from(&self.txid) + } + + #[inline] + pub(crate) fn is_segwit(&self) -> bool { + self.base_size != self.total_size + } + + #[inline] + pub(crate) fn weight(&self) -> bitcoin::Weight { + brk_types::Weight::from_sizes(self.base_size, self.total_size).into() + } +} diff --git a/crates/brk_indexer/src/processor/transaction/mod.rs b/crates/brk_indexer/src/processor/transaction/mod.rs new file mode 100644 index 000000000..7065308a3 --- /dev/null +++ b/crates/brk_indexer/src/processor/transaction/mod.rs @@ -0,0 +1,216 @@ +mod analysis; +mod computed; + +use brk_error::{Error, Result}; +use brk_store::Store; +use brk_types::{Height, StoredBool, TxIndex, TxVersion, Txid, TxidPrefix}; +use rayon::prelude::*; +use tracing::error; +use vecdb::{AnyVec, WritableVec, likely, unlikely}; + +use crate::constants::DUPLICATE_TXIDS; +use crate::{TransactionCounts, TransactionFeaturesVecs, TxMetadataVecs}; + +pub(super) use computed::ComputedTx; + +use self::analysis::TransactionAnalysis; +use super::{ + BlockProcessor, + txin::{self, InputSource}, + txout::{self, BlockAddresses, ProcessedOutput}, +}; + +impl<'a> BlockProcessor<'a> { + pub(crate) fn compute_txids(&self) -> Result>> { + let will_check_collisions = self.check_collisions; + let base_tx_index = self.lengths.tx_index; + + let mut txs = self + .block + .txdata + .par_iter() + .enumerate() + .map(|(index, tx)| { + let (btc_txid, base_size, total_size) = self.block.compute_tx_id_and_sizes(index); + let txid = Txid::from(btc_txid); + let tx_index = base_tx_index + TxIndex::from(index); + + let insert_txid_prefix = if likely(!will_check_collisions) { + true + } else { + let txid_prefix = TxidPrefix::from(&txid); + let prev_tx_index = self + .stores + .txid_prefix_to_tx_index + .get(&txid_prefix)? + .map(|value| *value); + + if let Some(prev_tx_index) = prev_tx_index { + self.validate_txid_collision(tx_index, prev_tx_index)?; + false + } else { + true + } + }; + + Ok(ComputedTx::new( + tx_index, + tx, + txid, + insert_txid_prefix, + base_size, + total_size, + )) + }) + .collect::>>()?; + + ComputedTx::set_block_offsets(&mut txs); + Ok(txs) + } + + fn validate_txid_collision(&self, tx_index: TxIndex, prev_tx_index: TxIndex) -> Result<()> { + if tx_index == prev_tx_index { + return Ok(()); + } + + let len = self.vecs.transactions.txid.len(); + let prev_txid = self + .vecs + .transactions + .txid + .get_pushed_or_read(prev_tx_index, &self.readers.txid) + .ok_or(Error::Internal("Missing txid for tx_index")) + .inspect_err(|_| { + error!(?tx_index, len, "Missing txid for tx_index"); + })?; + + if unlikely(!DUPLICATE_TXIDS.contains(&prev_txid)) { + error!( + height = ?self.height, + ?tx_index, + ?prev_txid, + ?prev_tx_index, + "Unexpected TXID collision" + ); + return Err(Error::Internal("Unexpected TXID collision")); + } + + Ok(()) + } + + /// Analyzes transactions, then finalizes outputs/inputs in parallel with metadata storage. + #[allow(clippy::too_many_arguments)] + pub(crate) fn analyze_and_finalize_transactions( + &mut self, + txs: Vec, + mut txouts: Vec, + txins: Vec, + addresses: &mut BlockAddresses, + ) -> Result<()> { + let transaction_analyses = self.analyze_transactions(&txs, &txins, &txouts); + let lengths = &mut *self.lengths; + let base_tx_index = lengths.tx_index; + let base_txin_index = lengths.txin_index; + let transactions = &self.block.txdata; + + // Split transactions vecs: finalize needs first_txout_index/first_txin_index, metadata needs the rest + let (first_txout_index, first_txin_index, mut tx_metadata) = + self.vecs.transactions.split_for_finalize(); + + let outputs = &mut self.vecs.outputs; + let inputs = &mut self.vecs.inputs; + let addrs = &mut self.vecs.addrs; + let scripts = &mut self.vecs.scripts; + let op_return = &mut self.vecs.op_return; + let transaction_features = &mut self.vecs.transaction_features; + let height = self.height; + + let addr_hash_stores = &mut self.stores.addr_type_to_addr_hash_to_addr_index; + let addr_tx_index_stores = &mut self.stores.addr_type_to_addr_index_and_tx_index; + let addr_outpoint_stores = &mut self.stores.addr_type_to_addr_index_and_unspent_outpoint; + let txid_prefix_store = &mut self.stores.txid_prefix_to_tx_index; + + let (finalize_result, metadata_result) = rayon::join( + || -> Result<()> { + txout::finalize_outputs( + transactions, + base_tx_index, + lengths, + first_txout_index, + outputs, + addrs, + scripts, + op_return, + addr_hash_stores, + addr_tx_index_stores, + addr_outpoint_stores, + &mut txouts, + addresses, + )?; + txin::finalize_inputs( + transactions, + base_tx_index, + base_txin_index, + first_txin_index, + inputs, + addr_tx_index_stores, + addr_outpoint_stores, + txins, + &txouts, + ) + }, + || { + store_tx_metadata( + height, + txs, + transaction_analyses, + txid_prefix_store, + &mut tx_metadata, + transaction_features, + ) + }, + ); + + finalize_result?; + metadata_result?; + Ok(()) + } +} + +fn store_tx_metadata( + height: Height, + txs: Vec, + transaction_analyses: Vec, + store: &mut Store, + md: &mut TxMetadataVecs<'_>, + features: &mut TransactionFeaturesVecs, +) -> Result<()> { + debug_assert_eq!(txs.len(), transaction_analyses.len()); + let mut counts = TransactionCounts::default(); + for (ct, analysis) in txs.into_iter().zip(transaction_analyses) { + if likely(ct.insert_txid_prefix) { + store.insert(ct.txid_prefix(), ct.tx_index); + } + let tx_version = TxVersion::from(ct.tx.version); + md.tx_version.checked_push(ct.tx_index, tx_version)?; + md.txid.checked_push(ct.tx_index, ct.txid)?; + md.raw_locktime + .checked_push(ct.tx_index, ct.tx.lock_time.into())?; + md.weight.checked_push(ct.tx_index, ct.weight().into())?; + md.total_size + .checked_push(ct.tx_index, ct.total_size.into())?; + md.total_sigop_cost + .checked_push(ct.tx_index, analysis.total_sigop_cost)?; + md.is_explicitly_rbf + .checked_push(ct.tx_index, StoredBool::from(analysis.explicitly_rbf))?; + counts.add_base( + ct.tx.input.len(), + ct.tx.output.len(), + tx_version, + analysis.explicitly_rbf, + ); + features.push_and_count(analysis.features, &mut counts); + } + features.count.checked_push(height, counts)?; + Ok(()) +} diff --git a/crates/brk_indexer/src/processor/tx.rs b/crates/brk_indexer/src/processor/tx.rs deleted file mode 100644 index c3174b635..000000000 --- a/crates/brk_indexer/src/processor/tx.rs +++ /dev/null @@ -1,116 +0,0 @@ -use brk_error::{Error, Result}; -use brk_store::Store; -use brk_types::{SigOps, StoredBool, TxIndex, Txid, TxidPrefix}; -use rayon::prelude::*; -use tracing::error; -use vecdb::{AnyVec, WritableVec, likely}; - -use crate::TxMetadataVecs; -use crate::constants::DUPLICATE_TXIDS; - -use super::{BlockProcessor, ComputedTx}; - -impl<'a> BlockProcessor<'a> { - pub fn compute_txids(&self) -> Result>> { - let will_check_collisions = self.check_collisions; - let base_tx_index = self.lengths.tx_index; - - self.block - .txdata - .par_iter() - .enumerate() - .map(|(index, tx)| { - let (btc_txid, base_size, total_size) = self.block.compute_tx_id_and_sizes(index); - let txid = Txid::from(btc_txid); - let txid_prefix = TxidPrefix::from(&txid); - - let prev_tx_index_opt = if will_check_collisions { - self.stores - .txid_prefix_to_tx_index - .get(&txid_prefix)? - .map(|v| *v) - } else { - None - }; - - Ok(ComputedTx { - tx_index: base_tx_index + TxIndex::from(index), - tx, - txid, - txid_prefix, - prev_tx_index_opt, - base_size, - total_size, - }) - }) - .collect() - } - - /// Only for known duplicate TXIDs (BIP-30). - pub fn check_txid_collisions(&self, txs: &[ComputedTx]) -> Result<()> { - if likely(!self.check_collisions) { - return Ok(()); - } - - for ct in txs.iter() { - let Some(prev_tx_index) = ct.prev_tx_index_opt else { - continue; - }; - - if ct.tx_index == prev_tx_index { - continue; - } - - let len = self.vecs.transactions.txid.len(); - let prev_txid = self - .vecs - .transactions - .txid - .get_pushed_or_read(prev_tx_index, &self.readers.txid) - .ok_or(Error::Internal("Missing txid for tx_index")) - .inspect_err(|_| { - error!(tx_index = ?ct.tx_index, len, "Missing txid for tx_index"); - })?; - - let is_dup = DUPLICATE_TXIDS.contains(&prev_txid); - - if !is_dup { - error!( - height = ?self.height, tx_index = ?ct.tx_index, - ?prev_txid, ?prev_tx_index, - "Unexpected TXID collision" - ); - return Err(Error::Internal("Unexpected TXID collision")); - } - } - - Ok(()) - } -} - -pub(super) fn store_tx_metadata( - txs: Vec, - sigops: Vec, - store: &mut Store, - md: &mut TxMetadataVecs<'_>, -) -> Result<()> { - debug_assert_eq!(txs.len(), sigops.len()); - for (ct, sigops) in txs.into_iter().zip(sigops) { - if ct.prev_tx_index_opt.is_none() { - store.insert(ct.txid_prefix, ct.tx_index); - } - md.tx_version - .checked_push(ct.tx_index, ct.tx.version.into())?; - md.txid.checked_push(ct.tx_index, ct.txid)?; - md.raw_locktime - .checked_push(ct.tx_index, ct.tx.lock_time.into())?; - md.base_size - .checked_push(ct.tx_index, ct.base_size.into())?; - md.total_size - .checked_push(ct.tx_index, ct.total_size.into())?; - md.total_sigop_cost.checked_push(ct.tx_index, sigops)?; - md.is_explicitly_rbf - .checked_push(ct.tx_index, StoredBool::from(ct.tx.is_explicitly_rbf()))?; - } - Ok(()) -} diff --git a/crates/brk_indexer/src/processor/txin.rs b/crates/brk_indexer/src/processor/txin.rs deleted file mode 100644 index c1eb39504..000000000 --- a/crates/brk_indexer/src/processor/txin.rs +++ /dev/null @@ -1,222 +0,0 @@ -use brk_cohort::ByAddrType; -use brk_error::{Error, Result}; -use brk_store::Store; -use brk_types::{ - AddrIndexOutPoint, AddrIndexTxIndex, OutPoint, OutputType, TxInIndex, TxIndex, Txid, - TxidPrefix, TypeIndex, Unit, Vin, Vout, -}; -use rayon::prelude::*; -use rustc_hash::{FxHashMap, FxHashSet}; -use tracing::error; -use vecdb::{PcoVec, WritableVec}; - -use super::{BlockProcessor, ComputedTx, InputSource, SameBlockOutputInfo}; -use crate::InputsVecs; - -impl<'a> BlockProcessor<'a> { - pub fn process_inputs( - &self, - txs: &[ComputedTx], - txid_prefix_to_tx_index: &mut FxHashMap, - ) -> Result> { - txid_prefix_to_tx_index.clear(); - txid_prefix_to_tx_index.extend(txs.iter().map(|ct| (ct.txid_prefix, ct.tx_index))); - - let base_tx_index = self.lengths.tx_index; - let base_txin_index = self.lengths.txin_index; - - let total_inputs: usize = self.block.txdata.iter().map(|tx| tx.input.len()).sum(); - let mut items = Vec::with_capacity(total_inputs); - for (index, tx) in self.block.txdata.iter().enumerate() { - for (vin, txin) in tx.input.iter().enumerate() { - items.push((TxIndex::from(index), Vin::from(vin), txin, tx)); - } - } - - let txid_prefix_to_tx_index = &*txid_prefix_to_tx_index; - - let txins = items - .into_par_iter() - .enumerate() - .map( - |(block_txin_index, (block_tx_index, vin, txin, tx))| -> Result<(TxInIndex, InputSource)> { - let tx_index = base_tx_index + block_tx_index; - let txin_index = base_txin_index + TxInIndex::from(block_txin_index); - - if tx.is_coinbase() { - return Ok(( - txin_index, - InputSource::SameBlock { - tx_index, - vin, - outpoint: OutPoint::COINBASE, - }, - )); - } - - let outpoint = txin.previous_output; - let txid = Txid::from(outpoint.txid); - let txid_prefix = TxidPrefix::from(&txid); - let vout = Vout::from(outpoint.vout); - - if let Some(&same_block_tx_index) = txid_prefix_to_tx_index - .get(&txid_prefix) { - let outpoint = OutPoint::new(same_block_tx_index, vout); - return Ok(( - txin_index, - InputSource::SameBlock { - tx_index, - vin, - outpoint, - }, - )); - } - - let store_result = self - .stores - .txid_prefix_to_tx_index - .get(&txid_prefix)? - .map(|v| *v); - - let prev_tx_index = match store_result { - Some(tx_index) if tx_index < self.lengths.tx_index => tx_index, - _ => { - error!( - "UnknownTxid: txid={}, prefix={:?}, store_result={:?}, current_tx_index={:?}", - txid, txid_prefix, store_result, self.lengths.tx_index - ); - return Err(Error::UnknownTxid); - } - }; - - let txout_index = self - .vecs - .transactions - .first_txout_index - .get_pushed_or_read(prev_tx_index, &self.readers.tx_index_to_first_txout_index) - .ok_or(Error::Internal("Missing txout_index"))? - + vout; - - let outpoint = OutPoint::new(prev_tx_index, vout); - - let output_type = self - .vecs - .outputs - .output_type - .get_pushed_or_read(txout_index, &self.readers.txout_index_to_output_type) - .ok_or(Error::Internal("Missing output_type"))?; - - let type_index = self - .vecs - .outputs - .type_index - .get_pushed_or_read(txout_index, &self.readers.txout_index_to_type_index) - .ok_or(Error::Internal("Missing type_index"))?; - - Ok(( - txin_index, - InputSource::PreviousBlock { - vin, - tx_index, - outpoint, - output_type, - type_index, - }, - )) - }, - ) - .collect::>>()?; - - Ok(txins) - } - - pub fn collect_same_block_spent_outpoints( - txins: &[(TxInIndex, InputSource)], - out: &mut FxHashSet, - ) { - out.clear(); - out.extend( - txins - .iter() - .filter_map(|(_, input_source)| match input_source { - InputSource::SameBlock { outpoint, .. } if !outpoint.is_coinbase() => { - Some(*outpoint) - } - _ => None, - }), - ); - } -} - -pub(super) fn finalize_inputs( - first_txin_index: &mut PcoVec, - inputs: &mut InputsVecs, - addr_tx_index_stores: &mut ByAddrType>, - addr_outpoint_stores: &mut ByAddrType>, - txins: Vec<(TxInIndex, InputSource)>, - same_block_output_info: &mut FxHashMap, -) -> Result<()> { - for (txin_index, input_source) in txins { - let (vin, tx_index, outpoint, output_type, type_index) = match input_source { - InputSource::PreviousBlock { - vin, - tx_index, - outpoint, - output_type, - type_index, - } => (vin, tx_index, outpoint, output_type, type_index), - InputSource::SameBlock { - tx_index, - vin, - outpoint, - } => { - if outpoint.is_coinbase() { - ( - vin, - tx_index, - outpoint, - OutputType::Unknown, - TypeIndex::COINBASE, - ) - } else { - let info = same_block_output_info - .remove(&outpoint) - .ok_or(Error::Internal("Same-block output not found")) - .inspect_err(|_| { - error!( - ?outpoint, - remaining = same_block_output_info.len(), - "Same-block output not found" - ); - })?; - (vin, tx_index, outpoint, info.output_type, info.type_index) - } - } - }; - - if vin.is_zero() { - first_txin_index.checked_push(tx_index, txin_index)?; - } - - inputs.tx_index.checked_push(txin_index, tx_index)?; - inputs.outpoint.checked_push(txin_index, outpoint)?; - inputs.output_type.checked_push(txin_index, output_type)?; - inputs.type_index.checked_push(txin_index, type_index)?; - - if !output_type.is_addr() { - continue; - } - let addr_type = output_type; - let addr_index = type_index; - - addr_tx_index_stores - .get_mut_unwrap(addr_type) - .insert(AddrIndexTxIndex::from((addr_index, tx_index)), Unit); - - addr_outpoint_stores - .get_mut_unwrap(addr_type) - .remove(AddrIndexOutPoint::from((addr_index, outpoint))); - } - - Ok(()) -} diff --git a/crates/brk_indexer/src/processor/txin/mod.rs b/crates/brk_indexer/src/processor/txin/mod.rs new file mode 100644 index 000000000..742ace26d --- /dev/null +++ b/crates/brk_indexer/src/processor/txin/mod.rs @@ -0,0 +1,98 @@ +mod resolver; +mod source; + +pub(crate) use resolver::InputResolver; +pub(super) use source::InputSource; + +use bitcoin::Transaction; +use brk_cohort::ByAddrType; +use brk_error::Result; +use brk_store::Store; +use brk_types::{ + AddrIndexOutPoint, AddrIndexTxIndex, OutPoint, OutputType, TxInIndex, TxIndex, TypeIndex, Unit, + Vin, +}; +use vecdb::{PcoVec, WritableVec, unlikely}; + +use super::{BlockProcessor, transaction::ComputedTx, txout::ProcessedOutput}; +use crate::InputsVecs; + +impl<'a> BlockProcessor<'a> { + pub(crate) fn process_inputs( + &self, + txs: &[ComputedTx], + resolver: &mut InputResolver, + ) -> Result> { + resolver.resolve(self, txs) + } +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn finalize_inputs( + transactions: &[Transaction], + base_tx_index: TxIndex, + base_txin_index: TxInIndex, + first_txin_index: &mut PcoVec, + inputs: &mut InputsVecs, + addr_tx_index_stores: &mut ByAddrType>, + addr_outpoint_stores: &mut ByAddrType>, + txins: Vec, + txouts: &[ProcessedOutput], +) -> Result<()> { + let mut input_offset = 0; + for (block_tx_index, tx) in transactions.iter().enumerate() { + let tx_index = base_tx_index + TxIndex::from(block_tx_index); + let next_input_offset = input_offset + tx.input.len(); + + for (vin, input_source) in txins[input_offset..next_input_offset].iter().enumerate() { + let block_txin_index = input_offset + vin; + let txin_index = base_txin_index + TxInIndex::from(block_txin_index); + let vin = Vin::from(vin); + let (outpoint, output_type, type_index) = match input_source { + InputSource::PreviousBlock { + outpoint, + output_type, + legacy_sigops: _, + type_index, + } => (*outpoint, *output_type, *type_index), + InputSource::Coinbase => { + (OutPoint::COINBASE, OutputType::Unknown, TypeIndex::COINBASE) + } + InputSource::SameBlock { + outpoint, + txout_offset, + } => { + let output = &txouts[*txout_offset]; + (*outpoint, output.output_type, output.resolved_type_index()) + } + }; + + if vin.is_zero() { + first_txin_index.checked_push(tx_index, txin_index)?; + } + + inputs.tx_index.checked_push(txin_index, tx_index)?; + inputs.outpoint.checked_push(txin_index, outpoint)?; + inputs.output_type.checked_push(txin_index, output_type)?; + inputs.type_index.checked_push(txin_index, type_index)?; + + if unlikely(!output_type.is_addr()) { + continue; + } + let addr_type = output_type; + let addr_index = type_index; + + addr_tx_index_stores + .get_mut_unwrap(addr_type) + .insert(AddrIndexTxIndex::from((addr_index, tx_index)), Unit); + + addr_outpoint_stores + .get_mut_unwrap(addr_type) + .remove(AddrIndexOutPoint::from((addr_index, outpoint))); + } + + input_offset = next_input_offset; + } + + Ok(()) +} diff --git a/crates/brk_indexer/src/processor/txin/resolver.rs b/crates/brk_indexer/src/processor/txin/resolver.rs new file mode 100644 index 000000000..d041d0ec7 --- /dev/null +++ b/crates/brk_indexer/src/processor/txin/resolver.rs @@ -0,0 +1,205 @@ +use std::collections::hash_map::Entry; + +use brk_error::{Error, Result}; +use brk_types::{OutPoint, SigOps, TxIndex, TxOutIndex, Txid, TxidPrefix, Vout}; +use rayon::prelude::*; +use rustc_hash::FxHashMap; +use tracing::error; +use vecdb::unlikely; + +use super::InputSource; +use crate::processor::{BlockProcessor, transaction::ComputedTx}; + +#[derive(Default)] +pub(crate) struct InputResolver { + same_block_transactions: FxHashMap, + previous_parent_indexes: FxHashMap, + previous_parents: Vec, + inputs: Vec, +} + +impl InputResolver { + pub(crate) fn resolve( + &mut self, + processor: &BlockProcessor<'_>, + txs: &[ComputedTx<'_>], + ) -> Result> { + self.prepare(txs, processor.lengths.tx_index); + + let current_tx_index = processor.lengths.tx_index; + let parents = self + .previous_parents + .par_iter() + .map(|parent| { + let store_result = processor + .stores + .txid_prefix_to_tx_index + .get(&parent.txid_prefix)? + .map(|value| *value); + + let tx_index = match store_result { + Some(tx_index) if tx_index < current_tx_index => tx_index, + _ => { + error!( + "UnknownTxid: txid={}, prefix={:?}, store_result={:?}, current_tx_index={:?}", + parent.txid, + parent.txid_prefix, + store_result, + current_tx_index + ); + return Err(Error::UnknownTxid); + } + }; + + let first_txout_index = processor + .vecs + .transactions + .first_txout_index + .get_pushed_or_read( + tx_index, + &processor.readers.tx_index_to_first_txout_index, + ) + .ok_or(Error::Internal("Missing txout_index"))?; + + Ok(ResolvedParent { + tx_index, + first_txout_index, + }) + }) + .collect::>>()?; + + let tracks_executed_legacy_sigops = processor.tracks_executed_legacy_sigops(); + + self.inputs + .par_iter() + .map(|input| match *input { + UnresolvedInput::Coinbase => Ok(InputSource::Coinbase), + UnresolvedInput::SameBlock { + outpoint, + txout_offset, + } => Ok(InputSource::SameBlock { + outpoint, + txout_offset, + }), + UnresolvedInput::PreviousBlock { parent_index, vout } => { + let parent = parents[parent_index]; + let txout_index = parent.first_txout_index + vout; + let outpoint = OutPoint::new(parent.tx_index, vout); + + let output_type = processor + .vecs + .outputs + .output_type + .get_pushed_or_read( + txout_index, + &processor.readers.txout_index_to_output_type, + ) + .ok_or(Error::Internal("Missing output_type"))?; + + let type_index = processor + .vecs + .outputs + .type_index + .get_pushed_or_read( + txout_index, + &processor.readers.txout_index_to_type_index, + ) + .ok_or(Error::Internal("Missing type_index"))?; + + let legacy_sigops = if tracks_executed_legacy_sigops { + processor + .vecs + .scripts + .legacy_sigops(output_type, type_index, &processor.readers.scripts) + .ok_or(Error::Internal("Missing legacy_sigops"))? + } else { + SigOps::ZERO + }; + + Ok(InputSource::PreviousBlock { + outpoint, + output_type, + legacy_sigops, + type_index, + }) + } + }) + .collect() + } + + fn prepare(&mut self, txs: &[ComputedTx<'_>], block_first_tx_index: TxIndex) { + self.same_block_transactions.clear(); + self.previous_parent_indexes.clear(); + self.previous_parents.clear(); + self.inputs.clear(); + + self.same_block_transactions.reserve(txs.len()); + self.same_block_transactions + .extend(txs.iter().map(|tx| (tx.txid_prefix(), tx.tx_index))); + + let total_inputs = txs.iter().map(|tx| tx.tx.input.len()).sum(); + self.inputs.reserve(total_inputs); + + for tx in txs { + for txin in &tx.tx.input { + let previous_output = &txin.previous_output; + if unlikely(previous_output.is_null()) { + self.inputs.push(UnresolvedInput::Coinbase); + continue; + } + + let txid = *<&Txid>::from(&previous_output.txid); + let txid_prefix = TxidPrefix::from(&txid); + let vout = Vout::from(previous_output.vout); + + if let Some(&tx_index) = self.same_block_transactions.get(&txid_prefix) { + let block_tx_index = usize::from(tx_index) - usize::from(block_first_tx_index); + self.inputs.push(UnresolvedInput::SameBlock { + outpoint: OutPoint::new(tx_index, vout), + txout_offset: txs[block_tx_index].txout_offset(vout), + }); + continue; + } + + let parent_index = match self.previous_parent_indexes.entry(txid_prefix) { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(entry) => { + let parent_index = self.previous_parents.len(); + entry.insert(parent_index); + self.previous_parents + .push(PreviousParent { txid, txid_prefix }); + parent_index + } + }; + + self.inputs + .push(UnresolvedInput::PreviousBlock { parent_index, vout }); + } + } + } +} + +#[derive(Clone, Copy)] +struct PreviousParent { + txid: Txid, + txid_prefix: TxidPrefix, +} + +#[derive(Clone, Copy)] +struct ResolvedParent { + tx_index: TxIndex, + first_txout_index: TxOutIndex, +} + +#[derive(Clone, Copy)] +enum UnresolvedInput { + Coinbase, + PreviousBlock { + parent_index: usize, + vout: Vout, + }, + SameBlock { + outpoint: OutPoint, + txout_offset: usize, + }, +} diff --git a/crates/brk_indexer/src/processor/txin/source.rs b/crates/brk_indexer/src/processor/txin/source.rs new file mode 100644 index 000000000..87fdecd17 --- /dev/null +++ b/crates/brk_indexer/src/processor/txin/source.rs @@ -0,0 +1,16 @@ +use brk_types::{OutPoint, OutputType, SigOps, TypeIndex}; + +#[derive(Debug)] +pub(crate) enum InputSource { + Coinbase, + PreviousBlock { + outpoint: OutPoint, + output_type: OutputType, + legacy_sigops: SigOps, + type_index: TypeIndex, + }, + SameBlock { + outpoint: OutPoint, + txout_offset: usize, + }, +} diff --git a/crates/brk_indexer/src/processor/txout.rs b/crates/brk_indexer/src/processor/txout.rs deleted file mode 100644 index 335d06826..000000000 --- a/crates/brk_indexer/src/processor/txout.rs +++ /dev/null @@ -1,235 +0,0 @@ -use brk_cohort::ByAddrType; -use brk_error::{Error, Result}; -use brk_store::Store; -use brk_types::{ - AddrBytes, AddrHash, AddrIndexOutPoint, AddrIndexTxIndex, OutPoint, OutputType, Sats, TxIndex, - TxOutIndex, TypeIndex, Unit, Vout, -}; -use rayon::prelude::*; -use rustc_hash::{FxHashMap, FxHashSet}; -use tracing::error; -use vecdb::{BytesVec, WritableVec}; - -use super::{BlockProcessor, ProcessedOutput, SameBlockOutputInfo}; -use crate::{AddrsVecs, Lengths, OutputsVecs, ScriptsVecs}; - -impl<'a> BlockProcessor<'a> { - pub fn process_outputs(&self) -> Result>> { - let height = self.height; - let check_collisions = self.check_collisions; - - let base_tx_index = self.lengths.tx_index; - let base_txout_index = self.lengths.txout_index; - - let total_outputs: usize = self.block.txdata.iter().map(|tx| tx.output.len()).sum(); - let mut items = Vec::with_capacity(total_outputs); - for (index, tx) in self.block.txdata.iter().enumerate() { - for (vout, txout) in tx.output.iter().enumerate() { - items.push((TxIndex::from(index), Vout::from(vout), txout)); - } - } - - items - .into_par_iter() - .enumerate() - .map( - |(block_txout_index, (block_tx_index, vout, txout))| -> Result { - let tx_index = base_tx_index + block_tx_index; - let txout_index = base_txout_index + TxOutIndex::from(block_txout_index); - - let script = &txout.script_pubkey; - let output_type = OutputType::from(script); - - if output_type.is_not_addr() { - return Ok(ProcessedOutput { - txout_index, - txout, - tx_index, - vout, - output_type, - addr_info: None, - existing_type_index: None, - }); - } - - let addr_type = output_type; - let addr_bytes = AddrBytes::try_from((script, addr_type)).unwrap(); - let addr_hash = AddrHash::from(&addr_bytes); - - let existing_type_index = self - .stores - .addr_type_to_addr_hash_to_addr_index - .get_unwrap(addr_type) - .get(&addr_hash)? - .map(|v| *v) - .and_then(|type_index_local| { - (type_index_local < self.lengths.to_type_index(addr_type)) - .then_some(type_index_local) - }); - - if check_collisions && let Some(type_index) = existing_type_index { - let prev_addrbytes = self - .vecs - .addrs - .get_bytes_by_type(addr_type, type_index, &self.readers.addrbytes) - .ok_or(Error::Internal("Missing addrbytes"))?; - - if prev_addrbytes != addr_bytes { - error!( - ?height, - ?vout, - ?block_tx_index, - ?addr_type, - ?prev_addrbytes, - ?addr_bytes, - ?type_index, - "Address hash collision" - ); - return Err(Error::Internal("Address hash collision")); - } - } - - Ok(ProcessedOutput { - txout_index, - txout, - tx_index, - vout, - output_type, - addr_info: Some((addr_bytes, addr_hash)), - existing_type_index, - }) - }, - ) - .collect() - } -} - -#[allow(clippy::too_many_arguments)] -pub(super) fn finalize_outputs( - lengths: &mut Lengths, - first_txout_index: &mut BytesVec, - outputs: &mut OutputsVecs, - addrs: &mut AddrsVecs, - scripts: &mut ScriptsVecs, - addr_hash_stores: &mut ByAddrType>, - addr_tx_index_stores: &mut ByAddrType>, - addr_outpoint_stores: &mut ByAddrType>, - txouts: Vec, - same_block_spent_outpoints: &FxHashSet, - already_added_addr_hash: &mut ByAddrType>, - same_block_output_info: &mut FxHashMap, -) -> Result<()> { - already_added_addr_hash.values_mut().for_each(|m| m.clear()); - same_block_output_info.clear(); - - for ProcessedOutput { - txout_index, - txout, - tx_index, - vout, - output_type, - addr_info, - existing_type_index, - } in txouts - { - let sats = Sats::from(txout.value); - - if vout.is_zero() { - first_txout_index.checked_push(tx_index, txout_index)?; - } - - outputs.tx_index.checked_push(txout_index, tx_index)?; - - let type_index = if let Some(ti) = existing_type_index { - ti - } else if let Some((addr_bytes, addr_hash)) = addr_info { - let addr_type = output_type; - if let Some(&ti) = already_added_addr_hash - .get_unwrap(addr_type) - .get(&addr_hash) - { - ti - } else { - let ti = lengths.increment_addr_index(addr_type); - - already_added_addr_hash - .get_mut_unwrap(addr_type) - .insert(addr_hash, ti); - addr_hash_stores - .get_mut_unwrap(addr_type) - .insert(addr_hash, ti); - addrs.push_bytes_if_needed(ti, addr_bytes)?; - - ti - } - } else { - match output_type { - OutputType::P2MS => { - scripts - .p2ms - .to_tx_index - .checked_push(lengths.p2ms_output_index, tx_index)?; - lengths.p2ms_output_index.copy_then_increment() - } - OutputType::OpReturn => { - scripts - .op_return - .to_tx_index - .checked_push(lengths.op_return_index, tx_index)?; - lengths.op_return_index.copy_then_increment() - } - OutputType::Empty => { - scripts - .empty - .to_tx_index - .checked_push(lengths.empty_output_index, tx_index)?; - lengths.empty_output_index.copy_then_increment() - } - OutputType::Unknown => { - scripts - .unknown - .to_tx_index - .checked_push(lengths.unknown_output_index, tx_index)?; - lengths.unknown_output_index.copy_then_increment() - } - _ => unreachable!(), - } - }; - - outputs.value.checked_push(txout_index, sats)?; - outputs.output_type.checked_push(txout_index, output_type)?; - outputs.type_index.checked_push(txout_index, type_index)?; - - if output_type.is_unspendable() { - continue; - } else if output_type.is_addr() { - let addr_type = output_type; - let addr_index = type_index; - - addr_tx_index_stores - .get_mut_unwrap(addr_type) - .insert(AddrIndexTxIndex::from((addr_index, tx_index)), Unit); - } - - let outpoint = OutPoint::new(tx_index, vout); - - if same_block_spent_outpoints.contains(&outpoint) { - same_block_output_info.insert( - outpoint, - SameBlockOutputInfo { - output_type, - type_index, - }, - ); - } else if output_type.is_addr() { - let addr_type = output_type; - let addr_index = type_index; - - addr_outpoint_stores - .get_mut_unwrap(addr_type) - .insert(AddrIndexOutPoint::from((addr_index, outpoint)), Unit); - } - } - - Ok(()) -} diff --git a/crates/brk_indexer/src/processor/txout/address.rs b/crates/brk_indexer/src/processor/txout/address.rs new file mode 100644 index 000000000..c613fb7ad --- /dev/null +++ b/crates/brk_indexer/src/processor/txout/address.rs @@ -0,0 +1,172 @@ +mod cache; + +use std::collections::hash_map::Entry; + +use brk_cohort::ByAddrType; +use brk_error::{Error, Result}; +use brk_types::{AddrBytes, AddrHash, OutputType, TxIndex, TypeIndex, Vout}; +use rayon::prelude::*; +use rustc_hash::FxHashMap; +use tracing::error; +use vecdb::likely; + +use self::cache::AddressCache; +use super::{ProcessedOutput, processed::ProcessedOutputData}; +use crate::processor::BlockProcessor; + +struct Lookup { + index: usize, + output_type: OutputType, + hash: AddrHash, + type_index: Option, +} + +#[derive(Default)] +pub(crate) struct BlockAddresses { + cache: AddressCache, + indexes: ByAddrType>, + lookups: Vec, + unique: Vec<(OutputType, AddrHash)>, + resolved: Vec>, +} + +impl BlockAddresses { + pub(super) fn resolve( + &mut self, + processor: &BlockProcessor, + outputs: &[ProcessedOutput], + ) -> Result<()> { + self.clear_block(); + + for output in outputs { + let ProcessedOutputData::Address(addr_hash) = &output.data else { + continue; + }; + + if let Entry::Vacant(entry) = self + .indexes + .get_mut_unwrap(output.output_type) + .entry(*addr_hash) + { + entry.insert(self.unique.len()); + self.unique.push((output.output_type, *addr_hash)); + } + } + + self.resolved.resize(self.unique.len(), None); + + for (index, &(output_type, hash)) in self.unique.iter().enumerate() { + if let Some(type_index) = self.cache.get(output_type, hash) { + self.resolved[index] = Some(type_index); + } else { + self.lookups.push(Lookup { + index, + output_type, + hash, + type_index: None, + }); + } + } + + let stores = &processor.stores.addr_type_to_addr_hash_to_addr_index; + let lengths = &*processor.lengths; + + self.lookups + .par_iter_mut() + .try_for_each(|lookup| -> Result<()> { + lookup.type_index = stores + .get_unwrap(lookup.output_type) + .get(&lookup.hash)? + .map(|type_index| *type_index) + .filter(|type_index| *type_index < lengths.to_type_index(lookup.output_type)); + Ok(()) + })?; + + for lookup in &self.lookups { + self.resolved[lookup.index] = lookup.type_index; + if let Some(type_index) = lookup.type_index { + self.cache + .insert(lookup.output_type, lookup.hash, type_index); + } + } + + if likely(!processor.check_collisions) { + return Ok(()); + } + + let mut output_offset = 0; + for (block_tx_index, tx) in processor.block.txdata.iter().enumerate() { + let tx_index = processor.lengths.tx_index + TxIndex::from(block_tx_index); + let next_output_offset = output_offset + tx.output.len(); + + for (vout, (txout, output)) in tx + .output + .iter() + .zip(&outputs[output_offset..next_output_offset]) + .enumerate() + { + let ProcessedOutputData::Address(addr_hash) = &output.data else { + continue; + }; + let Some(type_index) = self.index(output.output_type, addr_hash) else { + continue; + }; + let addr_bytes = + AddrBytes::try_from((&txout.script_pubkey, output.output_type)).unwrap(); + + let prev_addrbytes = processor + .vecs + .addrs + .get_bytes_by_type(output.output_type, type_index, &processor.readers.addrbytes) + .ok_or(Error::Internal("Missing addrbytes"))?; + + if prev_addrbytes != addr_bytes { + error!( + height = ?processor.height, + vout = ?Vout::from(vout), + ?tx_index, + addr_type = ?output.output_type, + ?prev_addrbytes, + ?addr_bytes, + ?type_index, + "Address hash collision" + ); + return Err(Error::Internal("Address hash collision")); + } + } + + output_offset = next_output_offset; + } + + Ok(()) + } + + pub(super) fn index_mut( + &mut self, + addr_type: OutputType, + addr_hash: &AddrHash, + ) -> &mut Option { + let index = *self + .indexes + .get_mut_unwrap(addr_type) + .get_mut(addr_hash) + .unwrap(); + &mut self.resolved[index] + } + + fn index(&self, addr_type: OutputType, addr_hash: &AddrHash) -> Option { + let index = *self.indexes.get_unwrap(addr_type).get(addr_hash).unwrap(); + self.resolved[index] + } + + pub(crate) fn clear_cache(&mut self) { + self.cache.clear(); + } + + fn clear_block(&mut self) { + self.indexes.values_mut().for_each(FxHashMap::clear); + self.lookups.clear(); + self.unique.clear(); + self.resolved.clear(); + } +} diff --git a/crates/brk_indexer/src/processor/txout/address/cache.rs b/crates/brk_indexer/src/processor/txout/address/cache.rs new file mode 100644 index 000000000..83dfc9600 --- /dev/null +++ b/crates/brk_indexer/src/processor/txout/address/cache.rs @@ -0,0 +1,140 @@ +use std::mem; + +use brk_types::{AddrHash, OutputType, TypeIndex}; + +const ASSOCIATIVITY: usize = 4; +const SET_COUNT: usize = 1 << 18; +const SET_MASK: usize = SET_COUNT - 1; +const TYPE_MIX: u64 = 0x9e37_79b9_7f4a_7c15; +const REFERENCE_SHIFT: usize = 2; + +#[derive(Clone, Copy)] +#[repr(C)] +struct Entry { + hash: AddrHash, + type_index: TypeIndex, + output_type: OutputType, +} + +impl Entry { + const EMPTY: Self = Self { + hash: AddrHash::new(0), + type_index: TypeIndex::COINBASE, + output_type: OutputType::Unknown, + }; + + #[inline] + fn matches(&self, output_type: OutputType, hash: AddrHash) -> bool { + self.output_type == output_type && self.hash == hash + } + + #[inline] + fn is_empty(&self) -> bool { + self.output_type == OutputType::Unknown + } +} + +#[derive(Clone, Copy)] +#[repr(C, align(64))] +struct Set([Entry; ASSOCIATIVITY]); + +impl Set { + const EMPTY: Self = Self([Entry::EMPTY; ASSOCIATIVITY]); +} + +#[derive(Clone, Copy, Default)] +struct SetState(u8); + +impl SetState { + #[inline] + fn mark_referenced(&mut self, entry_index: usize) { + self.0 |= 1 << (REFERENCE_SHIFT + entry_index); + } + + #[inline] + fn victim(&mut self) -> usize { + loop { + let entry_index = usize::from(self.0 & (ASSOCIATIVITY as u8 - 1)); + self.0 = (self.0 & !(ASSOCIATIVITY as u8 - 1)) + | ((entry_index + 1) & (ASSOCIATIVITY - 1)) as u8; + + let reference = 1 << (REFERENCE_SHIFT + entry_index); + if self.0 & reference == 0 { + return entry_index; + } + self.0 &= !reference; + } + } +} + +const _: () = assert!(mem::size_of::() == 16); +const _: () = assert!(mem::size_of::() == 64); +const _: () = assert!(mem::size_of::() == 1); + +struct Storage { + sets: Box<[Set]>, + states: Box<[SetState]>, +} + +impl Storage { + fn new() -> Self { + Self { + sets: vec![Set::EMPTY; SET_COUNT].into_boxed_slice(), + states: vec![SetState::default(); SET_COUNT].into_boxed_slice(), + } + } +} + +#[derive(Default)] +pub(super) struct AddressCache { + storage: Option, +} + +impl AddressCache { + #[inline] + pub(super) fn get(&mut self, output_type: OutputType, hash: AddrHash) -> Option { + let storage = self.storage.as_mut()?; + let set_index = Self::set_index(output_type, hash); + let entry_index = storage.sets[set_index] + .0 + .iter() + .position(|entry| entry.matches(output_type, hash))?; + storage.states[set_index].mark_referenced(entry_index); + Some(storage.sets[set_index].0[entry_index].type_index) + } + + #[inline] + pub(super) fn insert( + &mut self, + output_type: OutputType, + hash: AddrHash, + type_index: TypeIndex, + ) { + let set_index = Self::set_index(output_type, hash); + let storage = self.storage.get_or_insert_with(Storage::new); + let set = &mut storage.sets[set_index].0; + let state = &mut storage.states[set_index]; + + let entry_index = set + .iter() + .position(Entry::is_empty) + .unwrap_or_else(|| state.victim()); + + set[entry_index] = Entry { + hash, + type_index, + output_type, + }; + state.mark_referenced(entry_index); + } + + pub(super) fn clear(&mut self) { + self.storage = None; + } + + #[inline] + fn set_index(output_type: OutputType, hash: AddrHash) -> usize { + let hash = *hash ^ (output_type as u64).wrapping_mul(TYPE_MIX); + hash as usize & SET_MASK + } +} diff --git a/crates/brk_indexer/src/processor/txout/mod.rs b/crates/brk_indexer/src/processor/txout/mod.rs new file mode 100644 index 000000000..9676440d0 --- /dev/null +++ b/crates/brk_indexer/src/processor/txout/mod.rs @@ -0,0 +1,207 @@ +mod address; +mod op_return; +mod processed; + +pub(crate) use address::BlockAddresses; +pub(super) use processed::ProcessedOutput; +use processed::ProcessedOutputData; + +use bitcoin::{Script, Transaction, constants::WITNESS_SCALE_FACTOR}; +use brk_cohort::ByAddrType; +use brk_error::Result; +use brk_store::Store; +use brk_types::{ + AddrBytes, AddrHash, AddrIndexOutPoint, AddrIndexTxIndex, OutPoint, OutputType, Sats, SigOps, + TxIndex, TxOutIndex, TypeIndex, Unit, Vout, +}; +use rayon::prelude::*; +use vecdb::{BytesVec, WritableVec, likely}; + +use super::BlockProcessor; +use crate::{AddrsVecs, Lengths, OpReturnVecs, OutputsVecs, ScriptsVecs}; + +impl<'a> BlockProcessor<'a> { + pub(crate) fn process_outputs( + &self, + addresses: &mut BlockAddresses, + ) -> Result> { + let total_outputs: usize = self.block.txdata.iter().map(|tx| tx.output.len()).sum(); + let mut items = Vec::with_capacity(total_outputs); + for tx in &self.block.txdata { + items.extend(&tx.output); + } + + let outputs = items + .into_par_iter() + .map(|txout| { + let script = &txout.script_pubkey; + let output_type = OutputType::from(script); + let legacy_sigops = executed_legacy_sigops_for_output(output_type, script); + let data = if output_type.is_addr() { + ProcessedOutputData::Address( + AddrHash::from_script(script, output_type).unwrap(), + ) + } else if likely(output_type == OutputType::OpReturn) { + ProcessedOutputData::OpReturn(op_return::analyze(script)) + } else { + ProcessedOutputData::None + }; + + ProcessedOutput { + output_type, + legacy_sigops, + data, + } + }) + .collect::>(); + + addresses.resolve(self, &outputs)?; + + Ok(outputs) + } +} + +fn executed_legacy_sigops_for_output(output_type: OutputType, script_pubkey: &Script) -> SigOps { + SigOps::from( + match output_type { + OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65 => 1, + OutputType::P2MS | OutputType::Unknown => script_pubkey.count_sigops(), + _ => 0, + } + .saturating_mul(WITNESS_SCALE_FACTOR), + ) +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn finalize_outputs( + transactions: &[Transaction], + base_tx_index: TxIndex, + lengths: &mut Lengths, + first_txout_index: &mut BytesVec, + outputs: &mut OutputsVecs, + addrs: &mut AddrsVecs, + scripts: &mut ScriptsVecs, + op_return_vecs: &mut OpReturnVecs, + addr_hash_stores: &mut ByAddrType>, + addr_tx_index_stores: &mut ByAddrType>, + addr_outpoint_stores: &mut ByAddrType>, + txouts: &mut [ProcessedOutput], + addresses: &mut BlockAddresses, +) -> Result<()> { + let base_txout_index = lengths.txout_index; + let mut output_offset = 0; + for (block_tx_index, tx) in transactions.iter().enumerate() { + let tx_index = base_tx_index + TxIndex::from(block_tx_index); + let next_output_offset = output_offset + tx.output.len(); + + for (vout, (txout, processed)) in tx + .output + .iter() + .zip(&mut txouts[output_offset..next_output_offset]) + .enumerate() + { + let block_txout_index = output_offset + vout; + let txout_index = base_txout_index + TxOutIndex::from(block_txout_index); + let vout = Vout::from(vout); + let output_type = processed.output_type; + let legacy_sigops = processed.legacy_sigops; + let data = processed.data; + let sats = Sats::from(txout.value); + + if vout.is_zero() { + first_txout_index.checked_push(tx_index, txout_index)?; + } + + outputs.tx_index.checked_push(txout_index, tx_index)?; + + let type_index = match data { + ProcessedOutputData::Address(addr_hash) => { + let addr_type = output_type; + let type_index = addresses.index_mut(addr_type, &addr_hash); + + if let Some(ti) = *type_index { + ti + } else { + let ti = lengths.increment_addr_index(addr_type); + + *type_index = Some(ti); + addr_hash_stores + .get_mut_unwrap(addr_type) + .insert(addr_hash, ti); + let addr_bytes = + AddrBytes::try_from((&txout.script_pubkey, addr_type)).unwrap(); + addrs.push_bytes_if_needed(ti, addr_bytes)?; + + ti + } + } + ProcessedOutputData::OpReturn(op_return) => { + let op_return_index = lengths.op_return_index; + + op_return_vecs + .to_tx_index + .checked_push(lengths.op_return_index, tx_index)?; + op_return_vecs + .kind + .checked_push(op_return_index, op_return.kind)?; + op_return_vecs + .post_op_return_bytes + .checked_push(op_return_index, op_return.post_op_return_bytes)?; + lengths.op_return_index.copy_then_increment() + } + ProcessedOutputData::None => match output_type { + OutputType::P2MS => { + let index = lengths.p2ms_output_index; + scripts.p2ms.to_tx_index.checked_push(index, tx_index)?; + scripts + .p2ms + .legacy_sigops + .checked_push(index, legacy_sigops)?; + lengths.p2ms_output_index.copy_then_increment() + } + OutputType::Empty => { + scripts + .empty + .to_tx_index + .checked_push(lengths.empty_output_index, tx_index)?; + lengths.empty_output_index.copy_then_increment() + } + OutputType::Unknown => { + let index = lengths.unknown_output_index; + scripts.unknown.to_tx_index.checked_push(index, tx_index)?; + scripts + .unknown + .legacy_sigops + .checked_push(index, legacy_sigops)?; + lengths.unknown_output_index.copy_then_increment() + } + _ => unreachable!(), + }, + ProcessedOutputData::Resolved(_) => unreachable!(), + }; + + outputs.value.checked_push(txout_index, sats)?; + outputs.output_type.checked_push(txout_index, output_type)?; + outputs.type_index.checked_push(txout_index, type_index)?; + processed.data = ProcessedOutputData::Resolved(type_index); + + if likely(output_type.is_addr()) { + let addr_type = output_type; + let addr_index = type_index; + + addr_tx_index_stores + .get_mut_unwrap(addr_type) + .insert(AddrIndexTxIndex::from((addr_index, tx_index)), Unit); + + addr_outpoint_stores.get_mut_unwrap(addr_type).insert( + AddrIndexOutPoint::from((addr_index, OutPoint::new(tx_index, vout))), + Unit, + ); + } + } + + output_offset = next_output_offset; + } + + Ok(()) +} diff --git a/crates/brk_indexer/src/processor/txout/op_return.rs b/crates/brk_indexer/src/processor/txout/op_return.rs new file mode 100644 index 000000000..b12711cc4 --- /dev/null +++ b/crates/brk_indexer/src/processor/txout/op_return.rs @@ -0,0 +1,221 @@ +use bitcoin::{ + Script, + opcodes::all::{OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY, OP_CHECKSIG, OP_CHECKSIGVERIFY}, + script::Instruction, +}; +use brk_types::{OpReturnKind, StoredU32}; + +#[derive(Debug, Clone, Copy)] +pub(super) struct Facts { + pub(super) kind: OpReturnKind, + pub(super) legacy_sigops: StoredU32, + pub(super) post_op_return_bytes: StoredU32, +} + +pub(super) fn analyze(script: &Script) -> Facts { + let data = &script.as_bytes()[1..]; + let (prefix, legacy_sigops) = scan(script); + Facts { + kind: classify(data, prefix), + legacy_sigops: StoredU32::from(legacy_sigops), + post_op_return_bytes: StoredU32::from(data.len()), + } +} + +fn classify(data: &[u8], prefix: Option<&[u8]>) -> OpReturnKind { + if data.first() == Some(&0x5d) { + return OpReturnKind::Runes; + } + + let Some(prefix) = prefix else { + return OpReturnKind::Empty; + }; + + if prefix.starts_with(b"omni") { + OpReturnKind::Omni + } else if prefix.starts_with(b"X2") || prefix.starts_with(b"X1") { + OpReturnKind::Stacks + } else if prefix.starts_with(b"id") { + OpReturnKind::Blockstack + } else if prefix.starts_with(b"CC") { + OpReturnKind::Colu + } else if prefix.starts_with(b"OA\x01\x00") { + OpReturnKind::OpenAssets + } else if prefix.starts_with(b"SPK") { + OpReturnKind::CoinSpark + } else if prefix.starts_with(b"POET") { + OpReturnKind::Poet + } else if prefix.starts_with(b"DOCPROOF") { + OpReturnKind::Docproof + } else if prefix.starts_with(b"\x05\x88\x96\x0d\x73\xd7\x19\x01") { + OpReturnKind::OpenTimestamps + } else if prefix.starts_with(b"Factom!!") { + OpReturnKind::Factom + } else if prefix.starts_with(b"EW") { + OpReturnKind::EternityWall + } else if is_memo(prefix) { + OpReturnKind::Memo + } else if prefix.starts_with(b"BP") { + OpReturnKind::Bitproof + } else if prefix.starts_with(b"ASCRIBE\0") { + OpReturnKind::Ascribe + } else if prefix.starts_with(b"Stampery") { + OpReturnKind::Stampery + } else if prefix.starts_with(b"EPOBC") { + OpReturnKind::Epobc + } else if data.len() == 82 { + OpReturnKind::VeriBlock + } else if (36..=38).contains(&data.len()) { + OpReturnKind::Komodo + } else if matches!(prefix.len(), 20 | 32) { + OpReturnKind::BareHash + } else if is_text(prefix) { + OpReturnKind::Text + } else { + OpReturnKind::Unknown + } +} + +fn scan(script: &Script) -> (Option<&[u8]>, usize) { + let mut first_push = None; + let mut legacy_sigops = 0; + + for instruction in script.instructions().skip(1) { + match instruction { + Ok(Instruction::PushBytes(bytes)) => { + if first_push.is_none() && !bytes.is_empty() { + first_push = Some(bytes.as_bytes()); + } + } + Ok(Instruction::Op(opcode)) => match opcode { + OP_CHECKSIG | OP_CHECKSIGVERIFY => legacy_sigops += 1, + OP_CHECKMULTISIG | OP_CHECKMULTISIGVERIFY => legacy_sigops += 20, + _ => {} + }, + Err(_) => break, + } + } + + (first_push, legacy_sigops) +} + +fn is_memo(prefix: &[u8]) -> bool { + prefix.len() >= 2 + && prefix[0] == 0x6d + && (matches!(prefix[1], 0x01..=0x07) || prefix[1] == 0x0c) +} + +fn is_text(prefix: &[u8]) -> bool { + prefix.len() >= 4 + && prefix + .iter() + .filter(|byte| byte.is_ascii_graphic() || **byte == b' ') + .count() + * 10 + >= prefix.len() * 9 +} + +#[cfg(test)] +mod tests { + use bitcoin::ScriptBuf; + use bitcoin::opcodes::all::OP_RETURN; + use bitcoin::script::{Builder, PushBytesBuf}; + + use super::*; + + fn pushed(data: &[u8]) -> ScriptBuf { + Builder::new() + .push_opcode(OP_RETURN) + .push_slice(PushBytesBuf::try_from(data.to_vec()).unwrap()) + .into_script() + } + + #[test] + fn classifies_exact_prefix_before_heuristics() { + let mut payload = b"omni".to_vec(); + payload.resize(80, 0); + + assert_eq!(analyze(&pushed(&payload)).kind, OpReturnKind::Omni); + } + + #[test] + fn classifies_runes_opcode() { + assert_eq!( + analyze(&ScriptBuf::from_bytes(vec![OP_RETURN.to_u8(), 0x5d])).kind, + OpReturnKind::Runes + ); + } + + #[test] + fn classifies_empty_and_unknown() { + assert_eq!( + analyze(&ScriptBuf::from_bytes(vec![OP_RETURN.to_u8()])).kind, + OpReturnKind::Empty + ); + assert_eq!( + analyze(&pushed(&[0, 1, 2, 3, 4])).kind, + OpReturnKind::Unknown + ); + } + + #[test] + fn classifies_known_protocol_prefixes() { + let cases: &[(&[u8], OpReturnKind)] = &[ + (b"omni", OpReturnKind::Omni), + (b"X2", OpReturnKind::Stacks), + (b"id", OpReturnKind::Blockstack), + (b"CC", OpReturnKind::Colu), + (b"OA\x01\x00", OpReturnKind::OpenAssets), + (b"SPK", OpReturnKind::CoinSpark), + (b"POET", OpReturnKind::Poet), + (b"DOCPROOF", OpReturnKind::Docproof), + ( + b"\x05\x88\x96\x0d\x73\xd7\x19\x01", + OpReturnKind::OpenTimestamps, + ), + (b"Factom!!", OpReturnKind::Factom), + (b"EW", OpReturnKind::EternityWall), + (b"\x6d\x01", OpReturnKind::Memo), + (b"BP", OpReturnKind::Bitproof), + (b"ASCRIBE\0", OpReturnKind::Ascribe), + (b"Stampery", OpReturnKind::Stampery), + (b"EPOBC", OpReturnKind::Epobc), + ]; + + for (prefix, expected) in cases { + assert_eq!(analyze(&pushed(prefix)).kind, *expected); + } + } + + #[test] + fn classifies_length_and_content_heuristics() { + assert_eq!(analyze(&pushed(&[1; 80])).kind, OpReturnKind::VeriBlock); + assert_eq!(analyze(&pushed(&[1; 35])).kind, OpReturnKind::Komodo); + assert_eq!(analyze(&pushed(&[1; 20])).kind, OpReturnKind::BareHash); + assert_eq!(analyze(&pushed(b"plain text")).kind, OpReturnKind::Text); + } + + #[test] + fn counts_only_executed_legacy_sigop_opcodes() { + use bitcoin::opcodes::all::{ + OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY, OP_CHECKSIG, OP_CHECKSIGVERIFY, + }; + + let opcodes = ScriptBuf::from_bytes(vec![ + OP_RETURN.to_u8(), + OP_CHECKSIG.to_u8(), + OP_CHECKSIGVERIFY.to_u8(), + OP_CHECKMULTISIG.to_u8(), + OP_CHECKMULTISIGVERIFY.to_u8(), + ]); + assert_eq!(u32::from(analyze(&opcodes).legacy_sigops), 42); + + let pushed = pushed(&[ + OP_CHECKSIG.to_u8(), + OP_CHECKSIGVERIFY.to_u8(), + OP_CHECKMULTISIG.to_u8(), + OP_CHECKMULTISIGVERIFY.to_u8(), + ]); + assert_eq!(u32::from(analyze(&pushed).legacy_sigops), 0); + } +} diff --git a/crates/brk_indexer/src/processor/txout/processed.rs b/crates/brk_indexer/src/processor/txout/processed.rs new file mode 100644 index 000000000..098e0d00a --- /dev/null +++ b/crates/brk_indexer/src/processor/txout/processed.rs @@ -0,0 +1,34 @@ +use brk_types::{AddrHash, OutputType, SigOps, TypeIndex}; + +use super::op_return; + +#[derive(Clone, Copy)] +pub(super) enum ProcessedOutputData { + Address(AddrHash), + OpReturn(op_return::Facts), + None, + Resolved(TypeIndex), +} + +#[derive(Clone, Copy)] +pub(crate) struct ProcessedOutput { + pub(crate) output_type: OutputType, + pub(crate) legacy_sigops: SigOps, + pub(super) data: ProcessedOutputData, +} + +impl ProcessedOutput { + pub(crate) fn op_return_legacy_sigops(&self) -> usize { + let ProcessedOutputData::OpReturn(facts) = self.data else { + unreachable!(); + }; + u32::from(facts.legacy_sigops) as usize + } + + pub(crate) fn resolved_type_index(&self) -> TypeIndex { + let ProcessedOutputData::Resolved(type_index) = self.data else { + unreachable!(); + }; + type_index + } +} diff --git a/crates/brk_indexer/src/processor/types.rs b/crates/brk_indexer/src/processor/types.rs deleted file mode 100644 index d76c89153..000000000 --- a/crates/brk_indexer/src/processor/types.rs +++ /dev/null @@ -1,70 +0,0 @@ -use bitcoin::{Transaction, TxOut}; -use brk_cohort::ByAddrType; -use brk_types::{ - AddrBytes, AddrHash, OutPoint, OutputType, TxIndex, TxOutIndex, Txid, TxidPrefix, TypeIndex, - Vin, Vout, -}; -use rustc_hash::{FxHashMap, FxHashSet}; - -#[derive(Debug)] -pub enum InputSource { - PreviousBlock { - vin: Vin, - tx_index: TxIndex, - outpoint: OutPoint, - output_type: OutputType, - type_index: TypeIndex, - }, - SameBlock { - tx_index: TxIndex, - vin: Vin, - outpoint: OutPoint, - }, -} - -#[derive(Debug, Clone, Copy)] -pub struct SameBlockOutputInfo { - pub output_type: OutputType, - pub type_index: TypeIndex, -} - -pub struct ProcessedOutput<'a> { - pub txout_index: TxOutIndex, - pub txout: &'a TxOut, - pub tx_index: TxIndex, - pub vout: Vout, - pub output_type: OutputType, - pub addr_info: Option<(AddrBytes, AddrHash)>, - pub existing_type_index: Option, -} - -pub struct ComputedTx<'a> { - pub tx_index: TxIndex, - pub tx: &'a Transaction, - pub txid: Txid, - pub txid_prefix: TxidPrefix, - pub prev_tx_index_opt: Option, - pub base_size: u32, - pub total_size: u32, -} - -impl ComputedTx<'_> { - #[inline] - pub fn is_segwit(&self) -> bool { - self.base_size != self.total_size - } - - #[inline] - pub fn weight(&self) -> usize { - self.base_size as usize * 3 + self.total_size as usize - } -} - -/// Reusable buffers cleared and refilled each block to avoid allocation churn. -#[derive(Default)] -pub struct BlockBuffers { - pub txid_prefix_map: FxHashMap, - pub same_block_spent: FxHashSet, - pub already_added_addrs: ByAddrType>, - pub same_block_output_info: FxHashMap, -} diff --git a/crates/brk_indexer/src/readers.rs b/crates/brk_indexer/src/readers.rs index d66ecbf97..c5c40271a 100644 --- a/crates/brk_indexer/src/readers.rs +++ b/crates/brk_indexer/src/readers.rs @@ -1,8 +1,9 @@ use bitcoin::ScriptBuf; use brk_types::{ - AddrBytes, OutputType, P2AAddrIndex, P2ABytes, P2PK33AddrIndex, P2PK33Bytes, P2PK65AddrIndex, - P2PK65Bytes, P2PKHAddrIndex, P2PKHBytes, P2SHAddrIndex, P2SHBytes, P2TRAddrIndex, P2TRBytes, - P2WPKHAddrIndex, P2WPKHBytes, P2WSHAddrIndex, P2WSHBytes, TxIndex, TxOutIndex, Txid, TypeIndex, + AddrBytes, OutputType, P2AAddrIndex, P2ABytes, P2MSOutputIndex, P2PK33AddrIndex, P2PK33Bytes, + P2PK65AddrIndex, P2PK65Bytes, P2PKHAddrIndex, P2PKHBytes, P2SHAddrIndex, P2SHBytes, + P2TRAddrIndex, P2TRBytes, P2WPKHAddrIndex, P2WPKHBytes, P2WSHAddrIndex, P2WSHBytes, SigOps, + TxIndex, TxOutIndex, Txid, TypeIndex, UnknownOutputIndex, }; use vecdb::{BytesStrategy, VecReader}; @@ -46,6 +47,7 @@ pub struct Readers { pub tx_index_to_first_txout_index: VecReader>, pub txout_index_to_output_type: VecReader>, pub txout_index_to_type_index: VecReader>, + pub scripts: ScriptReaders, pub addrbytes: AddrReaders, } @@ -56,7 +58,16 @@ impl Readers { tx_index_to_first_txout_index: vecs.transactions.first_txout_index.reader(), txout_index_to_output_type: vecs.outputs.output_type.reader(), txout_index_to_type_index: vecs.outputs.type_index.reader(), + scripts: ScriptReaders { + p2ms_legacy_sigops: vecs.scripts.p2ms.legacy_sigops.reader(), + unknown_legacy_sigops: vecs.scripts.unknown.legacy_sigops.reader(), + }, addrbytes: vecs.addrs.addr_readers(), } } } + +pub struct ScriptReaders { + pub p2ms_legacy_sigops: VecReader>, + pub unknown_legacy_sigops: VecReader>, +} diff --git a/crates/brk_indexer/src/vecs/mod.rs b/crates/brk_indexer/src/vecs/mod.rs index 1ae97ee7e..563ce5502 100644 --- a/crates/brk_indexer/src/vecs/mod.rs +++ b/crates/brk_indexer/src/vecs/mod.rs @@ -14,6 +14,7 @@ mod addrs; mod blocks; mod inputs; mod macros; +mod op_return; mod outputs; mod scripts; mod transactions; @@ -21,6 +22,7 @@ mod transactions; pub use addrs::*; pub use blocks::*; pub use inputs::*; +pub use op_return::*; pub use outputs::*; pub use scripts::*; pub use transactions::*; @@ -34,6 +36,8 @@ pub struct Vecs { pub blocks: BlocksVecs, #[traversable(wrap = "transactions", rename = "raw")] pub transactions: TransactionsVecs, + #[traversable(wrap = "transactions", rename = "features")] + pub transaction_features: TransactionFeaturesVecs, #[traversable(wrap = "inputs", rename = "raw")] pub inputs: InputsVecs, #[traversable(wrap = "outputs", rename = "raw")] @@ -42,6 +46,8 @@ pub struct Vecs { pub addrs: AddrsVecs, #[traversable(wrap = "scripts", rename = "raw")] pub scripts: ScriptsVecs, + #[traversable(wrap = "op_return", rename = "raw")] + pub op_return: OpReturnVecs, } impl Vecs { @@ -51,23 +57,36 @@ impl Vecs { tracing::debug!("Setting min len..."); db.set_min_len(PAGE_SIZE * 60_000_000)?; - let (blocks, transactions, inputs, outputs, addrs, scripts) = parallel_import! { + let ( + blocks, + transactions, + transaction_features, + inputs, + outputs, + addrs, + scripts, + op_return, + ) = parallel_import! { blocks = BlocksVecs::forced_import(&db, version), transactions = TransactionsVecs::forced_import(&db, version), + transaction_features = TransactionFeaturesVecs::forced_import(&db, version), inputs = InputsVecs::forced_import(&db, version), outputs = OutputsVecs::forced_import(&db, version), addrs = AddrsVecs::forced_import(&db, version), scripts = ScriptsVecs::forced_import(&db, version), + op_return = OpReturnVecs::forced_import(&db, version), }; let this = Self { db, blocks, transactions, + transaction_features, inputs, outputs, addrs, scripts, + op_return, }; this.db.retain_regions( @@ -89,6 +108,12 @@ impl Vecs { self.transactions .truncate(starting_lengths.height, starting_lengths.tx_index, stamp)?; + self.transaction_features.truncate( + starting_lengths.height, + starting_lengths.tx_index, + stamp, + )?; + self.inputs .truncate(starting_lengths.height, starting_lengths.txin_index, stamp)?; @@ -111,12 +136,17 @@ impl Vecs { self.scripts.truncate( starting_lengths.height, starting_lengths.empty_output_index, - starting_lengths.op_return_index, starting_lengths.p2ms_output_index, starting_lengths.unknown_output_index, stamp, )?; + self.op_return.truncate( + starting_lengths.height, + starting_lengths.op_return_index, + stamp, + )?; + Ok(()) } @@ -167,19 +197,23 @@ impl Vecs { self.blocks .par_iter_mut_any() .chain(self.transactions.par_iter_mut_any()) + .chain(self.transaction_features.par_iter_mut_any()) .chain(self.inputs.par_iter_mut_any()) .chain(self.outputs.par_iter_mut_any()) .chain(self.addrs.par_iter_mut_any()) .chain(self.scripts.par_iter_mut_any()) + .chain(self.op_return.par_iter_mut_any()) } fn iter_any_stored_vec(&self) -> impl Iterator { self.blocks .iter_any() .chain(self.transactions.iter_any()) + .chain(self.transaction_features.iter_any()) .chain(self.inputs.iter_any()) .chain(self.outputs.iter_any()) .chain(self.addrs.iter_any()) .chain(self.scripts.iter_any()) + .chain(self.op_return.iter_any()) } } diff --git a/crates/brk_indexer/src/vecs/op_return.rs b/crates/brk_indexer/src/vecs/op_return.rs new file mode 100644 index 000000000..441514882 --- /dev/null +++ b/crates/brk_indexer/src/vecs/op_return.rs @@ -0,0 +1,70 @@ +use brk_error::Result; +use brk_traversable::Traversable; +use brk_types::{Height, OpReturnIndex, OpReturnKind, StoredU32, TxIndex, Version}; +use rayon::prelude::*; +use vecdb::{AnyStoredVec, Database, ImportableVec, PcoVec, Rw, Stamp, StorageMode, WritableVec}; + +use crate::parallel_import; + +#[derive(Traversable)] +pub struct OpReturnVecs { + pub first_index: M::Stored>, + pub to_tx_index: M::Stored>, + pub kind: M::Stored>, + pub post_op_return_bytes: M::Stored>, +} + +impl OpReturnVecs { + pub fn forced_import(db: &Database, version: Version) -> Result { + let (first_index, to_tx_index, kind, post_op_return_bytes) = parallel_import! { + first_index = PcoVec::forced_import(db, "first_op_return_index", version), + to_tx_index = PcoVec::forced_import(db, "tx_index", version), + kind = PcoVec::forced_import(db, "kind", version), + post_op_return_bytes = + PcoVec::forced_import(db, "op_return_post_op_return_bytes", version), + }; + Ok(Self { + first_index, + to_tx_index, + kind, + post_op_return_bytes, + }) + } + + pub fn truncate( + &mut self, + height: Height, + op_return_index: OpReturnIndex, + stamp: Stamp, + ) -> Result<()> { + self.first_index + .truncate_if_needed_with_stamp(height, stamp)?; + self.to_tx_index + .truncate_if_needed_with_stamp(op_return_index, stamp)?; + self.kind + .truncate_if_needed_with_stamp(op_return_index, stamp)?; + self.post_op_return_bytes + .truncate_if_needed_with_stamp(op_return_index, stamp)?; + Ok(()) + } + + pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator { + [ + &mut self.first_index as &mut dyn AnyStoredVec, + &mut self.to_tx_index, + &mut self.kind, + &mut self.post_op_return_bytes, + ] + .into_par_iter() + } + + pub fn iter_any(&self) -> impl Iterator { + [ + &self.first_index as &dyn AnyStoredVec, + &self.to_tx_index, + &self.kind, + &self.post_op_return_bytes, + ] + .into_iter() + } +} diff --git a/crates/brk_indexer/src/vecs/scripts.rs b/crates/brk_indexer/src/vecs/scripts.rs index 7459ec097..f9c3eb079 100644 --- a/crates/brk_indexer/src/vecs/scripts.rs +++ b/crates/brk_indexer/src/vecs/scripts.rs @@ -1,13 +1,14 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{ - EmptyOutputIndex, Height, OpReturnIndex, P2MSOutputIndex, TxIndex, UnknownOutputIndex, Version, + EmptyOutputIndex, Height, OutputType, P2MSOutputIndex, SigOps, TxIndex, TypeIndex, + UnknownOutputIndex, Version, }; use rayon::prelude::*; use schemars::JsonSchema; use serde::Serialize; use vecdb::{ - AnyStoredVec, Database, Formattable, ImportableVec, PcoVec, PcoVecValue, Rw, Stamp, + AnyStoredVec, BytesVec, Database, Formattable, ImportableVec, PcoVec, PcoVecValue, Rw, Stamp, StorageMode, VecIndex, WritableVec, }; @@ -22,51 +23,58 @@ pub struct ScriptTypeVecs< pub to_tx_index: M::Stored>, } +#[derive(Traversable)] +pub struct ScriptTypeWithSigOpsVecs< + I: VecIndex + PcoVecValue + Formattable + Serialize + JsonSchema, + M: StorageMode = Rw, +> { + pub first_index: M::Stored>, + pub to_tx_index: M::Stored>, + pub legacy_sigops: M::Stored>, +} + #[derive(Traversable)] pub struct ScriptsVecs { pub empty: ScriptTypeVecs, - pub op_return: ScriptTypeVecs, - pub p2ms: ScriptTypeVecs, - pub unknown: ScriptTypeVecs, + pub p2ms: ScriptTypeWithSigOpsVecs, + pub unknown: ScriptTypeWithSigOpsVecs, } impl ScriptsVecs { pub fn forced_import(db: &Database, version: Version) -> Result { let ( first_empty_output_index, - first_op_return_index, first_p2ms_output_index, first_unknown_output_index, empty_output_index_to_tx_index, - op_return_index_to_tx_index, p2ms_output_index_to_tx_index, unknown_output_index_to_tx_index, + p2ms_legacy_sigops, + unknown_legacy_sigops, ) = parallel_import! { first_empty_output_index = PcoVec::forced_import(db, "first_empty_output_index", version), - first_op_return_index = PcoVec::forced_import(db, "first_op_return_index", version), first_p2ms_output_index = PcoVec::forced_import(db, "first_p2ms_output_index", version), first_unknown_output_index = PcoVec::forced_import(db, "first_unknown_output_index", version), empty_output_index_to_tx_index = PcoVec::forced_import(db, "tx_index", version), - op_return_index_to_tx_index = PcoVec::forced_import(db, "tx_index", version), p2ms_output_index_to_tx_index = PcoVec::forced_import(db, "tx_index", version), unknown_output_index_to_tx_index = PcoVec::forced_import(db, "tx_index", version), + p2ms_legacy_sigops = BytesVec::forced_import(db, "p2ms_legacy_sigops", version), + unknown_legacy_sigops = BytesVec::forced_import(db, "unknown_legacy_sigops", version), }; Ok(Self { empty: ScriptTypeVecs { first_index: first_empty_output_index, to_tx_index: empty_output_index_to_tx_index, }, - op_return: ScriptTypeVecs { - first_index: first_op_return_index, - to_tx_index: op_return_index_to_tx_index, - }, - p2ms: ScriptTypeVecs { + p2ms: ScriptTypeWithSigOpsVecs { first_index: first_p2ms_output_index, to_tx_index: p2ms_output_index_to_tx_index, + legacy_sigops: p2ms_legacy_sigops, }, - unknown: ScriptTypeVecs { + unknown: ScriptTypeWithSigOpsVecs { first_index: first_unknown_output_index, to_tx_index: unknown_output_index_to_tx_index, + legacy_sigops: unknown_legacy_sigops, }, }) } @@ -75,7 +83,6 @@ impl ScriptsVecs { &mut self, height: Height, empty_output_index: EmptyOutputIndex, - op_return_index: OpReturnIndex, p2ms_output_index: P2MSOutputIndex, unknown_output_index: UnknownOutputIndex, stamp: Stamp, @@ -83,9 +90,6 @@ impl ScriptsVecs { self.empty .first_index .truncate_if_needed_with_stamp(height, stamp)?; - self.op_return - .first_index - .truncate_if_needed_with_stamp(height, stamp)?; self.p2ms .first_index .truncate_if_needed_with_stamp(height, stamp)?; @@ -95,28 +99,31 @@ impl ScriptsVecs { self.empty .to_tx_index .truncate_if_needed_with_stamp(empty_output_index, stamp)?; - self.op_return - .to_tx_index - .truncate_if_needed_with_stamp(op_return_index, stamp)?; self.p2ms .to_tx_index .truncate_if_needed_with_stamp(p2ms_output_index, stamp)?; + self.p2ms + .legacy_sigops + .truncate_if_needed_with_stamp(p2ms_output_index, stamp)?; self.unknown .to_tx_index .truncate_if_needed_with_stamp(unknown_output_index, stamp)?; + self.unknown + .legacy_sigops + .truncate_if_needed_with_stamp(unknown_output_index, stamp)?; Ok(()) } pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator { [ &mut self.empty.first_index as &mut dyn AnyStoredVec, - &mut self.op_return.first_index, &mut self.p2ms.first_index, &mut self.unknown.first_index, &mut self.empty.to_tx_index, - &mut self.op_return.to_tx_index, &mut self.p2ms.to_tx_index, + &mut self.p2ms.legacy_sigops, &mut self.unknown.to_tx_index, + &mut self.unknown.legacy_sigops, ] .into_par_iter() } @@ -124,14 +131,34 @@ impl ScriptsVecs { pub fn iter_any(&self) -> impl Iterator { [ &self.empty.first_index as &dyn AnyStoredVec, - &self.op_return.first_index, &self.p2ms.first_index, &self.unknown.first_index, &self.empty.to_tx_index, - &self.op_return.to_tx_index, &self.p2ms.to_tx_index, + &self.p2ms.legacy_sigops, &self.unknown.to_tx_index, + &self.unknown.legacy_sigops, ] .into_iter() } + + pub(crate) fn legacy_sigops( + &self, + output_type: OutputType, + type_index: TypeIndex, + readers: &crate::readers::ScriptReaders, + ) -> Option { + match output_type { + OutputType::P2PK65 | OutputType::P2PK33 | OutputType::P2PKH => Some(SigOps::new(4)), + OutputType::P2MS => self + .p2ms + .legacy_sigops + .get_pushed_or_read(type_index.into(), &readers.p2ms_legacy_sigops), + OutputType::Unknown => self + .unknown + .legacy_sigops + .get_pushed_or_read(type_index.into(), &readers.unknown_legacy_sigops), + _ => Some(SigOps::ZERO), + } + } } diff --git a/crates/brk_indexer/src/vecs/transactions/features/counts.rs b/crates/brk_indexer/src/vecs/transactions/features/counts.rs new file mode 100644 index 000000000..7c3bd45ac --- /dev/null +++ b/crates/brk_indexer/src/vecs/transactions/features/counts.rs @@ -0,0 +1,179 @@ +use brk_error::Result; +use brk_traversable::Traversable; +use brk_types::{Height, StoredU64, TxVersion, Version}; +use rayon::prelude::*; +use vecdb::{AnyStoredVec, Database, ImportableVec, PcoVec, Rw, Stamp, StorageMode, WritableVec}; + +use super::schema::with_transaction_features; + +macro_rules! define_counts { + ($($(#[$attribute:meta])* $vector:ident: $flag:ident = $bit:literal $(, count: $count:ident)?;)+) => { + #[derive(Default)] + pub(crate) struct TransactionCounts { + v1: u64, + v2: u64, + v3: u64, + other_version: u64, + explicitly_rbf: u64, + one_input: u64, + one_output: u64, + $($(pub(super) $count: u64,)?) + + } + + impl TransactionCounts { + pub(crate) fn add_base( + &mut self, + input_count: usize, + output_count: usize, + version: TxVersion, + explicitly_rbf: bool, + ) { + match version { + TxVersion::ONE => self.v1 += 1, + TxVersion::TWO => self.v2 += 1, + TxVersion::THREE => self.v3 += 1, + _ => self.other_version += 1, + } + self.explicitly_rbf += explicitly_rbf as u64; + self.one_input += (input_count == 1) as u64; + self.one_output += (output_count == 1) as u64; + } + } + + #[derive(Traversable)] + pub struct TransactionCountVecs { + pub v1: M::Stored>, + pub v2: M::Stored>, + pub v3: M::Stored>, + pub other_version: M::Stored>, + pub explicitly_rbf: M::Stored>, + pub one_input: M::Stored>, + pub one_output: M::Stored>, + $($(pub $count: M::Stored>,)?) + + } + + impl TransactionCountVecs { + pub fn forced_import(db: &Database, version: Version) -> Result { + let ( + v1, + v2, + v3, + other_version, + explicitly_rbf, + one_input, + one_output, + $($($count,)?) + + ) = crate::parallel_import! { + v1 = PcoVec::forced_import(db, "tx_count_v1", version), + v2 = PcoVec::forced_import(db, "tx_count_v2", version), + v3 = PcoVec::forced_import(db, "tx_count_v3", version), + other_version = PcoVec::forced_import(db, "tx_count_other_version", version), + explicitly_rbf = PcoVec::forced_import(db, "tx_count_explicitly_rbf", version), + one_input = PcoVec::forced_import(db, "tx_count_one_input", version), + one_output = PcoVec::forced_import(db, "tx_count_one_output", version), + $($($count = PcoVec::forced_import( + db, + concat!("tx_count_", stringify!($count)), + version, + ),)?) + + }; + Ok(Self { + v1, + v2, + v3, + other_version, + explicitly_rbf, + one_input, + one_output, + $($($count,)?) + + }) + } + + pub(crate) fn checked_push( + &mut self, + height: Height, + counts: TransactionCounts, + ) -> Result<()> { + self.v1.checked_push(height, counts.v1.into())?; + self.v2.checked_push(height, counts.v2.into())?; + self.v3.checked_push(height, counts.v3.into())?; + self.other_version + .checked_push(height, counts.other_version.into())?; + self.explicitly_rbf + .checked_push(height, counts.explicitly_rbf.into())?; + self.one_input.checked_push(height, counts.one_input.into())?; + self.one_output.checked_push(height, counts.one_output.into())?; + $($(self.$count.checked_push(height, counts.$count.into())?;)?) + + Ok(()) + } + + pub(crate) fn truncate(&mut self, height: Height, stamp: Stamp) -> Result<()> { + self.v1.truncate_if_needed_with_stamp(height, stamp)?; + self.v2.truncate_if_needed_with_stamp(height, stamp)?; + self.v3.truncate_if_needed_with_stamp(height, stamp)?; + self.other_version + .truncate_if_needed_with_stamp(height, stamp)?; + self.explicitly_rbf + .truncate_if_needed_with_stamp(height, stamp)?; + self.one_input.truncate_if_needed_with_stamp(height, stamp)?; + self.one_output.truncate_if_needed_with_stamp(height, stamp)?; + $($(self.$count.truncate_if_needed_with_stamp(height, stamp)?;)?) + + Ok(()) + } + + pub(crate) fn par_iter_mut_any( + &mut self, + ) -> impl ParallelIterator { + [ + &mut self.v1 as &mut dyn AnyStoredVec, + &mut self.v2, + &mut self.v3, + &mut self.other_version, + &mut self.explicitly_rbf, + &mut self.one_input, + &mut self.one_output, + $($(&mut self.$count,)?) + + ] + .into_par_iter() + } + + pub(crate) fn iter_any(&self) -> impl Iterator { + [ + &self.v1 as &dyn AnyStoredVec, + &self.v2, + &self.v3, + &self.other_version, + &self.explicitly_rbf, + &self.one_input, + &self.one_output, + $($(&self.$count,)?) + + ] + .into_iter() + } + } + }; +} + +with_transaction_features!(define_counts); + +#[cfg(test)] +mod tests { + use brk_types::TxVersion; + + use super::TransactionCounts; + + #[test] + fn counts_base_transaction_properties() { + let mut counts = TransactionCounts::default(); + counts.add_base(1, 2, TxVersion::TWO, true); + counts.add_base(2, 1, TxVersion::NON_STANDARD, false); + + assert_eq!(counts.v1, 0); + assert_eq!(counts.v2, 1); + assert_eq!(counts.v3, 0); + assert_eq!(counts.other_version, 1); + assert_eq!(counts.explicitly_rbf, 1); + assert_eq!(counts.one_input, 1); + assert_eq!(counts.one_output, 1); + } +} diff --git a/crates/brk_indexer/src/vecs/transactions/features/flags.rs b/crates/brk_indexer/src/vecs/transactions/features/flags.rs new file mode 100644 index 000000000..78044f44b --- /dev/null +++ b/crates/brk_indexer/src/vecs/transactions/features/flags.rs @@ -0,0 +1,66 @@ +use brk_types::OutputType; + +use super::schema::with_transaction_features; + +#[derive(Clone, Copy, Default)] +pub(crate) struct TxFeatureFlags(u32); + +macro_rules! define_flags { + ($($(#[$attribute:meta])* $vector:ident: $flag:ident = $bit:literal $(, count: $count:ident)?;)+) => { + impl TxFeatureFlags { + $(pub(crate) const $flag: u32 = 1 << $bit;)+ + } + }; +} + +with_transaction_features!(define_flags); + +impl TxFeatureFlags { + pub(crate) fn insert_type(&mut self, output_type: OutputType) { + self.0 |= match output_type { + OutputType::P2PK65 | OutputType::P2PK33 => Self::P2PK, + OutputType::P2MS => Self::P2MS, + OutputType::P2PKH => Self::P2PKH, + OutputType::P2SH => Self::P2SH, + OutputType::P2WPKH => Self::P2WPKH, + OutputType::P2WSH => Self::P2WSH, + OutputType::P2TR => Self::P2TR, + OutputType::P2A => Self::P2A, + OutputType::OpReturn => Self::OP_RETURN, + OutputType::Empty => Self::EMPTY, + OutputType::Unknown => Self::UNKNOWN, + }; + } + + #[inline] + pub(crate) fn insert(&mut self, flag: u32) { + self.0 |= flag; + } + + #[inline] + pub(crate) fn contains_all(self, flags: u32) -> bool { + self.0 & flags == flags + } + + #[inline] + pub(crate) fn is_set(self, flag: u32) -> bool { + self.0 & flag != 0 + } +} + +#[cfg(test)] +mod tests { + use brk_types::OutputType; + + use super::TxFeatureFlags; + + #[test] + fn type_flags_union_inputs_and_outputs() { + let mut flags = TxFeatureFlags::default(); + flags.insert_type(OutputType::P2PKH); + flags.insert_type(OutputType::P2TR); + + assert!(flags.is_set(TxFeatureFlags::P2PKH)); + assert!(flags.is_set(TxFeatureFlags::P2TR)); + } +} diff --git a/crates/brk_indexer/src/vecs/transactions/features/mod.rs b/crates/brk_indexer/src/vecs/transactions/features/mod.rs new file mode 100644 index 000000000..54900509e --- /dev/null +++ b/crates/brk_indexer/src/vecs/transactions/features/mod.rs @@ -0,0 +1,75 @@ +mod counts; +mod flags; +mod schema; + +pub use counts::TransactionCountVecs; +pub(crate) use counts::TransactionCounts; +pub(crate) use flags::TxFeatureFlags; + +use brk_error::Result; +use brk_traversable::Traversable; +use brk_types::{Height, StoredBool, TxIndex, Version}; +use rayon::prelude::*; +use vecdb::{AnyStoredVec, Database, ImportableVec, PcoVec, Rw, Stamp, StorageMode, WritableVec}; + +use self::schema::with_transaction_features; +use crate::parallel_import; + +macro_rules! define_vecs { + ($($(#[$attribute:meta])* $vector:ident: $flag:ident = $bit:literal $(, count: $count:ident)?;)+) => { + #[derive(Traversable)] + pub struct TransactionFeaturesVecs { + pub count: TransactionCountVecs, + $($(#[$attribute])* pub $vector: M::Stored>,) + + } + + impl TransactionFeaturesVecs { + pub fn forced_import(db: &Database, version: Version) -> Result { + let (count, $($vector,) +) = parallel_import! { + count = TransactionCountVecs::forced_import(db, version), + $($vector = PcoVec::forced_import(db, stringify!($vector), version),) + + }; + Ok(Self { count, $($vector,) + }) + } + + pub(crate) fn push_and_count( + &mut self, + flags: TxFeatureFlags, + counts: &mut TransactionCounts, + ) { + $( + let is_set = flags.is_set(TxFeatureFlags::$flag); + self.$vector.push(StoredBool::from(is_set)); + $(counts.$count += is_set as u64;)? + ) + + } + + pub fn truncate( + &mut self, + height: Height, + tx_index: TxIndex, + stamp: Stamp, + ) -> Result<()> { + self.count.truncate(height, stamp)?; + $(self.$vector.truncate_if_needed_with_stamp(tx_index, stamp)?;) + + Ok(()) + } + + pub fn par_iter_mut_any( + &mut self, + ) -> impl ParallelIterator { + [$( &mut self.$vector as &mut dyn AnyStoredVec, ) +] + .into_par_iter() + .chain(self.count.par_iter_mut_any()) + } + + pub fn iter_any(&self) -> impl Iterator { + [$( &self.$vector as &dyn AnyStoredVec, ) +] + .into_iter() + .chain(self.count.iter_any()) + } + } + }; +} + +with_transaction_features!(define_vecs); diff --git a/crates/brk_indexer/src/vecs/transactions/features/schema.rs b/crates/brk_indexer/src/vecs/transactions/features/schema.rs new file mode 100644 index 000000000..96dd1a9f8 --- /dev/null +++ b/crates/brk_indexer/src/vecs/transactions/features/schema.rs @@ -0,0 +1,31 @@ +macro_rules! with_transaction_features { + ($macro:ident) => { + $macro! { + has_p2pk: P2PK = 0, count: p2pk; + has_p2ms: P2MS = 1, count: p2ms; + has_p2pkh: P2PKH = 2, count: p2pkh; + has_p2sh: P2SH = 3, count: p2sh; + has_p2wpkh: P2WPKH = 4, count: p2wpkh; + has_p2wsh: P2WSH = 5, count: p2wsh; + has_p2tr: P2TR = 6, count: p2tr; + has_p2a: P2A = 7, count: p2a; + has_op_return: OP_RETURN = 8, count: op_return; + has_empty: EMPTY = 9, count: empty; + has_unknown: UNKNOWN = 10, count: unknown; + has_fake_pubkey: FAKE_PUBKEY = 11, count: fake_pubkey; + has_fake_scripthash: FAKE_SCRIPTHASH = 12, count: fake_scripthash; + has_inscription: INSCRIPTION = 13, count: inscription; + has_annex: ANNEX = 14, count: annex; + has_sighash_all: SIGHASH_ALL = 15, count: sighash_all; + has_sighash_none: SIGHASH_NONE = 16, count: sighash_none; + has_sighash_single: SIGHASH_SINGLE = 17, count: sighash_single; + has_sighash_default: SIGHASH_DEFAULT = 18, count: sighash_default; + has_sighash_anyone_can_pay: SIGHASH_ANYONE_CAN_PAY = 19, count: sighash_anyone_can_pay; + #[traversable(hidden)] + is_unconditionally_nonstandard: UNCONDITIONALLY_NONSTANDARD = 20; + has_dust_output: DUST_OUTPUT = 21, count: dust_output; + } + }; +} + +pub(super) use with_transaction_features; diff --git a/crates/brk_indexer/src/vecs/transactions.rs b/crates/brk_indexer/src/vecs/transactions/mod.rs similarity index 90% rename from crates/brk_indexer/src/vecs/transactions.rs rename to crates/brk_indexer/src/vecs/transactions/mod.rs index 07e35db5a..2817668ad 100644 --- a/crates/brk_indexer/src/vecs/transactions.rs +++ b/crates/brk_indexer/src/vecs/transactions/mod.rs @@ -1,8 +1,13 @@ +mod features; + +pub use features::{TransactionCountVecs, TransactionFeaturesVecs}; +pub(crate) use features::{TransactionCounts, TxFeatureFlags}; + use brk_error::Result; use brk_traversable::Traversable; use brk_types::{ BlkPosition, Height, RawLockTime, SigOps, StoredBool, StoredU32, TxInIndex, TxIndex, - TxOutIndex, TxVersion, Txid, Version, + TxOutIndex, TxVersion, Txid, Version, Weight, }; use rayon::prelude::*; use vecdb::{ @@ -17,7 +22,7 @@ pub struct TransactionsVecs { pub txid: M::Stored>, pub tx_version: M::Stored>, pub raw_locktime: M::Stored>, - pub base_size: M::Stored>, + pub weight: M::Stored>, pub total_size: M::Stored>, pub total_sigop_cost: M::Stored>, pub is_explicitly_rbf: M::Stored>, @@ -31,7 +36,7 @@ pub struct TxMetadataVecs<'a> { pub tx_version: &'a mut PcoVec, pub txid: &'a mut BytesVec, pub raw_locktime: &'a mut PcoVec, - pub base_size: &'a mut PcoVec, + pub weight: &'a mut PcoVec, pub total_size: &'a mut PcoVec, pub total_sigop_cost: &'a mut PcoVec, pub is_explicitly_rbf: &'a mut PcoVec, @@ -52,7 +57,7 @@ impl TransactionsVecs { tx_version: &mut self.tx_version, txid: &mut self.txid, raw_locktime: &mut self.raw_locktime, - base_size: &mut self.base_size, + weight: &mut self.weight, total_size: &mut self.total_size, total_sigop_cost: &mut self.total_sigop_cost, is_explicitly_rbf: &mut self.is_explicitly_rbf, @@ -66,7 +71,7 @@ impl TransactionsVecs { txid, tx_version, raw_locktime, - base_size, + weight, total_size, total_sigop_cost, is_explicitly_rbf, @@ -78,7 +83,7 @@ impl TransactionsVecs { txid = BytesVec::forced_import(db, "txid", version), tx_version = PcoVec::forced_import(db, "tx_version", version), raw_locktime = PcoVec::forced_import(db, "raw_locktime", version), - base_size = PcoVec::forced_import(db, "base_size", version), + weight = PcoVec::forced_import(db, "tx_weight", version), total_size = PcoVec::forced_import(db, "total_size", version), total_sigop_cost = PcoVec::forced_import(db, "total_sigop_cost", version), is_explicitly_rbf = PcoVec::forced_import(db, "is_explicitly_rbf", version), @@ -91,7 +96,7 @@ impl TransactionsVecs { txid, tx_version, raw_locktime, - base_size, + weight, total_size, total_sigop_cost, is_explicitly_rbf, @@ -109,8 +114,7 @@ impl TransactionsVecs { .truncate_if_needed_with_stamp(tx_index, stamp)?; self.raw_locktime .truncate_if_needed_with_stamp(tx_index, stamp)?; - self.base_size - .truncate_if_needed_with_stamp(tx_index, stamp)?; + self.weight.truncate_if_needed_with_stamp(tx_index, stamp)?; self.total_size .truncate_if_needed_with_stamp(tx_index, stamp)?; self.total_sigop_cost @@ -132,7 +136,7 @@ impl TransactionsVecs { &mut self.txid, &mut self.tx_version, &mut self.raw_locktime, - &mut self.base_size, + &mut self.weight, &mut self.total_size, &mut self.total_sigop_cost, &mut self.is_explicitly_rbf, @@ -149,7 +153,7 @@ impl TransactionsVecs { &self.txid, &self.tx_version, &self.raw_locktime, - &self.base_size, + &self.weight, &self.total_size, &self.total_sigop_cost, &self.is_explicitly_rbf, diff --git a/crates/brk_mempool/src/snapshot/cluster.rs b/crates/brk_mempool/src/snapshot/cluster.rs index 1d0fb9ed1..99a6092da 100644 --- a/crates/brk_mempool/src/snapshot/cluster.rs +++ b/crates/brk_mempool/src/snapshot/cluster.rs @@ -10,11 +10,11 @@ use smallvec::SmallVec; use super::{SnapTx, TxIndex}; -/// Matches Bitcoin Core 31's `MAX_CLUSTER_COUNT_LIMIT`. -pub const MAX_CLUSTER: usize = 64; - pub struct Cluster; +/// Matches Bitcoin Core 31's `MAX_CLUSTER_COUNT_LIMIT`. +const MAX_CLUSTER: usize = 64; + impl Cluster { /// Capped DFS over the undirected dependency graph (`parents ∪ /// children`) starting from `seed`. Returns the connected component @@ -99,11 +99,8 @@ impl Cluster { /// follows all its in-cluster parents. fn topo_sort(txs: &[SnapTx], component: &[TxIndex]) -> Vec { let n = component.len(); - let pos: FxHashMap = component - .iter() - .enumerate() - .map(|(i, &x)| (x, i)) - .collect(); + let pos: FxHashMap = + component.iter().enumerate().map(|(i, &x)| (x, i)).collect(); let mut indeg: Vec = vec![0; n]; let mut children: Vec> = vec![Vec::new(); n]; for (i, &idx) in component.iter().enumerate() { diff --git a/crates/brk_query/src/impl/cpfp.rs b/crates/brk_query/src/impl/cpfp.rs index b5496ea22..bca9a5377 100644 --- a/crates/brk_query/src/impl/cpfp.rs +++ b/crates/brk_query/src/impl/cpfp.rs @@ -1,49 +1,17 @@ -//! CPFP queries: dispatches between the live mempool path (handled by -//! `brk_mempool`) and the confirmed-tx path built here from indexer -//! and computer vecs. -//! -//! Confirmed clusters are materialized on demand by walking same-block -//! parent/child edges in `TxIndex` space (no `Transaction` -//! reconstruction, no `txid -> tx_index` lookup), then assembling the -//! wire shape directly. The seed's effective fee rate and the per-chunk -//! grouping both read precomputed `effective_fee_rate.tx_index`, which -//! carries the same chunk-rate semantics the live mempool produces. +//! CPFP queries shared by live mempool and confirmed transactions. -use brk_error::{Error, OptionData, Result}; -use brk_types::{ - CPFP_CHAIN_LIMIT, ChunkInput, CpfpCluster, CpfpClusterTx, CpfpClusterTxIndex, CpfpEntry, - CpfpInfo, FeeRate, Height, Sats, TxInIndex, TxIndex, Txid, TxidPrefix, VSize, Weight, - find_seed_chunk, linearize, -}; -use rustc_hash::{FxBuildHasher, FxHashMap}; -use smallvec::SmallVec; -use vecdb::{ReadableVec, VecIndex}; +mod confirmed; + +use brk_error::{Error, Result}; +use brk_types::{FeeRate, Txid, TxidPrefix}; +use vecdb::ReadableVec; use crate::Query; -struct WalkResult { - /// Cluster members in `[ancestors..., seed, descendants...]` order, - /// each paired with its in-cluster parent edges resolved to the - /// member's local index. The seed's local index is `ancestors.len()`. - members: Vec<(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)>, - seed_local: CpfpClusterTxIndex, -} - -struct Member { - txid: Txid, - fee: Sats, - weight: Weight, - vsize: VSize, - rate: FeeRate, - parents: SmallVec<[CpfpClusterTxIndex; 2]>, -} - impl Query { - /// CPFP cluster for `txid`. Returns the mempool cluster when the - /// txid is unconfirmed; otherwise reconstructs the confirmed - /// same-block cluster from indexer state. Works even when the - /// mempool feature is off. - pub fn cpfp(&self, txid: &Txid) -> Result { + /// Returns live mempool information when available, otherwise + /// reconstructs the confirmed same-block cluster from indexed data. + pub fn cpfp(&self, txid: &Txid) -> Result { let prefix = TxidPrefix::from(txid); if let Some(info) = self.mempool().and_then(|m| m.cpfp_info(&prefix)) { return Ok(info); @@ -51,17 +19,7 @@ impl Query { self.confirmed_cpfp(txid) } - /// Effective fee rate for `txid` using the same chunk-rate semantics - /// across paths: - /// - /// - Live mempool: snapshot's per-tx linearized `chunk_rate`. If - /// the tx is in the pool but not in the latest snapshot (e.g. - /// just added), falls back to the entry's simple `fee/vsize`. - /// - Confirmed: precomputed `effective_fee_rate.tx_index`. - /// - Graveyard-only RBF predecessor: linearized chunk rate - /// captured at burial. - /// - /// Returns `Error::UnknownTxid` for txids not seen in any of those. + /// Effective SFL chunk rate for live, confirmed, or replaced transactions. pub fn effective_fee_rate(&self, txid: &Txid) -> Result { let prefix = TxidPrefix::from(txid); @@ -71,14 +29,14 @@ impl Query { return Ok(rate); } - if let Ok(idx) = self.resolve_tx_index(txid) + if let Ok(index) = self.resolve_tx_index(txid) && let Some(rate) = self .computer() .transactions .fees .effective_fee_rate .tx_index - .collect_one(idx) + .collect_one(index) { return Ok(rate); } @@ -91,275 +49,4 @@ impl Query { Err(Error::UnknownTxid) } - - /// CPFP cluster for a confirmed tx: the connected component of - /// same-block parent/child edges, walked on demand. Per-tx - /// `effective_fee_rate.tx_index` provides each member's chunk rate. - fn confirmed_cpfp(&self, txid: &Txid) -> Result { - let tx_index = self.resolve_tx_index(txid)?; - let height = self.confirmed_status_height(tx_index)?; - let WalkResult { - members, - seed_local, - } = self.walk_same_block_cluster(tx_index, height)?; - - let resolved = self.resolve_members(&members)?; - let sigops = self - .indexer() - .vecs - .transactions - .total_sigop_cost - .collect_one(tx_index) - .data()?; - - Ok(build_cpfp_info(&resolved, seed_local, sigops)) - } - - fn resolve_members( - &self, - members: &[(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)], - ) -> Result> { - let indexer = self.indexer(); - let computer = self.computer(); - let mut base_size = indexer.vecs.transactions.base_size.cursor(); - let mut total_size = indexer.vecs.transactions.total_size.cursor(); - let mut fee_cursor = computer.transactions.fees.fee.tx_index.cursor(); - let mut rate_cursor = computer - .transactions - .fees - .effective_fee_rate - .tx_index - .cursor(); - let txid_reader = indexer.vecs.transactions.txid.reader(); - - members - .iter() - .map(|(tx_index, parents)| { - let i = tx_index.to_usize(); - let weight = - Weight::from_sizes(*base_size.get(i).data()?, *total_size.get(i).data()?); - let vsize = VSize::from(weight); - Ok(Member { - txid: txid_reader.get(i), - fee: fee_cursor.get(i).data()?, - weight, - vsize, - rate: rate_cursor.get(i).data()?, - parents: parents.clone(), - }) - }) - .collect() - } - - /// BFS the seed's same-block ancestors (via `outpoint`) and - /// descendants (via `spent.txin_index` -> `spending_tx`), capped - /// at `CPFP_CHAIN_LIMIT` each side to match Core/mempool.space. Returns members - /// laid out as `[ancestors..., seed, descendants...]` so the seed's - /// local index is `ancestors.len()`. - fn walk_same_block_cluster(&self, seed: TxIndex, height: Height) -> Result { - let indexer = self.indexer(); - let computer = self.computer(); - let safe = self.safe_lengths(); - let first_tx_index_vec = &indexer.vecs.transactions.first_tx_index; - let block_first = first_tx_index_vec.collect_one(height).data()?; - let next_height = height.incremented(); - let block_end = if next_height < safe.height { - first_tx_index_vec.collect_one(next_height).data()? - } else { - safe.tx_index - }; - let same_block = |idx: TxIndex| idx >= block_first && idx < block_end; - - let mut first_txin = indexer.vecs.transactions.first_txin_index.cursor(); - let mut first_txout = indexer.vecs.transactions.first_txout_index.cursor(); - let mut outpoint = indexer.vecs.inputs.outpoint.cursor(); - let mut spent = computer.outputs.spent.txin_index.cursor(); - let mut spending_tx = indexer.vecs.inputs.tx_index.cursor(); - - let mut walk_inputs = |tx: TxIndex| -> SmallVec<[TxIndex; 2]> { - let mut out: SmallVec<[TxIndex; 2]> = SmallVec::new(); - let Ok(start) = first_txin.get(tx.to_usize()).data() else { - return out; - }; - let Ok(end) = first_txin.get(tx.to_usize() + 1).data() else { - return out; - }; - for i in usize::from(start)..usize::from(end) { - let Ok(op) = outpoint.get(i).data() else { - continue; - }; - if op.is_coinbase() { - continue; - } - out.push(op.tx_index()); - } - out - }; - - let mut visited: FxHashMap = - FxHashMap::with_capacity_and_hasher(2 * CPFP_CHAIN_LIMIT + 1, FxBuildHasher); - visited.insert(seed, ()); - - // Ancestor BFS: each push records (tx_index, raw parent tx_indices) - // so we can filter against final cluster membership at the end. - let seed_inputs = walk_inputs(seed); - let mut ancestors: Vec<(TxIndex, SmallVec<[TxIndex; 2]>)> = Vec::new(); - let mut stack: Vec> = vec![seed_inputs.clone()]; - 'a: while let Some(parents) = stack.pop() { - for parent in parents { - if ancestors.len() >= CPFP_CHAIN_LIMIT { - break 'a; - } - if visited.insert(parent, ()).is_some() || !same_block(parent) { - continue; - } - let parent_inputs = walk_inputs(parent); - ancestors.push((parent, parent_inputs.clone())); - stack.push(parent_inputs); - } - } - - // Descendant BFS via spent outputs. - let mut descendants: Vec<(TxIndex, SmallVec<[TxIndex; 2]>)> = Vec::new(); - let mut stack: Vec = vec![seed]; - 'd: while let Some(cur) = stack.pop() { - let Ok(start) = first_txout.get(cur.to_usize()).data() else { - continue; - }; - let Ok(end) = first_txout.get(cur.to_usize() + 1).data() else { - continue; - }; - for i in usize::from(start)..usize::from(end) { - let Ok(txin_idx) = spent.get(i).data() else { - continue; - }; - if txin_idx == TxInIndex::UNSPENT { - continue; - } - let Ok(child) = spending_tx.get(usize::from(txin_idx)).data() else { - continue; - }; - if visited.insert(child, ()).is_some() || !same_block(child) { - continue; - } - descendants.push((child, walk_inputs(child))); - stack.push(child); - if descendants.len() >= CPFP_CHAIN_LIMIT { - break 'd; - } - } - } - - // Lay members out as [ancestors_reverse..., seed, descendants...] - // so parents come before children when a single ancestor chain - // walks back from seed. Reversing the BFS order is good enough - // for wire output; chunk grouping doesn't depend on it. - let ancestor_count = ancestors.len(); - let total = ancestor_count + 1 + descendants.len(); - let mut local_of: FxHashMap = - FxHashMap::with_capacity_and_hasher(total, FxBuildHasher); - let mut members: Vec<(TxIndex, SmallVec<[TxIndex; 2]>)> = Vec::with_capacity(total); - - for (tx, raw_parents) in ancestors.into_iter().rev() { - local_of.insert(tx, CpfpClusterTxIndex::from(members.len() as u32)); - members.push((tx, raw_parents)); - } - let seed_local = CpfpClusterTxIndex::from(members.len() as u32); - local_of.insert(seed, seed_local); - members.push((seed, seed_inputs)); - for (tx, raw_parents) in descendants { - local_of.insert(tx, CpfpClusterTxIndex::from(members.len() as u32)); - members.push((tx, raw_parents)); - } - - let resolved: Vec<(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)> = members - .into_iter() - .map(|(tx, raw_parents)| { - let parents: SmallVec<[CpfpClusterTxIndex; 2]> = raw_parents - .iter() - .filter_map(|p| local_of.get(p).copied()) - .collect(); - (tx, parents) - }) - .collect(); - - Ok(WalkResult { - members: resolved, - seed_local, - }) - } -} - -fn build_cpfp_info( - members: &[Member], - seed_local: CpfpClusterTxIndex, - sigops: brk_types::SigOps, -) -> CpfpInfo { - let seed_pos = u32::from(seed_local) as usize; - let seed = &members[seed_pos]; - - let ancestors: Vec = members[..seed_pos] - .iter() - .map(|m| CpfpEntry { - txid: m.txid, - weight: m.weight, - fee: m.fee, - }) - .collect(); - let descendants: Vec = members[seed_pos + 1..] - .iter() - .map(|m| CpfpEntry { - txid: m.txid, - weight: m.weight, - fee: m.fee, - }) - .collect(); - let best_descendant = descendants - .iter() - .max_by_key(|e| FeeRate::from((e.fee, e.weight))) - .cloned(); - - let (cluster, effective_fee_per_vsize) = if members.len() <= 1 { - (None, seed.rate) - } else { - let inputs: Vec> = members - .iter() - .map(|m| ChunkInput { - fee: m.fee, - vsize: m.vsize, - parents: m.parents.as_slice(), - }) - .collect(); - let chunks = linearize(&inputs); - let (chunk_index, seed_rate) = find_seed_chunk(&chunks, seed_local, seed.rate); - let cluster_txs: Vec = members - .iter() - .map(|m| CpfpClusterTx { - txid: m.txid, - weight: m.weight, - fee: m.fee, - parents: m.parents.iter().copied().collect(), - }) - .collect(); - ( - Some(CpfpCluster { - txs: cluster_txs, - chunks, - chunk_index, - }), - seed_rate, - ) - }; - - CpfpInfo { - ancestors, - best_descendant, - descendants, - effective_fee_per_vsize, - sigops, - fee: seed.fee, - vsize: seed.vsize, - adjusted_vsize: sigops.adjust_vsize(seed.vsize), - cluster, - } } diff --git a/crates/brk_query/src/impl/cpfp/confirmed.rs b/crates/brk_query/src/impl/cpfp/confirmed.rs new file mode 100644 index 000000000..ac067c29a --- /dev/null +++ b/crates/brk_query/src/impl/cpfp/confirmed.rs @@ -0,0 +1,336 @@ +use brk_error::{OptionData, Result}; +use brk_types::{ + CPFP_CHAIN_LIMIT, ChunkInput, CpfpCluster, CpfpClusterTx, CpfpClusterTxIndex, CpfpEntry, + CpfpInfo, FeeRate, Height, Sats, TxInIndex, TxIndex, Txid, VSize, Weight, find_seed_chunk, + linearize, +}; +use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet}; +use smallvec::SmallVec; +use vecdb::{ReadableVec, VecIndex}; + +use crate::Query; + +struct WalkResult { + members: Vec<(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)>, + seed_local: CpfpClusterTxIndex, + ancestors: Vec, + descendants: Vec, +} + +struct Member { + txid: Txid, + fee: Sats, + weight: Weight, + vsize: VSize, + parents: SmallVec<[CpfpClusterTxIndex; 2]>, +} + +impl Query { + pub(super) fn confirmed_cpfp(&self, txid: &Txid) -> Result { + let seed = self.resolve_tx_index(txid)?; + let height = self.confirmed_status_height(seed)?; + let walk = self.walk_same_block_cluster(seed, height)?; + let members = self.resolve_members(&walk.members)?; + let ancestors = self.resolve_entries(&walk.ancestors)?; + let descendants = self.resolve_entries(&walk.descendants)?; + let sigops = self + .indexer() + .vecs + .transactions + .total_sigop_cost + .collect_one(seed) + .data()?; + + Ok(build_cpfp_info( + &members, + walk.seed_local, + ancestors, + descendants, + sigops, + )) + } + + fn resolve_members( + &self, + members: &[(TxIndex, SmallVec<[CpfpClusterTxIndex; 2]>)], + ) -> Result> { + let indexer = self.indexer(); + let computer = self.computer(); + let mut weight = indexer.vecs.transactions.weight.cursor(); + let mut fee = computer.transactions.fees.fee.tx_index.cursor(); + let txid = indexer.vecs.transactions.txid.reader(); + + members + .iter() + .map(|(index, parents)| { + let position = index.to_usize(); + let weight = weight.get(position).data()?; + Ok(Member { + txid: txid.get(position), + fee: fee.get(position).data()?, + weight, + vsize: VSize::from(weight), + parents: parents.clone(), + }) + }) + .collect() + } + + fn resolve_entries(&self, indexes: &[TxIndex]) -> Result> { + let indexer = self.indexer(); + let computer = self.computer(); + let mut weight = indexer.vecs.transactions.weight.cursor(); + let mut fee = computer.transactions.fees.fee.tx_index.cursor(); + let txid = indexer.vecs.transactions.txid.reader(); + + indexes + .iter() + .map(|index| { + let position = index.to_usize(); + Ok(CpfpEntry { + txid: txid.get(position), + fee: fee.get(position).data()?, + weight: weight.get(position).data()?, + }) + }) + .collect() + } + + fn walk_same_block_cluster(&self, seed: TxIndex, height: Height) -> Result { + let indexer = self.indexer(); + let computer = self.computer(); + let safe = self.safe_lengths(); + let first_tx = &indexer.vecs.transactions.first_tx_index; + let block_first = first_tx.collect_one(height).data()?; + let next_height = height.incremented(); + let block_end = if next_height < safe.height { + first_tx.collect_one(next_height).data()? + } else { + safe.tx_index + }; + + let mut first_txin = indexer.vecs.transactions.first_txin_index.cursor(); + let mut input_count = computer.indexes.tx_index.input_count.cursor(); + let mut outpoint = indexer.vecs.inputs.outpoint.cursor(); + let mut first_txout = indexer.vecs.transactions.first_txout_index.cursor(); + let mut output_count = computer.indexes.tx_index.output_count.cursor(); + let mut spent = computer.outputs.spent.txin_index.cursor(); + let mut spending_tx = indexer.vecs.inputs.tx_index.cursor(); + + let mut parents_of = |tx: TxIndex| -> Result> { + let position = tx.to_usize(); + let first = usize::from(first_txin.get(position).data()?); + let count = u64::from(input_count.get(position).data()?) as usize; + let mut parents = SmallVec::new(); + for input in first..first + count { + let parent = outpoint.get(input).data()?; + if !parent.is_coinbase() + && parent.tx_index() >= block_first + && parent.tx_index() < block_end + && !parents.contains(&parent.tx_index()) + { + parents.push(parent.tx_index()); + } + } + parents.sort_unstable(); + Ok(parents) + }; + + let mut children_of = |tx: TxIndex| -> Result> { + let position = tx.to_usize(); + let first = usize::from(first_txout.get(position).data()?); + let count = u64::from(output_count.get(position).data()?) as usize; + let mut children = SmallVec::new(); + for output in first..first + count { + let input = spent.get(output).data()?; + if input == TxInIndex::UNSPENT { + continue; + } + let child = spending_tx.get(usize::from(input)).data()?; + if child >= block_first && child < block_end && !children.contains(&child) { + children.push(child); + } + } + children.sort_unstable(); + Ok(children) + }; + + let block_tx_count = block_end.to_usize() - block_first.to_usize(); + let mut component = + walk_component(seed, &mut parents_of, &mut children_of, block_tx_count)?; + component.sort_unstable(); + let ancestors = walk_direction(seed, &mut parents_of, CPFP_CHAIN_LIMIT)?; + let descendants = walk_direction(seed, &mut children_of, CPFP_CHAIN_LIMIT)?; + + let local_of: FxHashMap = component + .iter() + .enumerate() + .map(|(local, &tx)| (tx, CpfpClusterTxIndex::from(local as u32))) + .collect(); + let seed_local = local_of[&seed]; + let members = component + .into_iter() + .map(|tx| { + let parents = parents_of(tx)? + .iter() + .filter_map(|parent| local_of.get(parent).copied()) + .collect(); + Ok((tx, parents)) + }) + .collect::>()?; + + Ok(WalkResult { + members, + seed_local, + ancestors, + descendants, + }) + } +} + +fn walk_component( + seed: TxIndex, + parents: &mut impl FnMut(TxIndex) -> Result>, + children: &mut impl FnMut(TxIndex) -> Result>, + limit: usize, +) -> Result> { + let mut visited = FxHashSet::with_capacity_and_hasher(limit, FxBuildHasher); + visited.insert(seed); + let mut members = Vec::with_capacity(limit); + members.push(seed); + let mut stack = vec![seed]; + + while let Some(tx) = stack.pop() { + for neighbor in parents(tx)?.into_iter().chain(children(tx)?) { + if visited.insert(neighbor) { + if members.len() == limit { + return Ok(members); + } + members.push(neighbor); + stack.push(neighbor); + } + } + } + Ok(members) +} + +fn walk_direction( + seed: TxIndex, + next: &mut impl FnMut(TxIndex) -> Result>, + limit: usize, +) -> Result> { + let mut visited = FxHashSet::with_capacity_and_hasher(limit + 1, FxBuildHasher); + visited.insert(seed); + let mut members = Vec::with_capacity(limit); + let mut stack = next(seed)?.into_vec(); + + while let Some(tx) = stack.pop() { + if !visited.insert(tx) { + continue; + } + members.push(tx); + if members.len() == limit { + break; + } + stack.extend(next(tx)?); + } + Ok(members) +} + +fn build_cpfp_info( + members: &[Member], + seed_local: CpfpClusterTxIndex, + ancestors: Vec, + descendants: Vec, + sigops: brk_types::SigOps, +) -> CpfpInfo { + let seed_position = u32::from(seed_local) as usize; + let seed = &members[seed_position]; + let raw_rate = FeeRate::from((seed.fee, seed.vsize)); + let best_descendant = descendants + .iter() + .max_by_key(|entry| FeeRate::from((entry.fee, entry.weight))) + .cloned(); + + let (cluster, effective_fee_per_vsize) = if members.len() == 1 { + (None, raw_rate) + } else { + let inputs: Vec> = members + .iter() + .map(|member| ChunkInput { + fee: member.fee, + vsize: member.vsize, + parents: member.parents.as_slice(), + }) + .collect(); + let chunks = linearize(&inputs); + let (chunk_index, rate) = find_seed_chunk(&chunks, seed_local, raw_rate); + let txs = members + .iter() + .map(|member| CpfpClusterTx { + txid: member.txid, + weight: member.weight, + fee: member.fee, + parents: member.parents.iter().copied().collect(), + }) + .collect(); + ( + Some(CpfpCluster { + txs, + chunks, + chunk_index, + }), + rate, + ) + }; + + CpfpInfo { + ancestors, + best_descendant, + descendants, + effective_fee_per_vsize, + sigops, + fee: seed.fee, + vsize: seed.vsize, + adjusted_vsize: sigops.adjust_vsize(seed.vsize), + cluster, + } +} + +#[cfg(test)] +mod tests { + use brk_types::TxIndex; + use vecdb::VecIndex; + + use super::{walk_component, walk_direction}; + + fn adjacent( + graph: &[Vec], + tx: TxIndex, + ) -> brk_error::Result> { + Ok(graph[tx.to_usize()] + .iter() + .copied() + .map(TxIndex::from) + .collect()) + } + + #[test] + fn component_includes_siblings_but_directional_walk_does_not() { + let parents = vec![vec![], vec![0], vec![0]]; + let children = vec![vec![1, 2], vec![], vec![]]; + let seed = TxIndex::from(1usize); + let mut parent = |tx| adjacent(&parents, tx); + let mut child = |tx| adjacent(&children, tx); + + let mut component = walk_component(seed, &mut parent, &mut child, 64).unwrap(); + component.sort_unstable(); + assert_eq!(component, [0usize, 1, 2].map(TxIndex::from)); + + assert_eq!( + walk_direction(seed, &mut parent, 25).unwrap(), + [TxIndex::from(0usize)] + ); + assert!(walk_direction(seed, &mut child, 25).unwrap().is_empty()); + } +} diff --git a/crates/brk_query/src/impl/series.rs b/crates/brk_query/src/impl/series.rs index 1ccf7c46e..624a2ca0c 100644 --- a/crates/brk_query/src/impl/series.rs +++ b/crates/brk_query/src/impl/series.rs @@ -235,12 +235,7 @@ impl Query { Index::TxInIndex => v.inputs.first_txin_index.collect_one(h).map(usize::from), Index::TxOutIndex => v.outputs.first_txout_index.collect_one(h).map(usize::from), Index::EmptyOutputIndex => v.scripts.empty.first_index.collect_one(h).map(usize::from), - Index::OpReturnIndex => v - .scripts - .op_return - .first_index - .collect_one(h) - .map(usize::from), + Index::OpReturnIndex => v.op_return.first_index.collect_one(h).map(usize::from), Index::P2MSOutputIndex => v.scripts.p2ms.first_index.collect_one(h).map(usize::from), Index::UnknownOutputIndex => v .scripts diff --git a/crates/brk_store/Cargo.toml b/crates/brk_store/Cargo.toml index 59b945ec7..70d6fe36f 100644 --- a/crates/brk_store/Cargo.toml +++ b/crates/brk_store/Cargo.toml @@ -16,3 +16,4 @@ brk_types = { workspace = true } byteview = { workspace = true } fjall = { workspace = true } rustc-hash = { workspace = true } +tracing = { workspace = true } diff --git a/crates/brk_store/src/lib.rs b/crates/brk_store/src/lib.rs index 7dd72c870..bfbe83a05 100644 --- a/crates/brk_store/src/lib.rs +++ b/crates/brk_store/src/lib.rs @@ -1,12 +1,25 @@ #![doc = include_str!("../README.md")] -use std::{borrow::Cow, fmt::Debug, fs, hash::Hash, mem, ops::Range, path::Path}; +use std::{ + borrow::Cow, + fmt::Debug, + fs, + hash::Hash, + mem, + ops::Range, + path::Path, + sync::{ + Arc, + atomic::{AtomicU64, Ordering::Relaxed}, + }, +}; use brk_error::Result; use brk_types::{Height, Version}; use byteview::ByteView; use fjall::{Database, Keyspace, KeyspaceCreateOptions, config::*}; use rustc_hash::{FxHashMap, FxHashSet}; +use tracing::info; mod any; mod item; @@ -37,6 +50,7 @@ pub struct Store { puts: FxHashMap, dels: FxHashSet, caches: Vec>, + db_reads: Arc, } impl Store @@ -103,6 +117,7 @@ where puts: FxHashMap::default(), dels: FxHashSet::default(), caches, + db_reads: Arc::new(AtomicU64::new(0)), }) } @@ -164,6 +179,8 @@ where } } + self.db_reads.fetch_add(1, Relaxed); + if let Some(slice) = self.keyspace.get(ByteView::from(key))? { Ok(Some(Cow::Owned(V::from(ByteView::from(slice))))) } else { @@ -212,6 +229,7 @@ where for<'a> ByteView: From<&'a K> + From<&'a V>, { self.export_meta_if_needed(height)?; + self.report_db_reads(); let puts = mem::take(&mut self.puts); let dels = mem::take(&mut self.dels); @@ -286,6 +304,13 @@ where Ok(()) } + fn report_db_reads(&self) { + let reads = self.db_reads.swap(0, Relaxed); + if reads != 0 { + info!(store = self.name, reads, "Store DB reads"); + } + } + fn ingest<'a>( keyspace: &Keyspace, puts: impl Iterator, @@ -361,6 +386,7 @@ where fn commit(&mut self, height: Height) -> Result<()> { self.export_meta_if_needed(height)?; + self.report_db_reads(); let puts = mem::take(&mut self.puts); let dels = mem::take(&mut self.dels); diff --git a/crates/brk_types/src/addr_bytes.rs b/crates/brk_types/src/addr_bytes.rs index 58965ae40..a22a82b06 100644 --- a/crates/brk_types/src/addr_bytes.rs +++ b/crates/brk_types/src/addr_bytes.rs @@ -38,6 +38,37 @@ impl AddrBytes { rapidhash::v3::rapidhash_v3(self.as_slice()) } + pub(crate) fn script_payload( + script: &ScriptBuf, + output_type: OutputType, + ) -> Result<&[u8], Error> { + let bytes = script.as_bytes(); + match output_type { + OutputType::P2PK65 => match bytes.len() { + 67 => Ok(&bytes[1..66]), + received => Err(Error::WrongLength { + expected: 67, + received, + }), + }, + OutputType::P2PK33 => match bytes.len() { + 35 => Ok(&bytes[1..34]), + received => Err(Error::WrongLength { + expected: 35, + received, + }), + }, + OutputType::P2PKH => Ok(&bytes[3..23]), + OutputType::P2SH => Ok(&bytes[2..22]), + OutputType::P2WPKH | OutputType::P2WSH | OutputType::P2TR | OutputType::P2A => { + Ok(&bytes[2..]) + } + OutputType::P2MS | OutputType::Unknown | OutputType::Empty | OutputType::OpReturn => { + Err(Error::WrongAddrType) + } + } + } + /// Reconstruct the script_pubkey from the address bytes pub fn to_script_pubkey(&self) -> ScriptBuf { match self { @@ -80,64 +111,21 @@ impl TryFrom<(&ScriptBuf, OutputType)> for AddrBytes { type Error = Error; fn try_from(tuple: (&ScriptBuf, OutputType)) -> Result { let (script, output_type) = tuple; + let bytes = Self::script_payload(script, output_type)?; - match output_type { - OutputType::P2PK65 => { - let bytes = script.as_bytes(); - let bytes = match bytes.len() { - 67 => &bytes[1..66], - len => { - dbg!(bytes); - return Err(Error::WrongLength { - expected: 67, - received: len, - }); - } - }; - Ok(Self::P2PK65(P2PK65Bytes::from(bytes))) - } - OutputType::P2PK33 => { - let bytes = script.as_bytes(); - let bytes = match bytes.len() { - 35 => &bytes[1..34], - len => { - dbg!(bytes); - return Err(Error::WrongLength { - expected: 35, - received: len, - }); - } - }; - Ok(Self::P2PK33(P2PK33Bytes::from(bytes))) - } - OutputType::P2PKH => { - let bytes = &script.as_bytes()[3..23]; - Ok(Self::P2PKH(P2PKHBytes::from(bytes))) - } - OutputType::P2SH => { - let bytes = &script.as_bytes()[2..22]; - Ok(Self::P2SH(P2SHBytes::from(bytes))) - } - OutputType::P2WPKH => { - let bytes = &script.as_bytes()[2..]; - Ok(Self::P2WPKH(P2WPKHBytes::from(bytes))) - } - OutputType::P2WSH => { - let bytes = &script.as_bytes()[2..]; - Ok(Self::P2WSH(P2WSHBytes::from(bytes))) - } - OutputType::P2TR => { - let bytes = &script.as_bytes()[2..]; - Ok(Self::P2TR(P2TRBytes::from(bytes))) - } - OutputType::P2A => { - let bytes = &script.as_bytes()[2..]; - Ok(Self::P2A(P2ABytes::from(bytes))) - } + Ok(match output_type { + OutputType::P2PK65 => Self::P2PK65(P2PK65Bytes::from(bytes)), + OutputType::P2PK33 => Self::P2PK33(P2PK33Bytes::from(bytes)), + OutputType::P2PKH => Self::P2PKH(P2PKHBytes::from(bytes)), + OutputType::P2SH => Self::P2SH(P2SHBytes::from(bytes)), + OutputType::P2WPKH => Self::P2WPKH(P2WPKHBytes::from(bytes)), + OutputType::P2WSH => Self::P2WSH(P2WSHBytes::from(bytes)), + OutputType::P2TR => Self::P2TR(P2TRBytes::from(bytes)), + OutputType::P2A => Self::P2A(P2ABytes::from(bytes)), OutputType::P2MS | OutputType::Unknown | OutputType::Empty | OutputType::OpReturn => { - Err(Error::WrongAddrType) + unreachable!() } - } + }) } } diff --git a/crates/brk_types/src/addr_hash.rs b/crates/brk_types/src/addr_hash.rs index cce87c417..ce436c34b 100644 --- a/crates/brk_types/src/addr_hash.rs +++ b/crates/brk_types/src/addr_hash.rs @@ -1,8 +1,10 @@ +use bitcoin::ScriptBuf; +use brk_error::Error; use byteview::ByteView; use derive_more::Deref; use vecdb::Bytes; -use super::AddrBytes; +use super::{AddrBytes, OutputType}; #[derive(Debug, Deref, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Bytes, Hash)] pub struct AddrHash(u64); @@ -11,6 +13,13 @@ impl AddrHash { pub const fn new(value: u64) -> Self { Self(value) } + + #[inline] + pub fn from_script(script: &ScriptBuf, output_type: OutputType) -> Result { + Ok(Self(rapidhash::v3::rapidhash_v3( + AddrBytes::script_payload(script, output_type)?, + ))) + } } impl From<&AddrBytes> for AddrHash { @@ -38,3 +47,40 @@ impl From<&AddrHash> for ByteView { Self::new(&value.0.to_be_bytes()) } } + +#[cfg(test)] +mod tests { + use super::AddrHash; + use crate::{ + AddrBytes, OutputType, P2ABytes, P2PK33Bytes, P2PK65Bytes, P2PKHBytes, P2SHBytes, + P2TRBytes, P2WPKHBytes, P2WSHBytes, + }; + + #[test] + fn hashes_borrowed_script_payloads_like_owned_addresses() { + let addresses = [ + AddrBytes::from(P2PK65Bytes::from(&[0x04; 65][..])), + AddrBytes::from(P2PK33Bytes::from(&[0x02; 33][..])), + AddrBytes::from(P2PKHBytes::from(&[0x03; 20][..])), + AddrBytes::from(P2SHBytes::from(&[0x04; 20][..])), + AddrBytes::from(P2WPKHBytes::from(&[0x05; 20][..])), + AddrBytes::from(P2WSHBytes::from(&[0x06; 32][..])), + AddrBytes::from(P2TRBytes::from(&[0x07; 32][..])), + AddrBytes::from(P2ABytes::from(&[78, 115][..])), + ]; + + for address in addresses { + let script = address.to_script_pubkey(); + let output_type = OutputType::from(&script); + + assert_eq!( + AddrHash::from_script(&script, output_type).unwrap(), + AddrHash::from(&address) + ); + assert_eq!( + AddrBytes::try_from((&script, output_type)).unwrap(), + address + ); + } + } +} diff --git a/crates/brk_types/src/lib.rs b/crates/brk_types/src/lib.rs index 3533f0acd..0b264d193 100644 --- a/crates/brk_types/src/lib.rs +++ b/crates/brk_types/src/lib.rs @@ -98,6 +98,7 @@ mod month6; mod next_block_hash; mod ohlc; mod op_return_index; +mod op_return_kind; mod option_ext; mod outpoint; mod outpoint_prefix; @@ -296,6 +297,7 @@ pub use month6::*; pub use next_block_hash::*; pub use ohlc::*; pub use op_return_index::*; +pub use op_return_kind::*; pub use option_ext::*; pub use outpoint::*; pub use outpoint_prefix::*; diff --git a/crates/brk_types/src/op_return_kind.rs b/crates/brk_types/src/op_return_kind.rs new file mode 100644 index 000000000..425189ed7 --- /dev/null +++ b/crates/brk_types/src/op_return_kind.rs @@ -0,0 +1,98 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use strum::{AsRefStr, Display}; +use vecdb::{Bytes, Formattable, Pco, TransparentPco}; + +#[derive( + Debug, + Clone, + Copy, + AsRefStr, + Display, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + JsonSchema, + Hash, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +#[repr(u8)] +pub enum OpReturnKind { + Runes, + VeriBlock, + Omni, + Stacks, + Blockstack, + Colu, + OpenAssets, + Komodo, + CoinSpark, + Poet, + Docproof, + OpenTimestamps, + Factom, + EternityWall, + Memo, + Bitproof, + Ascribe, + Stampery, + Epobc, + BareHash, + Text, + Empty, + Unknown, +} + +impl OpReturnKind { + fn is_valid(value: u8) -> bool { + value <= Self::Unknown as u8 + } +} + +impl Formattable for OpReturnKind { + #[inline(always)] + fn write_to(&self, buf: &mut Vec) { + buf.extend_from_slice(self.as_ref().as_bytes()); + } + + fn fmt_json(&self, buf: &mut Vec) { + buf.push(b'"'); + self.write_to(buf); + buf.push(b'"'); + } +} + +impl Bytes for OpReturnKind { + type Array = [u8; size_of::()]; + + #[inline] + fn to_bytes(&self) -> Self::Array { + [*self as u8] + } + + #[inline] + fn from_bytes(bytes: &[u8]) -> vecdb::Result { + if bytes.len() != size_of::() { + return Err(vecdb::Error::WrongLength { + expected: size_of::(), + received: bytes.len(), + }); + } + let value = bytes[0]; + if !Self::is_valid(value) { + return Err(vecdb::Error::InvalidArgument("invalid OpReturnKind")); + } + // SAFETY: We validated that value is a valid variant. + Ok(unsafe { std::mem::transmute::(value) }) + } +} + +impl Pco for OpReturnKind { + type NumberType = u8; +} + +impl TransparentPco for OpReturnKind {} diff --git a/crates/brk_types/src/output_type.rs b/crates/brk_types/src/output_type.rs index eb46c153f..9949cb77b 100644 --- a/crates/brk_types/src/output_type.rs +++ b/crates/brk_types/src/output_type.rs @@ -3,7 +3,14 @@ reason = "P2PK65 and P2PK33 both serialize as 'p2pk'" )] -use bitcoin::{AddressType, ScriptBuf, opcodes::all::OP_PUSHBYTES_2}; +use bitcoin::{ + AddressType, ScriptBuf, + opcodes::all::{ + OP_CHECKSIG, OP_DUP, OP_EQUAL, OP_EQUALVERIFY, OP_HASH160, OP_PUSHBYTES_0, OP_PUSHBYTES_2, + OP_PUSHBYTES_20, OP_PUSHBYTES_32, OP_PUSHBYTES_33, OP_PUSHBYTES_65, OP_PUSHNUM_1, + OP_RETURN, + }, +}; use brk_error::Error; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -147,41 +154,43 @@ impl OutputType { impl From<&ScriptBuf> for OutputType { #[inline] fn from(script: &ScriptBuf) -> Self { - if script.is_p2pkh() { - Self::P2PKH - } else if script.is_p2wpkh() { - Self::P2WPKH - } else if script.is_p2wsh() { - Self::P2WSH - } else if script.is_p2tr() { - Self::P2TR - } else if script.is_p2sh() { - Self::P2SH - } else if script.is_op_return() { - Self::OpReturn - } else if script.is_p2pk() { - let bytes = script.as_bytes(); + let bytes = script.as_bytes(); - match bytes.len() { - 67 => Self::P2PK65, - 35 => Self::P2PK33, - _ => { - dbg!(bytes); - unreachable!() - } + match bytes.len() { + 0 => Self::Empty, + 4 if bytes == [OP_PUSHNUM_1.to_u8(), OP_PUSHBYTES_2.to_u8(), 78, 115] => Self::P2A, + 22 if bytes[0] == OP_PUSHBYTES_0.to_u8() && bytes[1] == OP_PUSHBYTES_20.to_u8() => { + Self::P2WPKH } - } else if script.is_multisig() { - Self::P2MS - } else if script.witness_version() == Some(bitcoin::WitnessVersion::V1) - && script.len() == 4 - && script.as_bytes()[1] == OP_PUSHBYTES_2.to_u8() - && script.as_bytes()[2..4] == [78, 115] - { - Self::P2A - } else if script.is_empty() { - Self::Empty - } else { - Self::Unknown + 23 if bytes[0] == OP_HASH160.to_u8() + && bytes[1] == OP_PUSHBYTES_20.to_u8() + && bytes[22] == OP_EQUAL.to_u8() => + { + Self::P2SH + } + 25 if bytes[0] == OP_DUP.to_u8() + && bytes[1] == OP_HASH160.to_u8() + && bytes[2] == OP_PUSHBYTES_20.to_u8() + && bytes[23] == OP_EQUALVERIFY.to_u8() + && bytes[24] == OP_CHECKSIG.to_u8() => + { + Self::P2PKH + } + 34 if bytes[0] == OP_PUSHBYTES_0.to_u8() && bytes[1] == OP_PUSHBYTES_32.to_u8() => { + Self::P2WSH + } + 34 if bytes[0] == OP_PUSHNUM_1.to_u8() && bytes[1] == OP_PUSHBYTES_32.to_u8() => { + Self::P2TR + } + 35 if bytes[0] == OP_PUSHBYTES_33.to_u8() && bytes[34] == OP_CHECKSIG.to_u8() => { + Self::P2PK33 + } + 67 if bytes[0] == OP_PUSHBYTES_65.to_u8() && bytes[66] == OP_CHECKSIG.to_u8() => { + Self::P2PK65 + } + _ if bytes[0] == OP_RETURN.to_u8() => Self::OpReturn, + _ if script.is_multisig() => Self::P2MS, + _ => Self::Unknown, } } } diff --git a/crates/brk_types/src/tx.rs b/crates/brk_types/src/tx.rs index 5cf5edc43..070952fca 100644 --- a/crates/brk_types/src/tx.rs +++ b/crates/brk_types/src/tx.rs @@ -2,7 +2,7 @@ use crate::{ FeeRate, RawLockTime, Sats, SigOps, TxIn, TxIndex, TxOut, TxStatus, TxVersionRaw, Txid, VSize, Weight, Witness, }; -use bitcoin::Script; +use bitcoin::{Script, constants::WITNESS_SCALE_FACTOR}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use vecdb::CheckedSub; @@ -153,8 +153,8 @@ impl Transaction { SigOps::from( legacy - .saturating_mul(4) - .saturating_add(redeem.saturating_mul(4)) + .saturating_mul(WITNESS_SCALE_FACTOR) + .saturating_add(redeem.saturating_mul(WITNESS_SCALE_FACTOR)) .saturating_add(witness), ) } diff --git a/crates/brk_types/src/weight.rs b/crates/brk_types/src/weight.rs index f2297a72a..753201875 100644 --- a/crates/brk_types/src/weight.rs +++ b/crates/brk_types/src/weight.rs @@ -38,15 +38,17 @@ pub struct Weight(u64); impl Weight { /// Maximum block weight in Bitcoin (4 million weight units). /// Note: Pre-SegWit 1MB blocks have weight = size * 4 = 4M, so this is consistent across all blocks. - pub const MAX_BLOCK: Self = Self(4_000_000); + pub const MAX_BLOCK: Self = Self(bitcoin::Weight::MAX_BLOCK.to_wu()); /// Compute weight from base size and total size. - /// Formula: weight = base_size * 3 + total_size - /// (since total_size = base_size + witness_size, this equals base_size * 4 + witness_size) #[inline] pub fn from_sizes(base_size: u32, total_size: u32) -> Self { - let wu = base_size as u64 * 3 + total_size as u64; - Self(wu) + let base_size = u64::from(base_size); + let witness_size = u64::from(total_size) - base_size; + Self::from( + bitcoin::Weight::from_non_witness_data_size(base_size) + + bitcoin::Weight::from_witness_data_size(witness_size), + ) } pub fn to_vbytes_ceil(&self) -> u64 { @@ -183,3 +185,14 @@ impl Formattable for Weight { buf.extend_from_slice(b.format(self.0).as_bytes()); } } + +#[cfg(test)] +mod tests { + use super::Weight; + + #[test] + fn derives_weight_from_base_and_witness_sizes() { + assert_eq!(*Weight::from_sizes(100, 100), 400); + assert_eq!(*Weight::from_sizes(100, 125), 425); + } +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 2d45363a5..398f3f015 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.96.1" +channel = "1.97.0"