mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-07-19 23:18:10 -07:00
Compare commits
67 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3af673132a | |||
| 0fc61d7932 | |||
| 5161ae2012 | |||
| 10c16335ed | |||
| 29ad37803e | |||
| 8d43f340a0 | |||
| e279116bff | |||
| 5c376acb9f | |||
| a43b00c12c | |||
| 15fbdc168f | |||
| 046c624abf | |||
| 4d7212fb63 | |||
| 2207ec1b7e | |||
| 3f2a48f248 | |||
| 3f9edb211e | |||
| eee1a10d2a | |||
| 0bf2cd77dc | |||
| a99c06013b | |||
| 10ef95497f | |||
| e0a618837e | |||
| 5bd1f0b625 | |||
| 4d3e8bbcaf | |||
| 153fcdf4e0 | |||
| 5d83ee4d70 | |||
| 10d29aa2ea | |||
| 528880bd04 | |||
| 5937fbe724 | |||
| 7a718293c0 | |||
| 75f37b6695 | |||
| f9688629ca | |||
| c9e704bdf4 | |||
| 1e0a7d6d0f | |||
| 565251f12f | |||
| 9879a986aa | |||
| 8d7ef8da61 | |||
| 17478a4ac4 | |||
| b3031b3375 | |||
| 2e401379a0 | |||
| 45ab6ebf71 | |||
| 00f7d69ea6 | |||
| 408d83c350 | |||
| 43df9e098c | |||
| 0c7861071d | |||
| 6f430bdb8c | |||
| 4b415b215d | |||
| 8614e9eded | |||
| c85da92cbc | |||
| 297fc3b855 | |||
| c9d5a62fcb | |||
| 90b3b51c48 | |||
| 5966ab05e4 | |||
| c3506339cd | |||
| e54843291e | |||
| b0b261fe9f | |||
| 6786be296d | |||
| e5068bbbf3 | |||
| 36cfe49b20 | |||
| 33cc13708a | |||
| 2389632812 | |||
| e0bcdb8105 | |||
| 45e83c98b9 | |||
| 753bbf3e7e | |||
| 54cc0cb446 | |||
| d64dcb75a9 | |||
| f599115f6c | |||
| 9fc45625ad | |||
| c68d1d1fda |
@@ -26,6 +26,7 @@ _*
|
||||
/playground
|
||||
/*.txt
|
||||
/*.csv
|
||||
/tmp
|
||||
|
||||
# Logs
|
||||
*.log*
|
||||
|
||||
Generated
+255
-398
File diff suppressed because it is too large
Load Diff
+35
-33
@@ -4,7 +4,7 @@ members = ["crates/*"]
|
||||
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
|
||||
package.license = "MIT"
|
||||
package.edition = "2024"
|
||||
package.version = "0.3.3"
|
||||
package.version = "0.3.6"
|
||||
package.homepage = "https://bitcoinresearchkit.org"
|
||||
package.repository = "https://github.com/bitcoinresearchkit/brk"
|
||||
package.readme = "README.md"
|
||||
@@ -35,57 +35,59 @@ debug = true
|
||||
[workspace.dependencies]
|
||||
aide = { version = "0.16.0-alpha.4", features = ["axum-json", "axum-query"] }
|
||||
axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "query", "tokio", "tracing"] }
|
||||
bitcoin = { version = "0.32.100", features = ["serde"] }
|
||||
brk_alloc = { version = "0.3.3", path = "crates/brk_alloc" }
|
||||
brk_bencher = { version = "0.3.3", path = "crates/brk_bencher" }
|
||||
brk_bindgen = { version = "0.3.3", path = "crates/brk_bindgen" }
|
||||
brk_cli = { version = "0.3.3", path = "crates/brk_cli" }
|
||||
brk_client = { version = "0.3.3", path = "crates/brk_client" }
|
||||
brk_cohort = { version = "0.3.3", path = "crates/brk_cohort" }
|
||||
brk_computer = { version = "0.3.3", path = "crates/brk_computer" }
|
||||
brk_error = { version = "0.3.3", path = "crates/brk_error" }
|
||||
brk_fetcher = { version = "0.3.3", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0.3.3", path = "crates/brk_indexer" }
|
||||
brk_iterator = { version = "0.3.3", path = "crates/brk_iterator" }
|
||||
brk_logger = { version = "0.3.3", path = "crates/brk_logger" }
|
||||
brk_mempool = { version = "0.3.3", path = "crates/brk_mempool" }
|
||||
brk_oracle = { version = "0.3.3", path = "crates/brk_oracle" }
|
||||
brk_query = { version = "0.3.3", path = "crates/brk_query", features = ["tokio"] }
|
||||
brk_reader = { version = "0.3.3", path = "crates/brk_reader" }
|
||||
brk_rpc = { version = "0.3.3", path = "crates/brk_rpc" }
|
||||
brk_server = { version = "0.3.3", path = "crates/brk_server" }
|
||||
brk_store = { version = "0.3.3", path = "crates/brk_store" }
|
||||
brk_traversable = { version = "0.3.3", path = "crates/brk_traversable", features = ["pco", "derive"] }
|
||||
brk_traversable_derive = { version = "0.3.3", path = "crates/brk_traversable_derive" }
|
||||
brk_types = { version = "0.3.3", path = "crates/brk_types" }
|
||||
brk_website = { version = "0.3.3", path = "crates/brk_website" }
|
||||
bitcoin = { version = "0.32.102", features = ["serde"] }
|
||||
brk_alloc = { version = "0.3.6", path = "crates/brk_alloc" }
|
||||
brk_bencher = { version = "0.3.6", path = "crates/brk_bencher" }
|
||||
brk_bindgen = { version = "0.3.6", path = "crates/brk_bindgen" }
|
||||
brk_cli = { version = "0.3.6", path = "crates/brk_cli" }
|
||||
brk_client = { version = "0.3.6", path = "crates/brk_client" }
|
||||
brk_cohort = { version = "0.3.6", path = "crates/brk_cohort" }
|
||||
brk_computer = { version = "0.3.6", path = "crates/brk_computer" }
|
||||
brk_error = { version = "0.3.6", path = "crates/brk_error" }
|
||||
brk_fetcher = { version = "0.3.6", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0.3.6", path = "crates/brk_indexer" }
|
||||
brk_iterator = { version = "0.3.6", path = "crates/brk_iterator" }
|
||||
brk_logger = { version = "0.3.6", path = "crates/brk_logger" }
|
||||
brk_mempool = { version = "0.3.6", path = "crates/brk_mempool" }
|
||||
brk_oracle = { version = "0.3.6", path = "crates/brk_oracle" }
|
||||
brk_query = { version = "0.3.6", path = "crates/brk_query", features = ["tokio"] }
|
||||
brk_reader = { version = "0.3.6", path = "crates/brk_reader" }
|
||||
brk_rpc = { version = "0.3.6", path = "crates/brk_rpc" }
|
||||
brk_server = { version = "0.3.6", path = "crates/brk_server" }
|
||||
brk_store = { version = "0.3.6", path = "crates/brk_store" }
|
||||
brk_traversable = { version = "0.3.6", path = "crates/brk_traversable", features = ["pco", "derive"] }
|
||||
brk_traversable_derive = { version = "0.3.6", path = "crates/brk_traversable_derive" }
|
||||
brk_types = { version = "0.3.6", path = "crates/brk_types" }
|
||||
brk_website = { version = "0.3.6", path = "crates/brk_website" }
|
||||
byteview = "0.10.1"
|
||||
color-eyre = "0.6.5"
|
||||
corepc-jsonrpc = { package = "jsonrpc", version = "0.19.0", features = ["simple_http"], default-features = false }
|
||||
corepc-types = { version = "0.14.0", features = ["std"], default-features = false }
|
||||
corepc-types = { version = "0.15.0", features = ["std"], default-features = false }
|
||||
derive_more = { version = "2.1.1", features = ["deref", "deref_mut"] }
|
||||
fjall = "3.1.4"
|
||||
fjall = "3.1.6"
|
||||
indexmap = { version = "2.14.0", features = ["serde"] }
|
||||
jiff = { version = "0.2.28", features = ["perf-inline", "tz-system"], default-features = false }
|
||||
jiff = { version = "0.2.32", features = ["perf-inline", "tz-system"], default-features = false }
|
||||
owo-colors = "4.3.0"
|
||||
parking_lot = "0.12.5"
|
||||
pco = "1.0.2"
|
||||
rayon = "1.12.0"
|
||||
rustc-hash = "2.1.2"
|
||||
rapidhash = "4.5.1"
|
||||
rustc-hash = "2.1.3"
|
||||
schemars = { version = "1.2.1", features = ["indexmap2"] }
|
||||
serde = "1.0.228"
|
||||
serde_bytes = "0.11.19"
|
||||
serde_derive = "1.0.228"
|
||||
serde_json = { version = "1.0.150", features = ["float_roundtrip", "preserve_order"] }
|
||||
smallvec = "1.15.1"
|
||||
tokio = { version = "1.52.3", features = ["rt-multi-thread"] }
|
||||
tower-http = { version = "0.6.11", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
|
||||
smallvec = "1.15.2"
|
||||
tokio = { version = "1.52.4", features = ["rt-multi-thread"] }
|
||||
tower-http = { version = "0.7.0", features = ["catch-panic", "compression-br", "compression-gzip", "compression-zstd", "cors", "normalize-path", "timeout", "trace"] }
|
||||
tower-layer = "0.3"
|
||||
tracing = { version = "0.1", default-features = false, features = ["std"] }
|
||||
ureq = { version = "3.3.0", features = ["json"] }
|
||||
vecdb = { version = "0.10.3", features = ["derive", "serde_json", "pco", "schemars"] }
|
||||
vecdb = { version = "0.10.4", features = ["derive", "serde_json", "pco", "schemars"] }
|
||||
# vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] }
|
||||
|
||||
|
||||
[workspace.metadata.release]
|
||||
shared-version = true
|
||||
tag-name = "v{{version}}"
|
||||
|
||||
@@ -6,9 +6,9 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use brk_cohort::{
|
||||
AGE_RANGE_NAMES, AMOUNT_RANGE_NAMES, CLASS_NAMES, EPOCH_NAMES, LOSS_NAMES, OVER_AGE_NAMES,
|
||||
OVER_AMOUNT_NAMES, PROFIT_NAMES, PROFITABILITY_RANGE_NAMES, SPENDABLE_TYPE_NAMES, TERM_NAMES,
|
||||
UNDER_AGE_NAMES, UNDER_AMOUNT_NAMES,
|
||||
AGE_RANGE_NAMES, AMOUNT_RANGE_NAMES, CLASS_NAMES, ENTRY_NAMES, EPOCH_NAMES, LOSS_NAMES,
|
||||
OVER_AGE_NAMES, OVER_AMOUNT_NAMES, PROFIT_NAMES, PROFITABILITY_RANGE_NAMES,
|
||||
SPENDABLE_TYPE_NAMES, TERM_NAMES, UNDER_AGE_NAMES, UNDER_AMOUNT_NAMES,
|
||||
};
|
||||
use brk_types::{Index, pools};
|
||||
use serde::Serialize;
|
||||
@@ -59,6 +59,7 @@ impl CohortConstants {
|
||||
("TERM_NAMES", to_value(&TERM_NAMES)),
|
||||
("EPOCH_NAMES", to_value(&EPOCH_NAMES)),
|
||||
("CLASS_NAMES", to_value(&CLASS_NAMES)),
|
||||
("ENTRY_NAMES", to_value(&ENTRY_NAMES)),
|
||||
("SPENDABLE_TYPE_NAMES", to_value(&SPENDABLE_TYPE_NAMES)),
|
||||
("AGE_RANGE_NAMES", to_value(&AGE_RANGE_NAMES)),
|
||||
("UNDER_AGE_NAMES", to_value(&UNDER_AGE_NAMES)),
|
||||
|
||||
@@ -51,7 +51,7 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
|
||||
}
|
||||
writeln!(
|
||||
output,
|
||||
" * @param {{{{ signal?: AbortSignal, onValue?: (value: {}) => void, cache?: boolean }}}} [options]",
|
||||
" * @param {{{{ signal?: AbortSignal, onValue?: (value: {}) => void, cache?: boolean, memCache?: boolean }}}} [options]",
|
||||
return_type
|
||||
)
|
||||
.unwrap();
|
||||
@@ -60,22 +60,22 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
|
||||
|
||||
let params = build_method_params(endpoint);
|
||||
let params_with_opts = if params.is_empty() {
|
||||
"{ signal, onValue, cache } = {}".to_string()
|
||||
"{ signal, onValue, cache, memCache } = {}".to_string()
|
||||
} else {
|
||||
format!("{}, {{ signal, onValue, cache }} = {{}}", params)
|
||||
format!("{}, {{ signal, onValue, cache, memCache }} = {{}}", params)
|
||||
};
|
||||
writeln!(output, " async {}({}) {{", method_name, params_with_opts).unwrap();
|
||||
|
||||
let path = build_path_template(&endpoint.path, &endpoint.path_params);
|
||||
|
||||
let fetch_call: String = if endpoint.returns_binary() {
|
||||
"this.getBytes(path, { signal, onValue, cache })".to_string()
|
||||
"this.getBytes(path, { signal, onValue, cache, memCache })".to_string()
|
||||
} else if endpoint.returns_json() {
|
||||
"this.getJson(path, { signal, onValue, cache })".to_string()
|
||||
"this.getJson(path, { signal, onValue, cache, memCache })".to_string()
|
||||
} else if endpoint.response_kind.text_is_numeric() {
|
||||
"Number(await this.getText(path, { signal, cache, onValue: onValue ? (v) => onValue(Number(v)) : undefined }))".to_string()
|
||||
"Number(await this.getText(path, { signal, cache, memCache, onValue: onValue ? (v) => onValue(Number(v)) : undefined }))".to_string()
|
||||
} else {
|
||||
"this.getText(path, { signal, onValue, cache })".to_string()
|
||||
"this.getText(path, { signal, onValue, cache, memCache })".to_string()
|
||||
};
|
||||
|
||||
write_path_assignment(output, endpoint, &path);
|
||||
@@ -83,7 +83,7 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
|
||||
if endpoint.supports_csv {
|
||||
writeln!(
|
||||
output,
|
||||
" if (format === 'csv') return this.getText(path, {{ signal, onValue, cache }});"
|
||||
" if (format === 'csv') return this.getText(path, {{ signal, onValue, cache, memCache }});"
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
@@ -51,6 +51,12 @@ const _openBrowserCache = (option) => {{
|
||||
return caches.open(name).catch(() => null);
|
||||
}};
|
||||
|
||||
/**
|
||||
* @param {{string}} url
|
||||
* @returns {{URL}}
|
||||
*/
|
||||
const _parseBaseUrl = (url) => new URL(url, typeof location === 'undefined' ? undefined : location.href);
|
||||
|
||||
/**
|
||||
* Custom error class for BRK client errors
|
||||
*/
|
||||
@@ -237,8 +243,8 @@ function _wrapSeriesData(raw) {{
|
||||
* @property {{(n: number) => RangeBuilder<T>}} first - Get first n items
|
||||
* @property {{(n: number) => RangeBuilder<T>}} last - Get last n items
|
||||
* @property {{(n: number) => SkippedBuilder<T>}} skip - Skip first n items, chain with take()
|
||||
* @property {{(onValue?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch all data
|
||||
* @property {{() => Promise<string>}} fetchCsv - Fetch all data as CSV
|
||||
* @property {{(arg?: SeriesFetchArg<T>, options?: ClientFetchOptions<SeriesData<T>>) => Promise<SeriesData<T>>}} fetch - Fetch all data
|
||||
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch all data as CSV
|
||||
* @property {{() => Promise<number>}} len - Get total number of data points
|
||||
* @property {{() => Promise<Version>}} version - Get the current version of the series
|
||||
* @property {{Thenable<T>}} then - Thenable (await endpoint)
|
||||
@@ -253,8 +259,8 @@ function _wrapSeriesData(raw) {{
|
||||
* @property {{(n: number) => DateRangeBuilder<T>}} first - Get first n items
|
||||
* @property {{(n: number) => DateRangeBuilder<T>}} last - Get last n items
|
||||
* @property {{(n: number) => DateSkippedBuilder<T>}} skip - Skip first n items, chain with take()
|
||||
* @property {{(onValue?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch all data
|
||||
* @property {{() => Promise<string>}} fetchCsv - Fetch all data as CSV
|
||||
* @property {{(arg?: DateSeriesFetchArg<T>, options?: ClientFetchOptions<DateSeriesData<T>>) => Promise<DateSeriesData<T>>}} fetch - Fetch all data
|
||||
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch all data as CSV
|
||||
* @property {{() => Promise<number>}} len - Get total number of data points
|
||||
* @property {{() => Promise<Version>}} version - Get the current version of the series
|
||||
* @property {{DateThenable<T>}} then - Thenable (await endpoint)
|
||||
@@ -263,41 +269,53 @@ function _wrapSeriesData(raw) {{
|
||||
|
||||
/** @typedef {{SeriesEndpoint<any>}} AnySeriesEndpoint */
|
||||
|
||||
/**
|
||||
* @template T
|
||||
* @typedef {{Object}} ClientFetchOptions
|
||||
* @property {{AbortSignal}} [signal] - Abort this request
|
||||
* @property {{boolean}} [cache] - Use HTTP/browser/client caches. Set false for a no-store network fetch.
|
||||
* @property {{boolean}} [memCache] - Use the parsed in-memory response cache. Set false for large one-shot reads.
|
||||
* @property {{(value: T) => void}} [onValue] - Receive stale/fresh values as they arrive
|
||||
*/
|
||||
|
||||
/** @template T @typedef {{ClientFetchOptions<SeriesData<T>> | ((value: SeriesData<T>) => void)}} SeriesFetchArg */
|
||||
/** @template T @typedef {{ClientFetchOptions<DateSeriesData<T>> | ((value: DateSeriesData<T>) => void)}} DateSeriesFetchArg */
|
||||
|
||||
/** @template T @typedef {{Object}} SingleItemBuilder
|
||||
* @property {{(onValue?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch the item
|
||||
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
|
||||
* @property {{(arg?: SeriesFetchArg<T>, options?: ClientFetchOptions<SeriesData<T>>) => Promise<SeriesData<T>>}} fetch - Fetch the item
|
||||
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch as CSV
|
||||
* @property {{Thenable<T>}} then - Thenable
|
||||
*/
|
||||
|
||||
/** @template T @typedef {{Object}} DateSingleItemBuilder
|
||||
* @property {{(onValue?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch the item
|
||||
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
|
||||
* @property {{(arg?: DateSeriesFetchArg<T>, options?: ClientFetchOptions<DateSeriesData<T>>) => Promise<DateSeriesData<T>>}} fetch - Fetch the item
|
||||
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch as CSV
|
||||
* @property {{DateThenable<T>}} then - Thenable
|
||||
*/
|
||||
|
||||
/** @template T @typedef {{Object}} SkippedBuilder
|
||||
* @property {{(n: number) => RangeBuilder<T>}} take - Take n items after skipped position
|
||||
* @property {{(onValue?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch from skipped position to end
|
||||
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
|
||||
* @property {{(arg?: SeriesFetchArg<T>, options?: ClientFetchOptions<SeriesData<T>>) => Promise<SeriesData<T>>}} fetch - Fetch from skipped position to end
|
||||
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch as CSV
|
||||
* @property {{Thenable<T>}} then - Thenable
|
||||
*/
|
||||
|
||||
/** @template T @typedef {{Object}} DateSkippedBuilder
|
||||
* @property {{(n: number) => DateRangeBuilder<T>}} take - Take n items after skipped position
|
||||
* @property {{(onValue?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch from skipped position to end
|
||||
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
|
||||
* @property {{(arg?: DateSeriesFetchArg<T>, options?: ClientFetchOptions<DateSeriesData<T>>) => Promise<DateSeriesData<T>>}} fetch - Fetch from skipped position to end
|
||||
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch as CSV
|
||||
* @property {{DateThenable<T>}} then - Thenable
|
||||
*/
|
||||
|
||||
/** @template T @typedef {{Object}} RangeBuilder
|
||||
* @property {{(onValue?: (value: SeriesData<T>) => void) => Promise<SeriesData<T>>}} fetch - Fetch the range
|
||||
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
|
||||
* @property {{(arg?: SeriesFetchArg<T>, options?: ClientFetchOptions<SeriesData<T>>) => Promise<SeriesData<T>>}} fetch - Fetch the range
|
||||
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch as CSV
|
||||
* @property {{Thenable<T>}} then - Thenable
|
||||
*/
|
||||
|
||||
/** @template T @typedef {{Object}} DateRangeBuilder
|
||||
* @property {{(onValue?: (value: DateSeriesData<T>) => void) => Promise<DateSeriesData<T>>}} fetch - Fetch the range
|
||||
* @property {{() => Promise<string>}} fetchCsv - Fetch as CSV
|
||||
* @property {{(arg?: DateSeriesFetchArg<T>, options?: ClientFetchOptions<DateSeriesData<T>>) => Promise<DateSeriesData<T>>}} fetch - Fetch the range
|
||||
* @property {{(options?: ClientFetchOptions<string>) => Promise<string>}} fetchCsv - Fetch as CSV
|
||||
* @property {{DateThenable<T>}} then - Thenable
|
||||
*/
|
||||
|
||||
@@ -344,8 +362,8 @@ function _endpoint(client, name, index) {{
|
||||
* @returns {{DateRangeBuilder<T>}}
|
||||
*/
|
||||
const rangeBuilder = (start, end) => ({{
|
||||
fetch(onValue) {{ return client._fetchSeriesData(buildPath(start, end), onValue); }},
|
||||
fetchCsv() {{ return client.getText(buildPath(start, end, 'csv')); }},
|
||||
fetch(arg, options) {{ return client._fetchSeriesData(buildPath(start, end), arg, options); }},
|
||||
fetchCsv(options) {{ return client.getText(buildPath(start, end, 'csv'), options); }},
|
||||
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
|
||||
}});
|
||||
|
||||
@@ -354,8 +372,8 @@ function _endpoint(client, name, index) {{
|
||||
* @returns {{DateSingleItemBuilder<T>}}
|
||||
*/
|
||||
const singleItemBuilder = (idx) => ({{
|
||||
fetch(onValue) {{ return client._fetchSeriesData(buildPath(idx, idx + 1), onValue); }},
|
||||
fetchCsv() {{ return client.getText(buildPath(idx, idx + 1, 'csv')); }},
|
||||
fetch(arg, options) {{ return client._fetchSeriesData(buildPath(idx, idx + 1), arg, options); }},
|
||||
fetchCsv(options) {{ return client.getText(buildPath(idx, idx + 1, 'csv'), options); }},
|
||||
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
|
||||
}});
|
||||
|
||||
@@ -365,8 +383,8 @@ function _endpoint(client, name, index) {{
|
||||
*/
|
||||
const skippedBuilder = (start) => ({{
|
||||
take(n) {{ return rangeBuilder(start, start + n); }},
|
||||
fetch(onValue) {{ return client._fetchSeriesData(buildPath(start, undefined), onValue); }},
|
||||
fetchCsv() {{ return client.getText(buildPath(start, undefined, 'csv')); }},
|
||||
fetch(arg, options) {{ return client._fetchSeriesData(buildPath(start, undefined), arg, options); }},
|
||||
fetchCsv(options) {{ return client.getText(buildPath(start, undefined, 'csv'), options); }},
|
||||
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
|
||||
}});
|
||||
|
||||
@@ -381,8 +399,8 @@ function _endpoint(client, name, index) {{
|
||||
first(n) {{ return rangeBuilder(undefined, n); }},
|
||||
last(n) {{ return n === 0 ? rangeBuilder(undefined, 0) : rangeBuilder(-n, undefined); }},
|
||||
skip(n) {{ return skippedBuilder(n); }},
|
||||
fetch(onValue) {{ return client._fetchSeriesData(buildPath(), onValue); }},
|
||||
fetchCsv() {{ return client.getText(buildPath(undefined, undefined, 'csv')); }},
|
||||
fetch(arg, options) {{ return client._fetchSeriesData(buildPath(), arg, options); }},
|
||||
fetchCsv(options) {{ return client.getText(buildPath(undefined, undefined, 'csv'), options); }},
|
||||
len() {{ return client.getSeriesLen(name, index); }},
|
||||
version() {{ return client.getSeriesVersion(name, index); }},
|
||||
then(resolve, reject) {{ return this.fetch().then(resolve, reject); }},
|
||||
@@ -403,6 +421,9 @@ class BrkClientBase {{
|
||||
const isString = typeof options === 'string';
|
||||
const rawUrl = isString ? options : options.baseUrl;
|
||||
this.baseUrl = rawUrl.endsWith('/') ? rawUrl.slice(0, -1) : rawUrl;
|
||||
const url = _parseBaseUrl(this.baseUrl);
|
||||
this.url = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;
|
||||
this.domain = url.hostname;
|
||||
this.timeout = isString ? 5000 : (options.timeout ?? 5000);
|
||||
/** @type {{Promise<Cache | null>}} */
|
||||
this._browserCachePromise = _openBrowserCache(isString ? undefined : options.browserCache);
|
||||
@@ -478,10 +499,10 @@ class BrkClientBase {{
|
||||
* @template T
|
||||
* @param {{string}} path
|
||||
* @param {{(res: Response) => Promise<T>}} parse - Response body reader
|
||||
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal, cache?: boolean }}}} [options]
|
||||
* @param {{ClientFetchOptions<T>}} [options]
|
||||
* @returns {{Promise<T>}}
|
||||
*/
|
||||
async _getCached(path, parse, {{ onValue, signal, cache = true }} = {{}}) {{
|
||||
async _getCached(path, parse, {{ onValue, signal, cache = true, memCache = true }} = {{}}) {{
|
||||
if (!cache) {{
|
||||
const res = await this.get(path, {{ signal, cache }});
|
||||
const value = await parse(res);
|
||||
@@ -490,8 +511,9 @@ class BrkClientBase {{
|
||||
}}
|
||||
|
||||
const url = `${{this.baseUrl}}${{path}}`;
|
||||
const useMemCache = memCache !== false;
|
||||
/** @type {{_MemEntry<T> | undefined}} */
|
||||
const memHit = this._memGet(url);
|
||||
const memHit = useMemCache ? this._memGet(url) : undefined;
|
||||
const browserCache = this._browserCache;
|
||||
|
||||
// L1 fast path: deliver from memCache, revalidate via network.
|
||||
@@ -504,7 +526,7 @@ class BrkClientBase {{
|
||||
if (netEtag && netEtag === memHit.etag) return memHit.value;
|
||||
const cloned = browserCache ? res.clone() : null;
|
||||
const value = await parse(res);
|
||||
this._memSet(url, netEtag, value);
|
||||
if (useMemCache) this._memSet(url, netEtag, value);
|
||||
if (onValue) onValue(value);
|
||||
if (cloned && browserCache) {{
|
||||
const cacheStore = browserCache;
|
||||
@@ -523,7 +545,7 @@ class BrkClientBase {{
|
||||
if (!res || networkSettled) return null;
|
||||
const value = await parse(res);
|
||||
if (networkSettled) return value;
|
||||
this._memSet(url, res.headers.get('ETag'), value);
|
||||
if (useMemCache) this._memSet(url, res.headers.get('ETag'), value);
|
||||
onValue(value);
|
||||
return value;
|
||||
}}).catch(() => null)
|
||||
@@ -534,11 +556,11 @@ class BrkClientBase {{
|
||||
networkSettled = true;
|
||||
const netEtag = res.headers.get('ETag');
|
||||
// Stale won and populated memCache with matching ETag → reuse, skip parse + second onValue.
|
||||
const populated = /** @type {{_MemEntry<T> | undefined}} */ (this._memGet(url));
|
||||
const populated = useMemCache ? /** @type {{_MemEntry<T> | undefined}} */ (this._memGet(url)) : undefined;
|
||||
if (populated && netEtag && netEtag === populated.etag) return populated.value;
|
||||
const cloned = browserCache ? res.clone() : null;
|
||||
const value = await parse(res);
|
||||
this._memSet(url, netEtag, value);
|
||||
if (useMemCache) this._memSet(url, netEtag, value);
|
||||
if (onValue) onValue(value);
|
||||
if (cloned && browserCache) {{
|
||||
const cacheStore = browserCache;
|
||||
@@ -556,7 +578,7 @@ class BrkClientBase {{
|
||||
* Make a GET request expecting a JSON response. Cached and supports `onValue`.
|
||||
* @template T
|
||||
* @param {{string}} path
|
||||
* @param {{{{ onValue?: (value: T) => void, signal?: AbortSignal, cache?: boolean }}}} [options]
|
||||
* @param {{ClientFetchOptions<T>}} [options]
|
||||
* @returns {{Promise<T>}}
|
||||
*/
|
||||
getJson(path, options) {{
|
||||
@@ -567,7 +589,7 @@ class BrkClientBase {{
|
||||
* Make a GET request expecting a text response (text/plain, text/csv, ...).
|
||||
* Cached and supports `onValue`, same as `getJson`.
|
||||
* @param {{string}} path
|
||||
* @param {{{{ onValue?: (value: string) => void, signal?: AbortSignal, cache?: boolean }}}} [options]
|
||||
* @param {{ClientFetchOptions<string>}} [options]
|
||||
* @returns {{Promise<string>}}
|
||||
*/
|
||||
getText(path, options) {{
|
||||
@@ -578,7 +600,7 @@ class BrkClientBase {{
|
||||
* Make a GET request expecting binary data (application/octet-stream).
|
||||
* Cached and supports `onValue`, same as `getJson`.
|
||||
* @param {{string}} path
|
||||
* @param {{{{ onValue?: (value: Uint8Array) => void, signal?: AbortSignal, cache?: boolean }}}} [options]
|
||||
* @param {{ClientFetchOptions<Uint8Array>}} [options]
|
||||
* @returns {{Promise<Uint8Array>}}
|
||||
*/
|
||||
getBytes(path, options) {{
|
||||
@@ -650,12 +672,17 @@ class BrkClientBase {{
|
||||
* Fetch series data and wrap with helper methods (internal)
|
||||
* @template T
|
||||
* @param {{string}} path
|
||||
* @param {{(value: DateSeriesData<T>) => void}} [onValue]
|
||||
* @param {{DateSeriesFetchArg<T>}} [arg]
|
||||
* @param {{ClientFetchOptions<DateSeriesData<T>>}} [options]
|
||||
* @returns {{Promise<DateSeriesData<T>>}}
|
||||
*/
|
||||
async _fetchSeriesData(path, onValue) {{
|
||||
async _fetchSeriesData(path, arg, options) {{
|
||||
const requestOptions = typeof arg === 'function'
|
||||
? {{ ...(options ?? {{}}), onValue: arg }}
|
||||
: {{ ...(arg ?? {{}}), ...(options ?? {{}}) }};
|
||||
const onValue = requestOptions.onValue;
|
||||
const wrappedOnValue = onValue ? (/** @type {{SeriesData<T>}} */ raw) => onValue(_wrapSeriesData(raw)) : undefined;
|
||||
const raw = await this.getJson(path, {{ onValue: wrappedOnValue }});
|
||||
const raw = await this.getJson(path, {{ ...requestOptions, onValue: wrappedOnValue }});
|
||||
return _wrapSeriesData(raw);
|
||||
}}
|
||||
}}
|
||||
@@ -679,6 +706,160 @@ const _p = (prefix, acc) => acc ? `${{prefix}}_${{acc}}` : prefix;
|
||||
"#
|
||||
)
|
||||
.unwrap();
|
||||
output.push_str(r##"
|
||||
const _MASK_64 = 0xffffffffffffffffn;
|
||||
const _RAPIDHASH_SECRETS = /** @type {const} */ ([
|
||||
0x2d358dccaa6c78a5n,
|
||||
0x8bb84b93962eacc9n,
|
||||
0x4b33a62ed433d4a3n,
|
||||
0x4d5a2da51de1aa47n,
|
||||
0xa0761d6478bd642fn,
|
||||
0xe7037ed1a0b428dbn,
|
||||
0x90ed1765281c388cn,
|
||||
]);
|
||||
const _RAPIDHASH_SEED = _rapidHashSeed(0n);
|
||||
|
||||
/** @param {bigint} value */
|
||||
function _u64(value) {
|
||||
return value & _MASK_64;
|
||||
}
|
||||
|
||||
/** @param {bigint} left @param {bigint} right */
|
||||
function _rapidMix(left, right) {
|
||||
const result = _u64(left) * _u64(right);
|
||||
return _u64(result) ^ _u64(result >> 64n);
|
||||
}
|
||||
|
||||
/** @param {bigint} left @param {bigint} right @returns {[bigint, bigint]} */
|
||||
function _rapidMum(left, right) {
|
||||
const result = _u64(left) * _u64(right);
|
||||
return [_u64(result), _u64(result >> 64n)];
|
||||
}
|
||||
|
||||
/** @param {bigint} seed */
|
||||
function _rapidHashSeed(seed) {
|
||||
return _u64(seed ^ _rapidMix(seed ^ _RAPIDHASH_SECRETS[2], _RAPIDHASH_SECRETS[1]));
|
||||
}
|
||||
|
||||
/** @param {Uint8Array} bytes @param {number} offset */
|
||||
function _readU32(bytes, offset) {
|
||||
return (
|
||||
BigInt(bytes[offset]) |
|
||||
(BigInt(bytes[offset + 1]) << 8n) |
|
||||
(BigInt(bytes[offset + 2]) << 16n) |
|
||||
(BigInt(bytes[offset + 3]) << 24n)
|
||||
);
|
||||
}
|
||||
|
||||
/** @param {Uint8Array} bytes @param {number} offset */
|
||||
function _readU64(bytes, offset) {
|
||||
return _readU32(bytes, offset) | (_readU32(bytes, offset + 4) << 32n);
|
||||
}
|
||||
|
||||
/** @param {Uint8Array | ArrayBuffer | ArrayBufferView | number[]} payload */
|
||||
function _asUint8Array(payload) {
|
||||
if (payload instanceof Uint8Array) return payload;
|
||||
if (payload instanceof ArrayBuffer) return new Uint8Array(payload);
|
||||
if (ArrayBuffer.isView(payload)) return new Uint8Array(payload.buffer, payload.byteOffset, payload.byteLength);
|
||||
if (Array.isArray(payload)) return new Uint8Array(payload);
|
||||
throw new Error("Expected address payload bytes");
|
||||
}
|
||||
|
||||
/** @param {Uint8Array | ArrayBuffer | ArrayBufferView | number[]} payload */
|
||||
function _rapidHashV3(payload) {
|
||||
const bytes = _asUint8Array(payload);
|
||||
const length = bytes.length;
|
||||
if (length === 0) throw new Error("Expected a non-empty address payload");
|
||||
if (length > 65) throw new Error("Expected at most 65 address payload bytes");
|
||||
|
||||
let seed = _RAPIDHASH_SEED;
|
||||
let a = 0n;
|
||||
let b = 0n;
|
||||
let remainder;
|
||||
|
||||
if (length <= 16) {
|
||||
if (length >= 4) {
|
||||
seed ^= BigInt(length);
|
||||
if (length >= 8) {
|
||||
a ^= _readU64(bytes, 0);
|
||||
b ^= _readU64(bytes, length - 8);
|
||||
} else {
|
||||
a ^= _readU32(bytes, 0);
|
||||
b ^= _readU32(bytes, length - 4);
|
||||
}
|
||||
} else if (length > 0) {
|
||||
a ^= (BigInt(bytes[0]) << 45n) | BigInt(bytes[length - 1]);
|
||||
b ^= BigInt(bytes[length >> 1]);
|
||||
}
|
||||
remainder = BigInt(length);
|
||||
} else {
|
||||
seed = _rapidMix(_readU64(bytes, 0) ^ _RAPIDHASH_SECRETS[2], _readU64(bytes, 8) ^ seed);
|
||||
if (length > 32) {
|
||||
seed = _rapidMix(_readU64(bytes, 16) ^ _RAPIDHASH_SECRETS[2], _readU64(bytes, 24) ^ seed);
|
||||
if (length > 48) {
|
||||
seed = _rapidMix(_readU64(bytes, 32) ^ _RAPIDHASH_SECRETS[1], _readU64(bytes, 40) ^ seed);
|
||||
if (length > 64) {
|
||||
seed = _rapidMix(_readU64(bytes, 48) ^ _RAPIDHASH_SECRETS[1], _readU64(bytes, 56) ^ seed);
|
||||
}
|
||||
}
|
||||
}
|
||||
remainder = BigInt(length);
|
||||
a ^= _readU64(bytes, length - 16) ^ remainder;
|
||||
b ^= _readU64(bytes, length - 8);
|
||||
}
|
||||
|
||||
a ^= _RAPIDHASH_SECRETS[1];
|
||||
b ^= seed;
|
||||
[a, b] = _rapidMum(a, b);
|
||||
return _rapidMix(a ^ 0xaaaaaaaaaaaaaaaan, b ^ _RAPIDHASH_SECRETS[1] ^ remainder);
|
||||
}
|
||||
|
||||
/** @param {number} nibbles */
|
||||
function _validateHashPrefixNibbles(nibbles) {
|
||||
if (!Number.isInteger(nibbles) || nibbles < 1 || nibbles > 16) {
|
||||
throw new Error("Expected hash-prefix length from 1 to 16 hex nibbles");
|
||||
}
|
||||
}
|
||||
|
||||
/** @param {OutputType} addrType @returns {number[]} */
|
||||
function _addressPayloadLengths(addrType) {
|
||||
switch (addrType) {
|
||||
case "p2a": return [2];
|
||||
case "p2pk": return [33, 65];
|
||||
case "p2pkh":
|
||||
case "p2sh":
|
||||
case "v0_p2wpkh": return [20];
|
||||
case "v0_p2wsh":
|
||||
case "v1_p2tr": return [32];
|
||||
default:
|
||||
throw new Error(`Unsupported address type for address payload hash-prefix: ${addrType}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {OutputType} addrType
|
||||
* @param {Uint8Array | ArrayBuffer | ArrayBufferView | number[]} payload
|
||||
*/
|
||||
function _validateAddressPayloadForType(addrType, payload) {
|
||||
const length = _asUint8Array(payload).length;
|
||||
const expected = _addressPayloadLengths(addrType);
|
||||
if (!expected.includes(length)) {
|
||||
throw new Error(`Expected ${addrType} address payload length ${expected.join(" or ")} bytes`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the RapidHash v3 hash-prefix used by `/api/address/hash-prefix/{addr_type}/{prefix}`.
|
||||
* @param {Uint8Array | ArrayBuffer | ArrayBufferView | number[]} payload - Raw address payload bytes
|
||||
* @param {number} nibbles - Prefix length from 1 to 16 hex nibbles
|
||||
* @returns {string}
|
||||
*/
|
||||
function addressPayloadHashPrefix(payload, nibbles) {
|
||||
_validateHashPrefixNibbles(nibbles);
|
||||
return _rapidHashV3(payload).toString(16).padStart(16, "0").slice(0, nibbles);
|
||||
}
|
||||
|
||||
"##);
|
||||
}
|
||||
|
||||
/// Generate static constants for the BrkClient class.
|
||||
|
||||
@@ -111,6 +111,32 @@ pub fn generate_main_client(
|
||||
writeln!(output, " this.series = this._buildTree();").unwrap();
|
||||
writeln!(output, " }}\n").unwrap();
|
||||
|
||||
output.push_str(r##" /**
|
||||
* Compute the RapidHash v3 hash-prefix for raw address payload bytes.
|
||||
* @param {Uint8Array | ArrayBuffer | ArrayBufferView | number[]} payload
|
||||
* @param {number} nibbles
|
||||
* @returns {string}
|
||||
*/
|
||||
static addressPayloadHashPrefix(payload, nibbles) {
|
||||
return addressPayloadHashPrefix(payload, nibbles);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch address hash-prefix matches from raw address payload bytes.
|
||||
* @param {OutputType} addrType
|
||||
* @param {Uint8Array | ArrayBuffer | ArrayBufferView | number[]} payload - Raw payload bytes matching addrType length
|
||||
* @param {number} nibbles
|
||||
* @param {{ signal?: AbortSignal, onValue?: (value: AddrHashPrefixMatches) => void, cache?: boolean, memCache?: boolean }} [options]
|
||||
* @returns {Promise<AddrHashPrefixMatches>}
|
||||
*/
|
||||
getAddressPayloadHashPrefixMatches(addrType, payload, nibbles, options = {}) {
|
||||
_validateAddressPayloadForType(addrType, payload);
|
||||
const prefix = addressPayloadHashPrefix(payload, nibbles);
|
||||
return this.getAddressHashPrefixMatches(addrType, prefix, options);
|
||||
}
|
||||
|
||||
"##);
|
||||
|
||||
writeln!(output, " /**").unwrap();
|
||||
writeln!(output, " * @private").unwrap();
|
||||
writeln!(output, " * @returns {{SeriesTree}}").unwrap();
|
||||
@@ -161,7 +187,11 @@ pub fn generate_main_client(
|
||||
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
|
||||
writeln!(output, "export {{ BrkClient, BrkError }};").unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
"export {{ BrkClient, BrkError, addressPayloadHashPrefix }};"
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
|
||||
@@ -84,6 +84,23 @@ pub fn generate_main_client(output: &mut String, endpoints: &[Endpoint]) {
|
||||
.unwrap();
|
||||
writeln!(output, " return _date_to_index(index, d)").unwrap();
|
||||
writeln!(output).unwrap();
|
||||
|
||||
output.push_str(r#" @staticmethod
|
||||
def address_payload_hash_prefix(payload: Union[bytes, bytearray, memoryview], nibbles: int) -> str:
|
||||
"""Compute the RapidHash v3 hash-prefix for raw address payload bytes."""
|
||||
return address_payload_hash_prefix(payload, nibbles)
|
||||
|
||||
def get_address_payload_hash_prefix_matches(
|
||||
self,
|
||||
addr_type: OutputType,
|
||||
payload: Union[bytes, bytearray, memoryview],
|
||||
nibbles: int,
|
||||
) -> AddrHashPrefixMatches:
|
||||
"""Fetch address hash-prefix matches from raw payload bytes matching addr_type length."""
|
||||
_validate_address_payload_for_type(addr_type, payload)
|
||||
return self.get_address_hash_prefix_matches(addr_type, address_payload_hash_prefix(payload, nibbles))
|
||||
|
||||
"#);
|
||||
// Generate API methods
|
||||
generate_api_methods(output, endpoints);
|
||||
}
|
||||
|
||||
@@ -147,6 +147,125 @@ def _p(prefix: str, acc: str) -> str:
|
||||
"#
|
||||
)
|
||||
.unwrap();
|
||||
output.push_str(r#"
|
||||
_MASK_64 = (1 << 64) - 1
|
||||
_RAPIDHASH_SECRETS = (
|
||||
0x2d358dccaa6c78a5,
|
||||
0x8bb84b93962eacc9,
|
||||
0x4b33a62ed433d4a3,
|
||||
0x4d5a2da51de1aa47,
|
||||
0xa0761d6478bd642f,
|
||||
0xe7037ed1a0b428db,
|
||||
0x90ed1765281c388c,
|
||||
)
|
||||
_RAPIDHASH_SEED = 0
|
||||
|
||||
|
||||
def _u64(value: int) -> int:
|
||||
return value & _MASK_64
|
||||
|
||||
|
||||
def _rapid_mix(left: int, right: int) -> int:
|
||||
result = _u64(left) * _u64(right)
|
||||
return _u64(result) ^ _u64(result >> 64)
|
||||
|
||||
|
||||
def _rapid_mum(left: int, right: int) -> Tuple[int, int]:
|
||||
result = _u64(left) * _u64(right)
|
||||
return _u64(result), _u64(result >> 64)
|
||||
|
||||
|
||||
def _rapid_hash_seed(seed: int) -> int:
|
||||
return _u64(seed ^ _rapid_mix(seed ^ _RAPIDHASH_SECRETS[2], _RAPIDHASH_SECRETS[1]))
|
||||
|
||||
|
||||
_RAPIDHASH_SEED = _rapid_hash_seed(0)
|
||||
|
||||
|
||||
def _read_u32(data: bytes, offset: int) -> int:
|
||||
return int.from_bytes(data[offset:offset + 4], "little")
|
||||
|
||||
|
||||
def _read_u64(data: bytes, offset: int) -> int:
|
||||
return int.from_bytes(data[offset:offset + 8], "little")
|
||||
|
||||
|
||||
def _rapid_hash_v3(payload: Union[bytes, bytearray, memoryview]) -> int:
|
||||
data = bytes(payload)
|
||||
length = len(data)
|
||||
if length == 0:
|
||||
raise ValueError("Expected a non-empty address payload")
|
||||
if length > 65:
|
||||
raise ValueError("Expected at most 65 address payload bytes")
|
||||
|
||||
seed = _RAPIDHASH_SEED
|
||||
a = 0
|
||||
b = 0
|
||||
|
||||
if length <= 16:
|
||||
if length >= 4:
|
||||
seed ^= length
|
||||
if length >= 8:
|
||||
a ^= _read_u64(data, 0)
|
||||
b ^= _read_u64(data, length - 8)
|
||||
else:
|
||||
a ^= _read_u32(data, 0)
|
||||
b ^= _read_u32(data, length - 4)
|
||||
elif length > 0:
|
||||
a ^= (data[0] << 45) | data[length - 1]
|
||||
b ^= data[length >> 1]
|
||||
remainder = length
|
||||
else:
|
||||
if length > 16:
|
||||
seed = _rapid_mix(_read_u64(data, 0) ^ _RAPIDHASH_SECRETS[2], _read_u64(data, 8) ^ seed)
|
||||
if length > 32:
|
||||
seed = _rapid_mix(_read_u64(data, 16) ^ _RAPIDHASH_SECRETS[2], _read_u64(data, 24) ^ seed)
|
||||
if length > 48:
|
||||
seed = _rapid_mix(_read_u64(data, 32) ^ _RAPIDHASH_SECRETS[1], _read_u64(data, 40) ^ seed)
|
||||
if length > 64:
|
||||
seed = _rapid_mix(_read_u64(data, 48) ^ _RAPIDHASH_SECRETS[1], _read_u64(data, 56) ^ seed)
|
||||
remainder = length
|
||||
a ^= _read_u64(data, length - 16) ^ remainder
|
||||
b ^= _read_u64(data, length - 8)
|
||||
|
||||
a ^= _RAPIDHASH_SECRETS[1]
|
||||
b ^= seed
|
||||
a, b = _rapid_mum(a, b)
|
||||
return _rapid_mix(a ^ 0xaaaaaaaaaaaaaaaa, b ^ _RAPIDHASH_SECRETS[1] ^ remainder)
|
||||
|
||||
|
||||
def _validate_hash_prefix_nibbles(nibbles: int) -> None:
|
||||
if isinstance(nibbles, bool) or not isinstance(nibbles, int) or nibbles < 1 or nibbles > 16:
|
||||
raise ValueError("Expected hash-prefix length from 1 to 16 hex nibbles")
|
||||
|
||||
|
||||
def _address_payload_lengths(addr_type: OutputType) -> Tuple[int, ...]:
|
||||
if addr_type == "p2a":
|
||||
return (2,)
|
||||
if addr_type == "p2pk":
|
||||
return (33, 65)
|
||||
if addr_type in ("p2pkh", "p2sh", "v0_p2wpkh"):
|
||||
return (20,)
|
||||
if addr_type in ("v0_p2wsh", "v1_p2tr"):
|
||||
return (32,)
|
||||
raise ValueError(f"Unsupported address type for address payload hash-prefix: {addr_type}")
|
||||
|
||||
|
||||
def _validate_address_payload_for_type(addr_type: OutputType, payload: Union[bytes, bytearray, memoryview]) -> None:
|
||||
length = len(bytes(payload))
|
||||
expected = _address_payload_lengths(addr_type)
|
||||
if length not in expected:
|
||||
joined = " or ".join(str(value) for value in expected)
|
||||
raise ValueError(f"Expected {addr_type} address payload length {joined} bytes")
|
||||
|
||||
|
||||
def address_payload_hash_prefix(payload: Union[bytes, bytearray, memoryview], nibbles: int) -> str:
|
||||
"""Compute the RapidHash v3 hash-prefix used by `/api/address/hash-prefix/{addr_type}/{prefix}`."""
|
||||
_validate_hash_prefix_nibbles(nibbles)
|
||||
return f"{_rapid_hash_v3(payload):016x}"[:nibbles]
|
||||
|
||||
|
||||
"#);
|
||||
}
|
||||
|
||||
/// Generate the SeriesData and SeriesEndpoint classes
|
||||
|
||||
@@ -76,6 +76,36 @@ impl BrkClient {{
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
output.push_str(r#" /// Decode a mainnet Bitcoin address into the BRK address type and raw payload bytes.
|
||||
pub fn decode_address_payload(address: &str) -> Result<AddressPayload> {
|
||||
decode_address_payload(address)
|
||||
}
|
||||
|
||||
/// Compute the RapidHash v3 hash-prefix for raw address payload bytes.
|
||||
pub fn address_payload_hash_prefix(payload: &[u8], nibbles: usize) -> Result<String> {
|
||||
address_payload_hash_prefix(payload, nibbles)
|
||||
}
|
||||
|
||||
/// Decode a mainnet Bitcoin address and compute its hash prefix.
|
||||
pub fn address_hash_prefix(address: &str, nibbles: usize) -> Result<AddressHashPrefix> {
|
||||
address_hash_prefix(address, nibbles)
|
||||
}
|
||||
|
||||
/// Fetch address hash-prefix matches from raw payload bytes matching `addr_type` length.
|
||||
pub fn get_address_payload_hash_prefix_matches(&self, addr_type: OutputType, payload: &[u8], nibbles: usize) -> Result<AddrHashPrefixMatches> {
|
||||
validate_address_payload_for_type(addr_type, payload)?;
|
||||
let prefix = address_payload_hash_prefix(payload, nibbles)?;
|
||||
self.get_address_hash_prefix_matches(addr_type, &prefix)
|
||||
}
|
||||
|
||||
/// Fetch address hash-prefix matches for a mainnet Bitcoin address.
|
||||
pub fn get_address_hash_prefix_matches_for_address(&self, address: &str, nibbles: usize) -> Result<AddrHashPrefixMatches> {
|
||||
let hashed = address_hash_prefix(address, nibbles)?;
|
||||
self.get_address_hash_prefix_matches(hashed.addr_type, &hashed.prefix)
|
||||
}
|
||||
|
||||
"#);
|
||||
|
||||
generate_api_methods(output, endpoints);
|
||||
|
||||
writeln!(output, "}}").unwrap();
|
||||
@@ -118,6 +148,23 @@ fn generate_get_method(output: &mut String, endpoint: &Endpoint) {
|
||||
"get_text"
|
||||
};
|
||||
|
||||
if endpoint.path == "/api/address/hash-prefix/{addr_type}/{prefix}" {
|
||||
writeln!(
|
||||
output,
|
||||
" let addr_type = address_payload_type_path(addr_type)?;"
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" self.base.{}(&format!(\"{}\"{}))",
|
||||
fetch_method, path, index_arg
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output).unwrap();
|
||||
return;
|
||||
}
|
||||
|
||||
if endpoint.query_params.is_empty() {
|
||||
writeln!(
|
||||
output,
|
||||
|
||||
@@ -11,7 +11,8 @@ use crate::{
|
||||
pub fn generate_imports(output: &mut String) {
|
||||
writeln!(
|
||||
output,
|
||||
r#"use std::sync::Arc;
|
||||
r#"use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
use std::ops::{{Bound, RangeBounds}};
|
||||
use serde::de::DeserializeOwned;
|
||||
pub use brk_cohort::*;
|
||||
@@ -43,6 +44,97 @@ impl std::error::Error for BrkError {{}}
|
||||
/// Result type for BRK client operations.
|
||||
pub type Result<T> = std::result::Result<T, BrkError>;
|
||||
|
||||
/// BRK address type and raw payload bytes used by the hash-prefix index.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AddressPayload {{
|
||||
pub addr_type: OutputType,
|
||||
pub payload: Vec<u8>,
|
||||
}}
|
||||
|
||||
/// BRK address type and leading hex nibbles of the address-payload hash.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AddressHashPrefix {{
|
||||
pub addr_type: OutputType,
|
||||
pub prefix: String,
|
||||
}}
|
||||
|
||||
/// Compute the RapidHash v3 hash-prefix used by `/api/address/hash-prefix/{{addr_type}}/{{prefix}}`.
|
||||
pub fn address_payload_hash_prefix(payload: &[u8], nibbles: usize) -> Result<String> {{
|
||||
if payload.is_empty() {{
|
||||
return Err(BrkError {{ message: "Expected a non-empty address payload".to_string() }});
|
||||
}}
|
||||
if payload.len() > 65 {{
|
||||
return Err(BrkError {{ message: "Expected at most 65 address payload bytes".to_string() }});
|
||||
}}
|
||||
if !(1..=16).contains(&nibbles) {{
|
||||
return Err(BrkError {{ message: "Expected hash-prefix length from 1 to 16 hex nibbles".to_string() }});
|
||||
}}
|
||||
Ok(format!("{{:016x}}", rapidhash::v3::rapidhash_v3(payload))[..nibbles].to_string())
|
||||
}}
|
||||
|
||||
fn validate_address_payload_for_type(addr_type: OutputType, payload: &[u8]) -> Result<()> {{
|
||||
let expected: &[usize] = match addr_type {{
|
||||
OutputType::P2A => &[2],
|
||||
OutputType::P2PK33 => &[33],
|
||||
OutputType::P2PK65 => &[65],
|
||||
OutputType::P2PKH | OutputType::P2SH | OutputType::P2WPKH => &[20],
|
||||
OutputType::P2WSH | OutputType::P2TR => &[32],
|
||||
OutputType::P2MS | OutputType::OpReturn | OutputType::Empty | OutputType::Unknown => {{
|
||||
return Err(BrkError {{ message: format!("Unsupported address type for address payload hash-prefix: {{addr_type:?}}") }});
|
||||
}},
|
||||
}};
|
||||
let addr_type = address_payload_type_path(addr_type)?;
|
||||
|
||||
if !expected.contains(&payload.len()) {{
|
||||
let joined = expected
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join(" or ");
|
||||
return Err(BrkError {{ message: format!("Expected {{addr_type}} address payload length {{joined}} bytes") }});
|
||||
}}
|
||||
|
||||
Ok(())
|
||||
}}
|
||||
|
||||
fn address_payload_type_path(addr_type: OutputType) -> Result<&'static str> {{
|
||||
match addr_type {{
|
||||
OutputType::P2A => Ok("p2a"),
|
||||
OutputType::P2PK33 | OutputType::P2PK65 => Ok("p2pk"),
|
||||
OutputType::P2PKH => Ok("p2pkh"),
|
||||
OutputType::P2SH => Ok("p2sh"),
|
||||
OutputType::P2WPKH => Ok("v0_p2wpkh"),
|
||||
OutputType::P2WSH => Ok("v0_p2wsh"),
|
||||
OutputType::P2TR => Ok("v1_p2tr"),
|
||||
OutputType::P2MS | OutputType::OpReturn | OutputType::Empty | OutputType::Unknown => {{
|
||||
Err(BrkError {{ message: format!("Unsupported address type for address payload hash-prefix: {{addr_type:?}}") }})
|
||||
}},
|
||||
}}
|
||||
}}
|
||||
|
||||
/// Decode a mainnet Bitcoin address into the BRK address type and raw payload bytes.
|
||||
pub fn decode_address_payload(address: &str) -> Result<AddressPayload> {{
|
||||
if address.is_empty() {{
|
||||
return Err(BrkError {{ message: "Expected an address string".to_string() }});
|
||||
}}
|
||||
let addr_bytes = AddrBytes::from_str(address).map_err(|e| BrkError {{ message: e.to_string() }})?;
|
||||
let addr_type = OutputType::from(&addr_bytes);
|
||||
|
||||
Ok(AddressPayload {{
|
||||
addr_type,
|
||||
payload: addr_bytes.as_slice().to_vec(),
|
||||
}})
|
||||
}}
|
||||
|
||||
/// Decode a mainnet Bitcoin address and compute its hash prefix.
|
||||
pub fn address_hash_prefix(address: &str, nibbles: usize) -> Result<AddressHashPrefix> {{
|
||||
let decoded = decode_address_payload(address)?;
|
||||
Ok(AddressHashPrefix {{
|
||||
addr_type: decoded.addr_type,
|
||||
prefix: address_payload_hash_prefix(&decoded.payload, nibbles)?,
|
||||
}})
|
||||
}}
|
||||
|
||||
/// Options for configuring the BRK client.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BrkClientOptions {{
|
||||
|
||||
@@ -25,7 +25,7 @@ owo-colors = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
toml = "1.1.2"
|
||||
toml = "1.1.3"
|
||||
vecdb = { workspace = true }
|
||||
|
||||
[[bin]]
|
||||
|
||||
@@ -13,6 +13,7 @@ exclude = ["examples/"]
|
||||
[dependencies]
|
||||
brk_cohort = { workspace = true }
|
||||
brk_types = { workspace = true }
|
||||
rapidhash = { workspace = true }
|
||||
ureq = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
+1129
-51
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,44 @@
|
||||
use brk_client::{
|
||||
BrkClient, OutputType, address_hash_prefix, address_payload_hash_prefix, decode_address_payload,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn address_payload_hash_prefix_vectors() {
|
||||
let vectors = [
|
||||
(vec![0x4e, 0x73], "58101afa51a1ecfd"),
|
||||
((0_u8..20).collect::<Vec<_>>(), "c3327ecb8ae1ff23"),
|
||||
((0_u8..32).collect::<Vec<_>>(), "c0186990f026b180"),
|
||||
((0_u8..65).collect::<Vec<_>>(), "0d4b77027ae7d700"),
|
||||
];
|
||||
|
||||
for (payload, expected) in vectors.iter() {
|
||||
assert_eq!(address_payload_hash_prefix(payload, 16).unwrap(), *expected);
|
||||
assert_eq!(
|
||||
BrkClient::address_payload_hash_prefix(payload, 8).unwrap(),
|
||||
&expected[..8]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn address_payload_hash_prefix_validation() {
|
||||
assert!(address_payload_hash_prefix(&[], 16).is_err());
|
||||
assert!(address_payload_hash_prefix(&[0; 66], 16).is_err());
|
||||
assert!(address_payload_hash_prefix(&[1, 2], 0).is_err());
|
||||
assert!(address_payload_hash_prefix(&[1, 2], 17).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn address_hash_prefix_uses_brk_address_parser() {
|
||||
let address = "1BoatSLRHtKNngkdXEeobR76b53LETtpyT";
|
||||
let decoded = decode_address_payload(address).unwrap();
|
||||
assert_eq!(decoded.addr_type, OutputType::P2PKH);
|
||||
assert_eq!(decoded.payload.len(), 20);
|
||||
|
||||
let hashed = address_hash_prefix(address, 8).unwrap();
|
||||
assert_eq!(hashed.addr_type, OutputType::P2PKH);
|
||||
assert_eq!(
|
||||
hashed.prefix,
|
||||
address_payload_hash_prefix(&decoded.payload, 8).unwrap()
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
use brk_traversable::Traversable;
|
||||
use rayon::iter::{IntoParallelIterator, ParallelIterator};
|
||||
use serde::Serialize;
|
||||
|
||||
use super::{CohortName, Filter};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum EntryPrice {
|
||||
Discount,
|
||||
Premium,
|
||||
}
|
||||
|
||||
impl EntryPrice {
|
||||
#[inline]
|
||||
pub const fn from_is_discount(is_discount: bool) -> Self {
|
||||
if is_discount {
|
||||
Self::Discount
|
||||
} else {
|
||||
Self::Premium
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub const fn is_discount(self) -> bool {
|
||||
matches!(self, Self::Discount)
|
||||
}
|
||||
}
|
||||
|
||||
pub const ENTRY_FILTERS: ByEntry<Filter> = ByEntry {
|
||||
discount: Filter::Entry(EntryPrice::Discount),
|
||||
premium: Filter::Entry(EntryPrice::Premium),
|
||||
};
|
||||
|
||||
pub const ENTRY_NAMES: ByEntry<CohortName> = ByEntry {
|
||||
discount: CohortName::new("veteran", "Veteran", "Veteran Coins"),
|
||||
premium: CohortName::new("rookie", "Rookie", "Rookie Coins"),
|
||||
};
|
||||
|
||||
#[derive(Default, Clone, Traversable, Serialize)]
|
||||
pub struct ByEntry<T> {
|
||||
pub discount: T,
|
||||
pub premium: T,
|
||||
}
|
||||
|
||||
impl ByEntry<CohortName> {
|
||||
pub const fn names() -> &'static Self {
|
||||
&ENTRY_NAMES
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ByEntry<T> {
|
||||
pub fn new<F>(mut create: F) -> Self
|
||||
where
|
||||
F: FnMut(Filter, &'static str) -> T,
|
||||
{
|
||||
let f = ENTRY_FILTERS;
|
||||
let n = ENTRY_NAMES;
|
||||
Self {
|
||||
discount: create(f.discount, n.discount.id),
|
||||
premium: create(f.premium, n.premium.id),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn try_new<F, E>(mut create: F) -> Result<Self, E>
|
||||
where
|
||||
F: FnMut(Filter, &'static str) -> Result<T, E>,
|
||||
{
|
||||
let f = ENTRY_FILTERS;
|
||||
let n = ENTRY_NAMES;
|
||||
Ok(Self {
|
||||
discount: create(f.discount, n.discount.id)?,
|
||||
premium: create(f.premium, n.premium.id)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get(&self, entry: EntryPrice) -> &T {
|
||||
match entry {
|
||||
EntryPrice::Discount => &self.discount,
|
||||
EntryPrice::Premium => &self.premium,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_mut(&mut self, entry: EntryPrice) -> &mut T {
|
||||
match entry {
|
||||
EntryPrice::Discount => &mut self.discount,
|
||||
EntryPrice::Premium => &mut self.premium,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> impl Iterator<Item = &T> {
|
||||
[&self.discount, &self.premium].into_iter()
|
||||
}
|
||||
|
||||
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
|
||||
[&mut self.discount, &mut self.premium].into_iter()
|
||||
}
|
||||
|
||||
pub fn par_iter_mut(&mut self) -> impl ParallelIterator<Item = &mut T>
|
||||
where
|
||||
T: Send + Sync,
|
||||
{
|
||||
[&mut self.discount, &mut self.premium].into_par_iter()
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,7 @@ impl CohortContext {
|
||||
/// Build full name for a filter, adding prefix only for Time/Amount filters.
|
||||
///
|
||||
/// Prefix rules:
|
||||
/// - No prefix: `All`, `Term`, `Epoch`, `Class`, `Type`
|
||||
/// - No prefix: `All`, `Term`, `Epoch`, `Class`, `Entry`, `Type`
|
||||
/// - Context prefix: `Time`, `Amount`
|
||||
pub fn full_name(&self, filter: &Filter, name: &str) -> String {
|
||||
match filter {
|
||||
@@ -32,6 +32,7 @@ impl CohortContext {
|
||||
| Filter::Term(_)
|
||||
| Filter::Epoch(_)
|
||||
| Filter::Class(_)
|
||||
| Filter::Entry(_)
|
||||
| Filter::Type(_) => name.to_string(),
|
||||
Filter::Time(_) | Filter::Amount(_) => self.prefixed(name),
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use brk_types::{Halving, OutputType, Sats, Year};
|
||||
|
||||
use super::{AmountFilter, CohortContext, Term, TimeFilter};
|
||||
use super::{AmountFilter, CohortContext, EntryPrice, Term, TimeFilter};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Filter {
|
||||
@@ -10,6 +10,7 @@ pub enum Filter {
|
||||
Amount(AmountFilter),
|
||||
Epoch(Halving),
|
||||
Class(Year),
|
||||
Entry(EntryPrice),
|
||||
Type(OutputType),
|
||||
}
|
||||
|
||||
@@ -68,7 +69,8 @@ impl Filter {
|
||||
}
|
||||
|
||||
/// Whether to compute extended metrics (realized cap ratios, profit/loss ratios, percentiles)
|
||||
/// For UTXO context: true only for age range cohorts (Range) and aggregate cohorts (All, Term)
|
||||
/// For UTXO context: true for age range cohorts (Range), aggregate cohorts (All, Term),
|
||||
/// and immutable entry valuation cohorts.
|
||||
/// For address context: always false
|
||||
pub fn is_extended(&self, context: CohortContext) -> bool {
|
||||
match context {
|
||||
@@ -76,7 +78,10 @@ impl Filter {
|
||||
CohortContext::Utxo => {
|
||||
matches!(
|
||||
self,
|
||||
Filter::All | Filter::Term(_) | Filter::Time(TimeFilter::Range(_))
|
||||
Filter::All
|
||||
| Filter::Term(_)
|
||||
| Filter::Time(TimeFilter::Range(_))
|
||||
| Filter::Entry(_)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ mod amount_range;
|
||||
mod by_addr_type;
|
||||
mod by_any_addr;
|
||||
mod by_epoch;
|
||||
mod by_entry;
|
||||
mod by_term;
|
||||
mod by_type;
|
||||
mod class;
|
||||
@@ -36,6 +37,7 @@ pub use amount_range::*;
|
||||
pub use by_addr_type::*;
|
||||
pub use by_any_addr::*;
|
||||
pub use by_epoch::*;
|
||||
pub use by_entry::*;
|
||||
pub use by_term::*;
|
||||
pub use by_type::*;
|
||||
pub use class::*;
|
||||
|
||||
@@ -2,8 +2,8 @@ use brk_traversable::Traversable;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::{
|
||||
AgeRange, AmountRange, ByEpoch, ByTerm, Class, Filter, OverAge, OverAmount, SpendableType,
|
||||
UnderAge, UnderAmount,
|
||||
AgeRange, AmountRange, ByEntry, ByEpoch, ByTerm, Class, Filter, OverAge, OverAmount,
|
||||
SpendableType, UnderAge, UnderAmount,
|
||||
};
|
||||
|
||||
#[derive(Default, Clone, Traversable)]
|
||||
@@ -12,6 +12,7 @@ pub struct UTXOGroups<T> {
|
||||
pub age_range: AgeRange<T>,
|
||||
pub epoch: ByEpoch<T>,
|
||||
pub class: Class<T>,
|
||||
pub entry: ByEntry<T>,
|
||||
pub over_age: OverAge<T>,
|
||||
pub over_amount: OverAmount<T>,
|
||||
pub amount_range: AmountRange<T>,
|
||||
@@ -31,6 +32,7 @@ impl<T> UTXOGroups<T> {
|
||||
age_range: AgeRange::new(&mut create),
|
||||
epoch: ByEpoch::new(&mut create),
|
||||
class: Class::new(&mut create),
|
||||
entry: ByEntry::new(&mut create),
|
||||
over_age: OverAge::new(&mut create),
|
||||
over_amount: OverAmount::new(&mut create),
|
||||
amount_range: AmountRange::new(&mut create),
|
||||
@@ -51,6 +53,7 @@ impl<T> UTXOGroups<T> {
|
||||
.chain(self.age_range.iter())
|
||||
.chain(self.epoch.iter())
|
||||
.chain(self.class.iter())
|
||||
.chain(self.entry.iter())
|
||||
.chain(self.amount_range.iter())
|
||||
.chain(self.under_amount.iter())
|
||||
.chain(self.type_.iter())
|
||||
@@ -66,6 +69,7 @@ impl<T> UTXOGroups<T> {
|
||||
.chain(self.age_range.iter_mut())
|
||||
.chain(self.epoch.iter_mut())
|
||||
.chain(self.class.iter_mut())
|
||||
.chain(self.entry.iter_mut())
|
||||
.chain(self.amount_range.iter_mut())
|
||||
.chain(self.under_amount.iter_mut())
|
||||
.chain(self.type_.iter_mut())
|
||||
@@ -84,6 +88,7 @@ impl<T> UTXOGroups<T> {
|
||||
.chain(self.age_range.par_iter_mut())
|
||||
.chain(self.epoch.par_iter_mut())
|
||||
.chain(self.class.par_iter_mut())
|
||||
.chain(self.entry.par_iter_mut())
|
||||
.chain(self.amount_range.par_iter_mut())
|
||||
.chain(self.under_amount.par_iter_mut())
|
||||
.chain(self.type_.par_iter_mut())
|
||||
@@ -94,6 +99,7 @@ impl<T> UTXOGroups<T> {
|
||||
.iter()
|
||||
.chain(self.epoch.iter())
|
||||
.chain(self.class.iter())
|
||||
.chain(self.entry.iter())
|
||||
.chain(self.amount_range.iter())
|
||||
.chain(self.type_.iter())
|
||||
}
|
||||
@@ -103,6 +109,7 @@ impl<T> UTXOGroups<T> {
|
||||
.iter_mut()
|
||||
.chain(self.epoch.iter_mut())
|
||||
.chain(self.class.iter_mut())
|
||||
.chain(self.entry.iter_mut())
|
||||
.chain(self.amount_range.iter_mut())
|
||||
.chain(self.type_.iter_mut())
|
||||
}
|
||||
@@ -115,6 +122,7 @@ impl<T> UTXOGroups<T> {
|
||||
.par_iter_mut()
|
||||
.chain(self.epoch.par_iter_mut())
|
||||
.chain(self.class.par_iter_mut())
|
||||
.chain(self.entry.par_iter_mut())
|
||||
.chain(self.amount_range.par_iter_mut())
|
||||
.chain(self.type_.par_iter_mut())
|
||||
}
|
||||
|
||||
@@ -30,18 +30,34 @@ const TREE_SIZE: usize = TIER0_COUNT + TIER1_COUNT + OVERFLOW; // 190,001
|
||||
pub(super) struct CostBasisNode {
|
||||
all_sats: i64,
|
||||
sth_sats: i64,
|
||||
discount_sats: i64,
|
||||
all_usd: i128,
|
||||
sth_usd: i128,
|
||||
discount_usd: i128,
|
||||
}
|
||||
|
||||
impl CostBasisNode {
|
||||
#[inline(always)]
|
||||
fn new(sats: i64, usd: i128, is_sth: bool) -> Self {
|
||||
fn new_supply(sats: i64, usd: i128, is_sth: bool) -> Self {
|
||||
Self {
|
||||
all_sats: sats,
|
||||
sth_sats: if is_sth { sats } else { 0 },
|
||||
discount_sats: 0,
|
||||
all_usd: usd,
|
||||
sth_usd: if is_sth { usd } else { 0 },
|
||||
discount_usd: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn new_discount(sats: i64, usd: i128) -> Self {
|
||||
Self {
|
||||
all_sats: 0,
|
||||
sth_sats: 0,
|
||||
discount_sats: sats,
|
||||
all_usd: 0,
|
||||
sth_usd: 0,
|
||||
discount_usd: usd,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -51,8 +67,10 @@ impl FenwickNode for CostBasisNode {
|
||||
fn add_assign(&mut self, other: &Self) {
|
||||
self.all_sats += other.all_sats;
|
||||
self.sth_sats += other.sth_sats;
|
||||
self.discount_sats += other.discount_sats;
|
||||
self.all_usd += other.all_usd;
|
||||
self.sth_usd += other.sth_usd;
|
||||
self.discount_usd += other.discount_usd;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,16 +169,34 @@ impl CostBasisFenwick {
|
||||
}
|
||||
let bucket = price_to_bucket(price);
|
||||
let delta =
|
||||
CostBasisNode::new(net_sats, price.as_u128() as i128 * net_sats as i128, is_sth);
|
||||
CostBasisNode::new_supply(net_sats, price.as_u128() as i128 * net_sats as i128, is_sth);
|
||||
self.tree.add(bucket, &delta);
|
||||
self.totals.add_assign(&delta);
|
||||
}
|
||||
|
||||
/// Bulk-initialize from BTreeMaps (one per age-range cohort).
|
||||
/// Call after state import when all pending maps have been drained.
|
||||
pub(super) fn bulk_init<'a>(
|
||||
/// Apply a net delta from the discount-entry cohort.
|
||||
///
|
||||
/// Supply totals are maintained from the age-range cohorts; this updates
|
||||
/// only the discount-entry partition so premium can be derived as all - discount.
|
||||
pub(super) fn apply_discount_delta(&mut self, price: CentsCompact, pending: &PendingDelta) {
|
||||
let net_sats = u64::from(pending.inc) as i64 - u64::from(pending.dec) as i64;
|
||||
if net_sats == 0 {
|
||||
return;
|
||||
}
|
||||
let bucket = price_to_bucket(price);
|
||||
let delta =
|
||||
CostBasisNode::new_discount(net_sats, price.as_u128() as i128 * net_sats as i128);
|
||||
self.tree.add(bucket, &delta);
|
||||
self.totals.add_assign(&delta);
|
||||
}
|
||||
|
||||
/// Bulk-initialize from age-range maps plus the discount-entry map.
|
||||
/// Age-range maps maintain all/STH/LTH totals; the discount-entry map
|
||||
/// maintains only the discount partition used to derive premium.
|
||||
pub(super) fn bulk_init_with_discount<'a>(
|
||||
&mut self,
|
||||
maps: impl Iterator<Item = (&'a std::collections::BTreeMap<CentsCompact, Sats>, bool)>,
|
||||
discount_maps: impl Iterator<Item = &'a std::collections::BTreeMap<CentsCompact, Sats>>,
|
||||
) {
|
||||
self.tree.reset();
|
||||
self.totals = CostBasisNode::default();
|
||||
@@ -169,7 +205,18 @@ impl CostBasisFenwick {
|
||||
for (&price, &sats) in map.iter() {
|
||||
let bucket = price_to_bucket(price);
|
||||
let s = u64::from(sats) as i64;
|
||||
let node = CostBasisNode::new(s, price.as_u128() as i128 * s as i128, is_sth);
|
||||
let node =
|
||||
CostBasisNode::new_supply(s, price.as_u128() as i128 * s as i128, is_sth);
|
||||
self.tree.add_raw(bucket, &node);
|
||||
self.totals.add_assign(&node);
|
||||
}
|
||||
}
|
||||
|
||||
for map in discount_maps {
|
||||
for (&price, &sats) in map.iter() {
|
||||
let bucket = price_to_bucket(price);
|
||||
let s = u64::from(sats) as i64;
|
||||
let node = CostBasisNode::new_discount(s, price.as_u128() as i128 * s as i128);
|
||||
self.tree.add_raw(bucket, &node);
|
||||
self.totals.add_assign(&node);
|
||||
}
|
||||
@@ -212,6 +259,26 @@ impl CostBasisFenwick {
|
||||
)
|
||||
}
|
||||
|
||||
/// Compute percentile prices for discount-entry cohort.
|
||||
pub(super) fn percentiles_discount_entry(&self) -> PercentileResult {
|
||||
self.compute_percentiles(
|
||||
self.totals.discount_sats,
|
||||
self.totals.discount_usd,
|
||||
|n| n.discount_sats,
|
||||
|n| n.discount_usd,
|
||||
)
|
||||
}
|
||||
|
||||
/// Compute percentile prices for premium-entry cohort (all - discount).
|
||||
pub(super) fn percentiles_premium_entry(&self) -> PercentileResult {
|
||||
self.compute_percentiles(
|
||||
self.totals.all_sats - self.totals.discount_sats,
|
||||
self.totals.all_usd - self.totals.discount_usd,
|
||||
|n| n.all_sats - n.discount_sats,
|
||||
|n| n.all_usd - n.discount_usd,
|
||||
)
|
||||
}
|
||||
|
||||
fn compute_percentiles(
|
||||
&self,
|
||||
total_sats: i64,
|
||||
@@ -271,6 +338,37 @@ impl CostBasisFenwick {
|
||||
return (0, 0, 0);
|
||||
}
|
||||
|
||||
let range = self.density_range(spot_price);
|
||||
let all_range = range.all_sats.max(0);
|
||||
let sth_range = range.sth_sats.max(0);
|
||||
let lth_range = all_range - sth_range;
|
||||
|
||||
let lth_total = self.totals.all_sats - self.totals.sth_sats;
|
||||
(
|
||||
Self::to_bps(all_range, self.totals.all_sats),
|
||||
Self::to_bps(sth_range, self.totals.sth_sats),
|
||||
Self::to_bps(lth_range, lth_total),
|
||||
)
|
||||
}
|
||||
|
||||
/// Compute supply density for entry cohorts: (discount_bps, premium_bps).
|
||||
pub(super) fn entry_density(&self, spot_price: Cents) -> (u16, u16) {
|
||||
if self.totals.all_sats <= 0 {
|
||||
return (0, 0);
|
||||
}
|
||||
|
||||
let range = self.density_range(spot_price);
|
||||
let discount_range = range.discount_sats.max(0);
|
||||
let premium_range = range.all_sats.max(0) - discount_range;
|
||||
let premium_total = self.totals.all_sats - self.totals.discount_sats;
|
||||
|
||||
(
|
||||
Self::to_bps(discount_range, self.totals.discount_sats),
|
||||
Self::to_bps(premium_range, premium_total),
|
||||
)
|
||||
}
|
||||
|
||||
fn density_range(&self, spot_price: Cents) -> CostBasisNode {
|
||||
let spot_f64 = u64::from(spot_price) as f64;
|
||||
let low = Cents::from((spot_f64 * 0.95) as u64);
|
||||
let high = Cents::from((spot_f64 * 1.05) as u64);
|
||||
@@ -285,24 +383,23 @@ impl CostBasisFenwick {
|
||||
CostBasisNode::default()
|
||||
};
|
||||
|
||||
let all_range = (cum_high.all_sats - cum_low.all_sats).max(0);
|
||||
let sth_range = (cum_high.sth_sats - cum_low.sth_sats).max(0);
|
||||
let lth_range = all_range - sth_range;
|
||||
CostBasisNode {
|
||||
all_sats: cum_high.all_sats - cum_low.all_sats,
|
||||
sth_sats: cum_high.sth_sats - cum_low.sth_sats,
|
||||
discount_sats: cum_high.discount_sats - cum_low.discount_sats,
|
||||
all_usd: cum_high.all_usd - cum_low.all_usd,
|
||||
sth_usd: cum_high.sth_usd - cum_low.sth_usd,
|
||||
discount_usd: cum_high.discount_usd - cum_low.discount_usd,
|
||||
}
|
||||
}
|
||||
|
||||
let to_bps = |range: i64, total: i64| -> u16 {
|
||||
if total <= 0 {
|
||||
0
|
||||
} else {
|
||||
(range as f64 / total as f64 * 10000.0).round() as u16
|
||||
}
|
||||
};
|
||||
|
||||
let lth_total = self.totals.all_sats - self.totals.sth_sats;
|
||||
(
|
||||
to_bps(all_range, self.totals.all_sats),
|
||||
to_bps(sth_range, self.totals.sth_sats),
|
||||
to_bps(lth_range, lth_total),
|
||||
)
|
||||
#[inline(always)]
|
||||
fn to_bps(range: i64, total: i64) -> u16 {
|
||||
if total <= 0 {
|
||||
0
|
||||
} else {
|
||||
(range as f64 / total as f64 * 10000.0).round() as u16
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_cohort::{
|
||||
AgeRange, AmountRange, ByEpoch, Class, CohortContext, Filter, Filtered, OverAge, OverAmount,
|
||||
SpendableType, Term, UnderAge, UnderAmount,
|
||||
AgeRange, AmountRange, ByEntry, ByEpoch, Class, CohortContext, Filter, Filtered, OverAge,
|
||||
OverAmount, SpendableType, Term, UnderAge, UnderAmount,
|
||||
};
|
||||
use brk_error::Result;
|
||||
use brk_indexer::Lengths;
|
||||
@@ -45,6 +45,7 @@ pub struct UTXOCohorts<M: StorageMode = Rw> {
|
||||
pub over_age: OverAge<UTXOCohortVecs<CoreCohortMetrics<M>>>,
|
||||
pub epoch: ByEpoch<UTXOCohortVecs<CoreCohortMetrics<M>>>,
|
||||
pub class: Class<UTXOCohortVecs<CoreCohortMetrics<M>>>,
|
||||
pub entry: ByEntry<UTXOCohortVecs<ExtendedCohortMetrics<M>>>,
|
||||
pub over_amount: OverAmount<UTXOCohortVecs<MinimalCohortMetrics<M>>>,
|
||||
pub amount_range: AmountRange<UTXOCohortVecs<MinimalCohortMetrics<M>>>,
|
||||
pub under_amount: UnderAmount<UTXOCohortVecs<MinimalCohortMetrics<M>>>,
|
||||
@@ -67,8 +68,10 @@ pub(crate) struct UTXOCohortsTransientState {
|
||||
}
|
||||
|
||||
impl UTXOCohorts<Rw> {
|
||||
/// ~71 separate cohorts (21 age + 5 epoch + 18 class + 15 amount + 12 type)
|
||||
const SEPARATE_COHORT_CAPACITY: usize = 80;
|
||||
/// Separate cohorts currently total 72:
|
||||
/// 21 age + 5 epoch + 18 class + 2 entry + 15 amount + 11 spendable type.
|
||||
/// Keep small headroom because this is only Vec allocation capacity.
|
||||
const SEPARATE_COHORT_CAPACITY: usize = 82;
|
||||
|
||||
/// Import all UTXO cohorts from database.
|
||||
pub(crate) fn forced_import(
|
||||
@@ -136,6 +139,26 @@ impl UTXOCohorts<Rw> {
|
||||
let epoch = ByEpoch::try_new(&core_separate)?;
|
||||
let class = Class::try_new(&core_separate)?;
|
||||
|
||||
let extended_separate =
|
||||
|f: Filter, name: &'static str| -> Result<UTXOCohortVecs<ExtendedCohortMetrics>> {
|
||||
let full_name = CohortContext::Utxo.full_name(&f, name);
|
||||
let cfg = ImportConfig {
|
||||
db,
|
||||
filter: &f,
|
||||
full_name: &full_name,
|
||||
version: v,
|
||||
indexes,
|
||||
cached_starts,
|
||||
};
|
||||
let state = Some(Box::new(UTXOCohortState::new(states_path, &full_name)));
|
||||
Ok(UTXOCohortVecs::new(
|
||||
state,
|
||||
ExtendedCohortMetrics::forced_import(&cfg)?,
|
||||
))
|
||||
};
|
||||
|
||||
let entry = ByEntry::try_new(&extended_separate)?;
|
||||
|
||||
// Helper for separate cohorts with MinimalCohortMetrics + MinimalRealizedState
|
||||
let minimal_separate =
|
||||
|f: Filter, name: &'static str| -> Result<UTXOCohortVecs<MinimalCohortMetrics>> {
|
||||
@@ -281,6 +304,7 @@ impl UTXOCohorts<Rw> {
|
||||
lth,
|
||||
epoch,
|
||||
class,
|
||||
entry,
|
||||
type_,
|
||||
under_age,
|
||||
over_age,
|
||||
@@ -309,6 +333,7 @@ impl UTXOCohorts<Rw> {
|
||||
sth,
|
||||
caches,
|
||||
age_range,
|
||||
entry,
|
||||
..
|
||||
} = self;
|
||||
caches
|
||||
@@ -327,7 +352,15 @@ impl UTXOCohorts<Rw> {
|
||||
Some((map, caches.fenwick.is_sth_at(i)))
|
||||
})
|
||||
.collect();
|
||||
caches.fenwick.bulk_init(maps.into_iter());
|
||||
let discount_maps = entry
|
||||
.discount
|
||||
.state
|
||||
.as_ref()
|
||||
.map(|state| state.cost_basis_map())
|
||||
.into_iter();
|
||||
caches
|
||||
.fenwick
|
||||
.bulk_init_with_discount(maps.into_iter(), discount_maps);
|
||||
}
|
||||
|
||||
/// Apply pending deltas from all age-range cohorts to the Fenwick tree.
|
||||
@@ -338,7 +371,10 @@ impl UTXOCohorts<Rw> {
|
||||
}
|
||||
// Destructure to get separate borrows on caches and age_range
|
||||
let Self {
|
||||
caches, age_range, ..
|
||||
caches,
|
||||
age_range,
|
||||
entry,
|
||||
..
|
||||
} = self;
|
||||
for (i, sub) in age_range.iter().enumerate() {
|
||||
if let Some(state) = sub.state.as_ref() {
|
||||
@@ -348,6 +384,11 @@ impl UTXOCohorts<Rw> {
|
||||
});
|
||||
}
|
||||
}
|
||||
if let Some(state) = entry.discount.state.as_ref() {
|
||||
state.for_each_cost_basis_pending(|&price, delta| {
|
||||
caches.fenwick.apply_discount_delta(price, delta);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Push maturation sats to the matured vecs for the given height.
|
||||
@@ -365,6 +406,7 @@ impl UTXOCohorts<Rw> {
|
||||
age_range,
|
||||
epoch,
|
||||
class,
|
||||
entry,
|
||||
amount_range,
|
||||
type_,
|
||||
..
|
||||
@@ -374,6 +416,7 @@ impl UTXOCohorts<Rw> {
|
||||
.map(|x| x as &mut dyn DynCohortVecs)
|
||||
.chain(epoch.par_iter_mut().map(|x| x as &mut dyn DynCohortVecs))
|
||||
.chain(class.par_iter_mut().map(|x| x as &mut dyn DynCohortVecs))
|
||||
.chain(entry.par_iter_mut().map(|x| x as &mut dyn DynCohortVecs))
|
||||
.chain(
|
||||
amount_range
|
||||
.par_iter_mut()
|
||||
@@ -389,6 +432,7 @@ impl UTXOCohorts<Rw> {
|
||||
age_range,
|
||||
epoch,
|
||||
class,
|
||||
entry,
|
||||
amount_range,
|
||||
type_,
|
||||
..
|
||||
@@ -398,6 +442,7 @@ impl UTXOCohorts<Rw> {
|
||||
.map(|x| x as &mut dyn DynCohortVecs)
|
||||
.chain(epoch.iter_mut().map(|x| x as &mut dyn DynCohortVecs))
|
||||
.chain(class.iter_mut().map(|x| x as &mut dyn DynCohortVecs))
|
||||
.chain(entry.iter_mut().map(|x| x as &mut dyn DynCohortVecs))
|
||||
.chain(amount_range.iter_mut().map(|x| x as &mut dyn DynCohortVecs))
|
||||
.chain(type_.iter_mut().map(|x| x as &mut dyn DynCohortVecs))
|
||||
}
|
||||
@@ -409,6 +454,7 @@ impl UTXOCohorts<Rw> {
|
||||
.map(|x| x as &dyn DynCohortVecs)
|
||||
.chain(self.epoch.iter().map(|x| x as &dyn DynCohortVecs))
|
||||
.chain(self.class.iter().map(|x| x as &dyn DynCohortVecs))
|
||||
.chain(self.entry.iter().map(|x| x as &dyn DynCohortVecs))
|
||||
.chain(self.amount_range.iter().map(|x| x as &dyn DynCohortVecs))
|
||||
.chain(self.type_.iter().map(|x| x as &dyn DynCohortVecs))
|
||||
}
|
||||
@@ -516,6 +562,7 @@ impl UTXOCohorts<Rw> {
|
||||
);
|
||||
all.extend(self.epoch.iter_mut().map(|x| x as &mut dyn DynCohortVecs));
|
||||
all.extend(self.class.iter_mut().map(|x| x as &mut dyn DynCohortVecs));
|
||||
all.extend(self.entry.iter_mut().map(|x| x as &mut dyn DynCohortVecs));
|
||||
all.extend(
|
||||
self.amount_range
|
||||
.iter_mut()
|
||||
@@ -604,6 +651,7 @@ impl UTXOCohorts<Rw> {
|
||||
under_amount,
|
||||
epoch,
|
||||
class,
|
||||
entry,
|
||||
type_,
|
||||
..
|
||||
} = self;
|
||||
@@ -676,6 +724,19 @@ impl UTXOCohorts<Rw> {
|
||||
.compute_rest_part2(prices, starting_lengths, ss, au, exit)
|
||||
})
|
||||
}),
|
||||
Box::new(|| {
|
||||
entry.par_iter_mut().try_for_each(|v| {
|
||||
v.metrics.compute_rest_part2(
|
||||
blocks,
|
||||
prices,
|
||||
starting_lengths,
|
||||
height_to_market_cap,
|
||||
ss,
|
||||
au,
|
||||
exit,
|
||||
)
|
||||
})
|
||||
}),
|
||||
Box::new(|| {
|
||||
amount_range.par_iter_mut().try_for_each(|v| {
|
||||
v.metrics
|
||||
@@ -730,6 +791,9 @@ impl UTXOCohorts<Rw> {
|
||||
for v in self.class.iter_mut() {
|
||||
vecs.extend(v.metrics.collect_all_vecs_mut());
|
||||
}
|
||||
for v in self.entry.iter_mut() {
|
||||
vecs.extend(v.metrics.collect_all_vecs_mut());
|
||||
}
|
||||
for v in self.amount_range.iter_mut() {
|
||||
vecs.extend(v.metrics.collect_all_vecs_mut());
|
||||
}
|
||||
@@ -813,7 +877,7 @@ impl UTXOCohorts<Rw> {
|
||||
|
||||
/// Aggregate RealizedFull fields from age_range states and push to all/sth/lth.
|
||||
/// Called during the block loop after separate cohorts' push_state but before reset.
|
||||
pub(crate) fn push_overlapping(&mut self, height_price: Cents) {
|
||||
pub(crate) fn push_overlapping(&mut self, height_price: Cents) -> Cents {
|
||||
let Self {
|
||||
all,
|
||||
sth,
|
||||
@@ -852,7 +916,7 @@ impl UTXOCohorts<Rw> {
|
||||
}
|
||||
}
|
||||
|
||||
all.metrics.realized.push_accum(&all_acc);
|
||||
let all_capitalized_price = all.metrics.realized.push_accum(&all_acc);
|
||||
sth.metrics.realized.push_accum(&sth_acc);
|
||||
lth.metrics.realized.push_accum(<h_acc);
|
||||
|
||||
@@ -880,6 +944,8 @@ impl UTXOCohorts<Rw> {
|
||||
.unrealized
|
||||
.capitalized_cap_in_loss_raw
|
||||
.push(CentsSquaredSats::new(lth_ccap.1));
|
||||
|
||||
all_capitalized_price
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -50,6 +50,22 @@ impl UTXOCohorts {
|
||||
let lth = self.caches.fenwick.percentiles_lth();
|
||||
push_cost_basis(<h, lth_d, &mut self.lth.metrics.cost_basis);
|
||||
|
||||
let (discount_d, premium_d) = self.caches.fenwick.entry_density(spot_price);
|
||||
|
||||
let discount = self.caches.fenwick.percentiles_discount_entry();
|
||||
push_cost_basis(
|
||||
&discount,
|
||||
discount_d,
|
||||
&mut self.entry.discount.metrics.cost_basis,
|
||||
);
|
||||
|
||||
let premium = self.caches.fenwick.percentiles_premium_entry();
|
||||
push_cost_basis(
|
||||
&premium,
|
||||
premium_d,
|
||||
&mut self.entry.premium.metrics.cost_basis,
|
||||
);
|
||||
|
||||
let prof = self.caches.fenwick.profitability(spot_price);
|
||||
push_profitability(&prof, &mut self.profitability);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use brk_cohort::EntryPrice;
|
||||
use brk_types::{Cents, CostBasisSnapshot, Height, Timestamp};
|
||||
use vecdb::Rw;
|
||||
|
||||
@@ -12,6 +13,7 @@ impl UTXOCohorts<Rw> {
|
||||
/// - The "under_1h" age cohort (all new UTXOs start at 0 hours old)
|
||||
/// - The appropriate epoch cohort based on block height
|
||||
/// - The appropriate class cohort based on block timestamp
|
||||
/// - The immutable entry valuation cohort based on creation price versus anchor
|
||||
/// - The appropriate output type cohort (P2PKH, P2SH, etc.)
|
||||
/// - The appropriate amount range cohort based on value
|
||||
pub(crate) fn receive(
|
||||
@@ -20,13 +22,14 @@ impl UTXOCohorts<Rw> {
|
||||
height: Height,
|
||||
timestamp: Timestamp,
|
||||
price: Cents,
|
||||
entry: EntryPrice,
|
||||
) {
|
||||
let supply_state = received.spendable_supply;
|
||||
|
||||
// Pre-compute snapshot once for the 3 cohorts sharing the same supply_state
|
||||
// Pre-compute snapshot once for cohorts sharing the block-level supply_state
|
||||
let snapshot = CostBasisSnapshot::from_utxo(price, &supply_state);
|
||||
|
||||
// New UTXOs go into under_1h, current epoch, and current class
|
||||
// New UTXOs go into under_1h plus immutable creation cohorts
|
||||
self.age_range
|
||||
.under_1h
|
||||
.state
|
||||
@@ -45,6 +48,12 @@ impl UTXOCohorts<Rw> {
|
||||
.unwrap()
|
||||
.receive_utxo_snapshot(&supply_state, &snapshot);
|
||||
}
|
||||
self.entry
|
||||
.get_mut(entry)
|
||||
.state
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.receive_utxo_snapshot(&supply_state, &snapshot);
|
||||
|
||||
// Update output type cohorts (skip types with no outputs this block)
|
||||
self.type_.iter_typed_mut().for_each(|(output_type, vecs)| {
|
||||
|
||||
@@ -49,7 +49,7 @@ impl UTXOCohorts<Rw> {
|
||||
// This is the max price between receive and send heights
|
||||
let peak_price = price_range_max.max_between(receive_height, send_height);
|
||||
|
||||
// Pre-compute once for age_range, epoch, year (all share sent.spendable_supply)
|
||||
// Pre-compute once for cohorts sharing the sent supply.
|
||||
if let Some(pre) = SendPrecomputed::new(
|
||||
&sent.spendable_supply,
|
||||
current_price,
|
||||
@@ -75,6 +75,12 @@ impl UTXOCohorts<Rw> {
|
||||
.unwrap()
|
||||
.send_utxo_precomputed(&sent.spendable_supply, &pre);
|
||||
}
|
||||
self.entry
|
||||
.get_mut(block_state.entry)
|
||||
.state
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.send_utxo_precomputed(&sent.spendable_supply, &pre);
|
||||
} else if sent.spendable_supply.utxo_count > 0 {
|
||||
// Zero-value UTXOs: just subtract supply
|
||||
self.age_range.get_mut(age).state.as_mut().unwrap().supply -=
|
||||
@@ -85,6 +91,12 @@ impl UTXOCohorts<Rw> {
|
||||
if let Some(v) = self.class.mut_vec_from_timestamp(block_state.timestamp) {
|
||||
v.state.as_mut().unwrap().supply -= &sent.spendable_supply;
|
||||
}
|
||||
self.entry
|
||||
.get_mut(block_state.entry)
|
||||
.state
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.supply -= &sent.spendable_supply;
|
||||
}
|
||||
|
||||
// Update output type cohorts (skip zero-supply entries)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use brk_cohort::ByAddrType;
|
||||
use brk_cohort::{ByAddrType, EntryPrice};
|
||||
use brk_error::Result;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_types::{
|
||||
@@ -46,6 +46,7 @@ pub(crate) fn process_blocks(
|
||||
last_height: Height,
|
||||
chain_state: &mut Vec<BlockState>,
|
||||
tx_index_to_height: &mut RangeMap<TxIndex, Height>,
|
||||
mut entry_anchor: Cents,
|
||||
cached_prices: &[Cents],
|
||||
cached_timestamps: &[Timestamp],
|
||||
cached_price_range_max: &PriceRangeMax,
|
||||
@@ -370,9 +371,14 @@ pub(crate) fn process_blocks(
|
||||
.iterate(Sats::FIFTY_BTC, OutputType::P2PK65);
|
||||
}
|
||||
|
||||
let entry = EntryPrice::from_is_discount(
|
||||
entry_anchor == Cents::ZERO || block_price <= entry_anchor,
|
||||
);
|
||||
|
||||
// Push current block state before processing cohort updates
|
||||
chain_state.push(BlockState {
|
||||
supply: transacted.spendable_supply,
|
||||
entry,
|
||||
price: block_price,
|
||||
timestamp,
|
||||
});
|
||||
@@ -411,7 +417,7 @@ pub(crate) fn process_blocks(
|
||||
|| {
|
||||
// UTXO cohorts receive/send
|
||||
vecs.utxo_cohorts
|
||||
.receive(transacted, height, timestamp, block_price);
|
||||
.receive(transacted, height, timestamp, block_price, entry);
|
||||
if let Some(min_h) =
|
||||
vecs.utxo_cohorts
|
||||
.send(height_to_sent, chain_state, ctx.price_range_max)
|
||||
@@ -460,7 +466,7 @@ pub(crate) fn process_blocks(
|
||||
let is_last_of_day = is_last_of_day[offset];
|
||||
let date_opt = is_last_of_day.then(|| Date::from(timestamp));
|
||||
|
||||
push_cohort_states(
|
||||
entry_anchor = push_cohort_states(
|
||||
&mut vecs.utxo_cohorts,
|
||||
&mut vecs.addr_cohorts,
|
||||
height,
|
||||
@@ -527,7 +533,7 @@ fn push_cohort_states(
|
||||
addr_cohorts: &mut AddrCohorts,
|
||||
height: Height,
|
||||
height_price: Cents,
|
||||
) {
|
||||
) -> Cents {
|
||||
// Phase 1: push + unrealized (no reset yet, states still needed for aggregation)
|
||||
rayon::join(
|
||||
|| {
|
||||
@@ -545,7 +551,7 @@ fn push_cohort_states(
|
||||
);
|
||||
|
||||
// Phase 2: aggregate age_range states → push to overlapping cohorts
|
||||
utxo_cohorts.push_overlapping(height_price);
|
||||
let all_capitalized_price = utxo_cohorts.push_overlapping(height_price);
|
||||
|
||||
// Phase 3: reset per-block values
|
||||
utxo_cohorts
|
||||
@@ -554,4 +560,6 @@ fn push_cohort_states(
|
||||
addr_cohorts
|
||||
.iter_separate_mut()
|
||||
.for_each(|v| v.reset_single_iteration_values());
|
||||
|
||||
all_capitalized_price
|
||||
}
|
||||
|
||||
@@ -206,7 +206,7 @@ impl RealizedFull {
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub(crate) fn push_accum(&mut self, accum: &RealizedFullAccum) {
|
||||
pub(crate) fn push_accum(&mut self, accum: &RealizedFullAccum) -> Cents {
|
||||
self.cap_raw.push(accum.cap_raw);
|
||||
self.capitalized.cap_raw.push(accum.capitalized_cap_raw);
|
||||
|
||||
@@ -221,6 +221,8 @@ impl RealizedFull {
|
||||
self.capitalized.price.cents.height.push(capitalized_price);
|
||||
|
||||
self.peak_regret.value.block.cents.push(accum.peak_regret());
|
||||
|
||||
capitalized_price
|
||||
}
|
||||
|
||||
pub(crate) fn compute_rest_part1(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::ops::{Add, AddAssign, SubAssign};
|
||||
|
||||
use brk_cohort::EntryPrice;
|
||||
use brk_types::{Cents, SupplyState, Timestamp};
|
||||
use serde::Serialize;
|
||||
|
||||
@@ -8,6 +9,8 @@ pub struct BlockState {
|
||||
#[serde(flatten)]
|
||||
pub supply: SupplyState,
|
||||
#[serde(skip)]
|
||||
pub entry: EntryPrice,
|
||||
#[serde(skip)]
|
||||
pub price: Cents,
|
||||
#[serde(skip)]
|
||||
pub timestamp: Timestamp,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use brk_cohort::{ByAddrType, Filter};
|
||||
use brk_cohort::{ByAddrType, EntryPrice, Filter};
|
||||
use brk_error::Result;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_traversable::Traversable;
|
||||
@@ -436,13 +436,34 @@ impl Vecs {
|
||||
let end = usize::from(recovered_height);
|
||||
debug!("building supply_state vec for {} heights", recovered_height);
|
||||
let supply_state_data: Vec<_> = self.supply_state.collect_range_at(0, end);
|
||||
let capitalized_price_data: Vec<_> = self
|
||||
.utxo_cohorts
|
||||
.all
|
||||
.metrics
|
||||
.realized
|
||||
.capitalized
|
||||
.price
|
||||
.cents
|
||||
.height
|
||||
.collect_range_at(0, end);
|
||||
|
||||
let mut entry_anchor = Cents::ZERO;
|
||||
chain_state = supply_state_data
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(h, supply)| BlockState {
|
||||
supply,
|
||||
price: self.caches.prices[h],
|
||||
timestamp: self.caches.timestamps[h],
|
||||
.map(|(h, supply)| {
|
||||
let price = self.caches.prices[h];
|
||||
let entry = EntryPrice::from_is_discount(
|
||||
entry_anchor == Cents::ZERO || price <= entry_anchor,
|
||||
);
|
||||
entry_anchor = capitalized_price_data[h];
|
||||
|
||||
BlockState {
|
||||
supply,
|
||||
entry,
|
||||
price,
|
||||
timestamp: self.caches.timestamps[h],
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
debug!("chain_state rebuilt");
|
||||
@@ -474,6 +495,20 @@ impl Vecs {
|
||||
let prices = std::mem::take(&mut self.caches.prices);
|
||||
let timestamps = std::mem::take(&mut self.caches.timestamps);
|
||||
let price_range_max = std::mem::take(&mut self.caches.price_range_max);
|
||||
let entry_anchor = starting_height
|
||||
.decremented()
|
||||
.and_then(|height| {
|
||||
self.utxo_cohorts
|
||||
.all
|
||||
.metrics
|
||||
.realized
|
||||
.capitalized
|
||||
.price
|
||||
.cents
|
||||
.height
|
||||
.collect_one(height)
|
||||
})
|
||||
.unwrap_or(Cents::ZERO);
|
||||
|
||||
process_blocks(
|
||||
self,
|
||||
@@ -486,6 +521,7 @@ impl Vecs {
|
||||
last_height,
|
||||
&mut chain_state,
|
||||
&mut tx_index_to_height,
|
||||
entry_anchor,
|
||||
&prices,
|
||||
×tamps,
|
||||
&price_range_max,
|
||||
|
||||
@@ -250,12 +250,7 @@ impl Vecs {
|
||||
identity: LazyVecFrom1::init(
|
||||
"op_return_index",
|
||||
version,
|
||||
indexer
|
||||
.vecs
|
||||
.scripts
|
||||
.op_return
|
||||
.to_tx_index
|
||||
.read_only_boxed_clone(),
|
||||
indexer.vecs.op_return.to_tx_index.read_only_boxed_clone(),
|
||||
|index, _| index,
|
||||
),
|
||||
},
|
||||
|
||||
@@ -6,6 +6,8 @@ use vecdb::{AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
|
||||
use super::{Vecs, WithInputTypes};
|
||||
use crate::internal::{CoinbasePolicy, PerBlockCumulativeRolling, walk_blocks};
|
||||
|
||||
const WRITE_INTERVAL: usize = 10_000;
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
|
||||
let starting_lengths = indexer.safe_lengths();
|
||||
@@ -37,6 +39,7 @@ impl Vecs {
|
||||
|
||||
let mut itype_cursor = indexer.vecs.inputs.output_type.cursor();
|
||||
let mut fi_in_cursor = indexer.vecs.transactions.first_txin_index.cursor();
|
||||
let mut height = skip;
|
||||
|
||||
walk_blocks(
|
||||
&fi_batch,
|
||||
@@ -65,7 +68,8 @@ impl Vecs {
|
||||
);
|
||||
push_block(&mut self.tx_count, agg.txs_all, &agg.txs_per_type);
|
||||
|
||||
if self.input_count.all.block.batch_limit_reached() {
|
||||
height += 1;
|
||||
if height.is_multiple_of(WRITE_INTERVAL) {
|
||||
let _lock = exit.lock();
|
||||
self.input_count.write()?;
|
||||
self.tx_count.write()?;
|
||||
@@ -79,12 +83,12 @@ impl Vecs {
|
||||
self.input_count.write()?;
|
||||
self.tx_count.write()?;
|
||||
}
|
||||
|
||||
self.input_count
|
||||
.compute_rest(starting_lengths.height, exit)?;
|
||||
self.tx_count.compute_rest(starting_lengths.height, exit)?;
|
||||
}
|
||||
|
||||
self.input_count
|
||||
.compute_rest(starting_lengths.height, exit)?;
|
||||
self.tx_count.compute_rest(starting_lengths.height, exit)?;
|
||||
|
||||
for (otype, source) in self.input_count.by_type.iter_typed() {
|
||||
self.input_share.get_mut(otype).compute_count_ratio(
|
||||
source,
|
||||
|
||||
@@ -3,13 +3,12 @@ use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{blocks, indexes};
|
||||
use crate::blocks;
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
@@ -18,7 +17,7 @@ impl Vecs {
|
||||
let starting_lengths = indexer.safe_lengths();
|
||||
|
||||
self.spent.compute(indexer, exit)?;
|
||||
self.count.compute(indexer, indexes, blocks, exit)?;
|
||||
self.count.compute(indexer, blocks, exit)?;
|
||||
self.per_sec.compute(&self.count, &starting_lengths, exit)?;
|
||||
self.by_type.compute(indexer, exit)?;
|
||||
|
||||
|
||||
@@ -3,27 +3,25 @@ use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{blocks, indexes};
|
||||
use crate::blocks;
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let starting_height = indexer.safe_lengths().height;
|
||||
let window_starts = blocks.lookback.window_starts();
|
||||
self.0.compute(
|
||||
|
||||
self.sum.compute_count_from_indexes(
|
||||
starting_height,
|
||||
&indexes.tx_index.input_count,
|
||||
&indexer.vecs.transactions.first_tx_index,
|
||||
&indexes.height.tx_index_count,
|
||||
&window_starts,
|
||||
&indexer.vecs.inputs.first_txin_index,
|
||||
&indexer.vecs.inputs.outpoint,
|
||||
exit,
|
||||
0,
|
||||
)?;
|
||||
self.compute_rest(starting_height, &window_starts, exit)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -2,10 +2,7 @@ use brk_error::Result;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::Height;
|
||||
use schemars::JsonSchema;
|
||||
use vecdb::{
|
||||
AnyStoredVec, AnyVec, Database, EagerVec, Exit, ImportableVec, PcoVec, ReadableVec, Rw,
|
||||
StorageMode, VecIndex, VecValue, Version, WritableVec,
|
||||
};
|
||||
use vecdb::{Database, EagerVec, Exit, ImportableVec, PcoVec, Rw, StorageMode, Version};
|
||||
|
||||
use crate::{
|
||||
indexes,
|
||||
@@ -52,82 +49,19 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn compute<A>(
|
||||
pub(crate) fn compute_rest(
|
||||
&mut self,
|
||||
max_from: Height,
|
||||
source: &impl ReadableVec<A, T>,
|
||||
first_indexes: &impl ReadableVec<Height, A>,
|
||||
count_indexes: &impl ReadableVec<Height, brk_types::StoredU64>,
|
||||
windows: &WindowStarts<'_>,
|
||||
exit: &Exit,
|
||||
skip_count: usize,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: From<f64> + Default + Copy + Ord,
|
||||
f64: From<T>,
|
||||
A: VecIndex + VecValue + brk_types::CheckedSub<A>,
|
||||
{
|
||||
let combined_version = source.version() + first_indexes.version() + count_indexes.version();
|
||||
|
||||
let mut index = max_from;
|
||||
index = {
|
||||
self.sum
|
||||
.validate_computed_version_or_reset(combined_version)?;
|
||||
index.min(Height::from(self.sum.len()))
|
||||
};
|
||||
index = {
|
||||
self.cumulative
|
||||
.height
|
||||
.validate_computed_version_or_reset(combined_version)?;
|
||||
index.min(Height::from(self.cumulative.height.len()))
|
||||
};
|
||||
|
||||
let start = index.to_usize();
|
||||
|
||||
self.sum.truncate_if_needed_at(start)?;
|
||||
self.cumulative.height.truncate_if_needed_at(start)?;
|
||||
|
||||
let mut cumulative_val = index.decremented().map_or(T::from(0_usize), |idx| {
|
||||
self.cumulative
|
||||
.height
|
||||
.collect_one_at(idx.to_usize())
|
||||
.unwrap_or(T::from(0_usize))
|
||||
});
|
||||
|
||||
let fi_len = first_indexes.len();
|
||||
let first_indexes_batch: Vec<A> = first_indexes.collect_range_at(start, fi_len);
|
||||
let count_indexes_batch: Vec<brk_types::StoredU64> =
|
||||
count_indexes.collect_range_at(start, fi_len);
|
||||
|
||||
first_indexes_batch
|
||||
.into_iter()
|
||||
.zip(count_indexes_batch)
|
||||
.try_for_each(|(first_index, count_index)| -> Result<()> {
|
||||
let count = u64::from(count_index) as usize;
|
||||
let effective_count = count.saturating_sub(skip_count);
|
||||
let effective_first_index = first_index + skip_count.min(count);
|
||||
|
||||
let efi = effective_first_index.to_usize();
|
||||
let sum_val = source.fold_range_at(
|
||||
efi,
|
||||
efi + effective_count,
|
||||
T::from(0_usize),
|
||||
|acc, val| acc + val,
|
||||
);
|
||||
|
||||
self.sum.push(sum_val);
|
||||
cumulative_val += sum_val;
|
||||
self.cumulative.height.push(cumulative_val);
|
||||
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
let _lock = exit.lock();
|
||||
self.sum.write()?;
|
||||
self.cumulative.height.write()?;
|
||||
drop(_lock);
|
||||
|
||||
self.cumulative
|
||||
.height
|
||||
.compute_cumulative(max_from, &self.sum, exit)?;
|
||||
self.rolling.compute(max_from, windows, &self.sum, exit)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
use brk_error::Result;
|
||||
use brk_indexer::{Indexer, Lengths};
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::TxIndex;
|
||||
use schemars::JsonSchema;
|
||||
use vecdb::{Database, Exit, LazyVecFrom2, ReadableVec, Rw, StorageMode, Version};
|
||||
|
||||
use crate::{
|
||||
indexes,
|
||||
internal::{ComputedVecValue, NumericValue, TxDerivedDistribution},
|
||||
};
|
||||
|
||||
#[derive(Traversable)]
|
||||
pub struct LazyPerTxDistribution<T, S1, S2, M: StorageMode = Rw>
|
||||
where
|
||||
T: ComputedVecValue + PartialOrd + JsonSchema,
|
||||
S1: ComputedVecValue,
|
||||
S2: ComputedVecValue,
|
||||
{
|
||||
pub tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
|
||||
#[traversable(flatten)]
|
||||
pub distribution: TxDerivedDistribution<T, M>,
|
||||
}
|
||||
|
||||
impl<T, S1, S2> LazyPerTxDistribution<T, S1, S2>
|
||||
where
|
||||
T: NumericValue + JsonSchema,
|
||||
S1: ComputedVecValue + JsonSchema,
|
||||
S2: ComputedVecValue + JsonSchema,
|
||||
{
|
||||
pub(crate) fn forced_import(
|
||||
db: &Database,
|
||||
name: &str,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
|
||||
) -> Result<Self> {
|
||||
let distribution = TxDerivedDistribution::forced_import(db, name, version, indexes)?;
|
||||
Ok(Self {
|
||||
tx_index,
|
||||
distribution,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn derive_from(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
starting_lengths: &Lengths,
|
||||
exit: &Exit,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: Copy + Ord + From<f64> + Default,
|
||||
f64: From<T>,
|
||||
LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>: ReadableVec<TxIndex, T>,
|
||||
{
|
||||
self.distribution
|
||||
.derive_from(indexer, indexes, starting_lengths, &self.tx_index, exit)
|
||||
}
|
||||
}
|
||||
@@ -1,36 +1,33 @@
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{TxIndex, Version};
|
||||
use schemars::JsonSchema;
|
||||
use vecdb::{LazyVecFrom2, UnaryTransform};
|
||||
use vecdb::{LazyVecFrom1, UnaryTransform};
|
||||
|
||||
use crate::internal::{ComputedVecValue, LazyTxDerivedDistribution, TxDerivedDistribution};
|
||||
|
||||
/// Like `LazyPerTxDistribution` but with a lazy-derived distribution
|
||||
/// (transformed from another type's distribution rather than eagerly computed).
|
||||
/// Per-transaction lazy values with a distribution transformed from another type's distribution.
|
||||
#[derive(Clone, Traversable)]
|
||||
pub struct LazyPerTxDistributionTransformed<T, S1, S2, DSource>
|
||||
pub struct LazyPerTxDistributionTransformed<T, S, DSource>
|
||||
where
|
||||
T: ComputedVecValue + JsonSchema,
|
||||
S1: ComputedVecValue,
|
||||
S2: ComputedVecValue,
|
||||
S: ComputedVecValue,
|
||||
DSource: ComputedVecValue,
|
||||
{
|
||||
pub tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
|
||||
pub tx_index: LazyVecFrom1<TxIndex, T, TxIndex, S>,
|
||||
#[traversable(flatten)]
|
||||
pub distribution: LazyTxDerivedDistribution<T, DSource>,
|
||||
}
|
||||
|
||||
impl<T, S1, S2, DSource> LazyPerTxDistributionTransformed<T, S1, S2, DSource>
|
||||
impl<T, S, DSource> LazyPerTxDistributionTransformed<T, S, DSource>
|
||||
where
|
||||
T: ComputedVecValue + JsonSchema + 'static,
|
||||
S1: ComputedVecValue + JsonSchema,
|
||||
S2: ComputedVecValue + JsonSchema,
|
||||
S: ComputedVecValue + JsonSchema,
|
||||
DSource: ComputedVecValue + JsonSchema,
|
||||
{
|
||||
pub(crate) fn new<F: UnaryTransform<DSource, T>>(
|
||||
name: &str,
|
||||
version: Version,
|
||||
tx_index: LazyVecFrom2<TxIndex, T, TxIndex, S1, TxIndex, S2>,
|
||||
tx_index: LazyVecFrom1<TxIndex, T, TxIndex, S>,
|
||||
source_distribution: &TxDerivedDistribution<DSource>,
|
||||
) -> Self {
|
||||
let distribution =
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
mod derived;
|
||||
mod distribution;
|
||||
mod lazy_derived;
|
||||
mod lazy_distribution;
|
||||
mod lazy_distribution_transformed;
|
||||
|
||||
pub use derived::*;
|
||||
pub use distribution::*;
|
||||
pub use lazy_derived::*;
|
||||
pub use lazy_distribution::*;
|
||||
pub use lazy_distribution_transformed::*;
|
||||
|
||||
@@ -146,11 +146,11 @@ impl UnaryTransform<StoredU64, Weight> for VBytesToWeight {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct VSizeToWeight;
|
||||
pub struct WeightToVSize;
|
||||
|
||||
impl UnaryTransform<VSize, Weight> for VSizeToWeight {
|
||||
impl UnaryTransform<Weight, VSize> for WeightToVSize {
|
||||
#[inline(always)]
|
||||
fn apply(vsize: VSize) -> Weight {
|
||||
Weight::from(vsize)
|
||||
fn apply(weight: Weight) -> VSize {
|
||||
VSize::from(weight)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ mod specialized;
|
||||
pub use arithmetic::{
|
||||
BlocksToDaysF32, DifficultyToHashF64, HalveCents, HalveDollars, HalveSats, HalveSatsToBitcoin,
|
||||
Identity, MaskSats, OneMinusBp16, OneMinusF64, ReturnF32Tenths, ReturnI8, ReturnU16,
|
||||
ThsToPhsF32, VBytesToWeight, VSizeToWeight,
|
||||
ThsToPhsF32, VBytesToWeight, WeightToVSize,
|
||||
};
|
||||
pub use bps::{
|
||||
Bp16ToFloat, Bp16ToPercent, Bp32ToFloat, Bp32ToPercent, Bps16ToFloat, Bps16ToPercent,
|
||||
|
||||
@@ -20,6 +20,7 @@ mod internal;
|
||||
mod investing;
|
||||
mod market;
|
||||
mod mining;
|
||||
mod op_return;
|
||||
mod outputs;
|
||||
mod pools;
|
||||
pub mod price;
|
||||
@@ -44,9 +45,10 @@ pub struct Computer<M: StorageMode = Rw> {
|
||||
pub supply: Box<supply::Vecs<M>>,
|
||||
pub inputs: Box<inputs::Vecs<M>>,
|
||||
pub outputs: Box<outputs::Vecs<M>>,
|
||||
pub op_return: Box<op_return::Vecs<M>>,
|
||||
}
|
||||
|
||||
const VERSION: Version = Version::new(6);
|
||||
const VERSION: Version = Version::new(7);
|
||||
|
||||
impl Computer {
|
||||
pub fn forced_import(outputs_path: &Path, indexer: &Indexer) -> Result<Self> {
|
||||
@@ -87,8 +89,9 @@ impl Computer {
|
||||
|
||||
let cached_starts = blocks.lookback.cached_window_starts();
|
||||
|
||||
let (inputs, outputs, mining, transactions, pools, cointime) =
|
||||
timed("Imported inputs/outputs/mining/tx/pools/cointime", || {
|
||||
let (inputs, outputs, mining, transactions, pools, cointime, op_return) = timed(
|
||||
"Imported inputs/outputs/mining/tx/pools/cointime/op_return",
|
||||
|| {
|
||||
thread::scope(|s| -> Result<_> {
|
||||
let inputs_handle = big_thread().spawn_scoped(s, || -> Result<_> {
|
||||
Ok(Box::new(inputs::Vecs::forced_import(
|
||||
@@ -143,15 +146,34 @@ impl Computer {
|
||||
&cached_starts,
|
||||
)?);
|
||||
|
||||
let op_return_handle = big_thread().spawn_scoped(s, || -> Result<_> {
|
||||
Ok(Box::new(op_return::Vecs::forced_import(
|
||||
&computed_path,
|
||||
VERSION,
|
||||
&indexes,
|
||||
&cached_starts,
|
||||
)?))
|
||||
})?;
|
||||
|
||||
let inputs = inputs_handle.join().unwrap()?;
|
||||
let outputs = outputs_handle.join().unwrap()?;
|
||||
let mining = mining_handle.join().unwrap()?;
|
||||
let transactions = transactions_handle.join().unwrap()?;
|
||||
let pools = pools_handle.join().unwrap()?;
|
||||
let op_return = op_return_handle.join().unwrap()?;
|
||||
|
||||
Ok((inputs, outputs, mining, transactions, pools, cointime))
|
||||
Ok((
|
||||
inputs,
|
||||
outputs,
|
||||
mining,
|
||||
transactions,
|
||||
pools,
|
||||
cointime,
|
||||
op_return,
|
||||
))
|
||||
})
|
||||
})?;
|
||||
},
|
||||
)?;
|
||||
|
||||
// Market, indicators, and distribution are independent; import in parallel.
|
||||
// Supply depends on distribution so it runs after.
|
||||
@@ -225,6 +247,7 @@ impl Computer {
|
||||
inputs,
|
||||
price,
|
||||
outputs,
|
||||
op_return,
|
||||
};
|
||||
|
||||
Self::retain_databases(&computed_path)?;
|
||||
@@ -249,6 +272,7 @@ impl Computer {
|
||||
supply::DB_NAME,
|
||||
inputs::DB_NAME,
|
||||
outputs::DB_NAME,
|
||||
op_return::DB_NAME,
|
||||
];
|
||||
|
||||
if !computed_path.exists() {
|
||||
@@ -292,8 +316,7 @@ impl Computer {
|
||||
let (inputs_result, prices_result) = rayon::join(
|
||||
|| {
|
||||
timed("Computed inputs", || {
|
||||
self.inputs
|
||||
.compute(indexer, &self.indexes, &self.blocks, exit)
|
||||
self.inputs.compute(indexer, &self.blocks, exit)
|
||||
})
|
||||
},
|
||||
|| {
|
||||
@@ -337,18 +360,19 @@ impl Computer {
|
||||
})
|
||||
});
|
||||
|
||||
let op_return = scope.spawn(|| {
|
||||
timed("Computed OP_RETURN", || {
|
||||
self.op_return.compute(indexer, &self.blocks, exit)
|
||||
})
|
||||
});
|
||||
|
||||
timed("Computed outputs", || {
|
||||
self.outputs.compute(
|
||||
indexer,
|
||||
&self.indexes,
|
||||
&self.inputs,
|
||||
&self.blocks,
|
||||
&self.price,
|
||||
exit,
|
||||
)
|
||||
self.outputs
|
||||
.compute(indexer, &self.inputs, &self.blocks, &self.price, exit)
|
||||
})?;
|
||||
|
||||
tx_mining.join().unwrap()?;
|
||||
op_return.join().unwrap()?;
|
||||
market.join().unwrap()?;
|
||||
Ok(())
|
||||
})?;
|
||||
@@ -502,7 +526,8 @@ impl_iter_named!(
|
||||
distribution,
|
||||
supply,
|
||||
inputs,
|
||||
outputs
|
||||
outputs,
|
||||
op_return
|
||||
);
|
||||
|
||||
fn timed<T>(label: &str, f: impl FnOnce() -> T) -> T {
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::OpReturnKind;
|
||||
|
||||
macro_rules! define_by_kind {
|
||||
($($field:ident => $kind:ident),+ $(,)?) => {
|
||||
#[derive(Traversable)]
|
||||
pub struct ByKind<T> {
|
||||
$(pub $field: T),+
|
||||
}
|
||||
|
||||
impl<T> ByKind<T> {
|
||||
pub fn try_new<E>(
|
||||
mut create: impl FnMut(OpReturnKind, &'static str) -> Result<T, E>,
|
||||
) -> Result<Self, E> {
|
||||
Ok(Self {
|
||||
$($field: create(OpReturnKind::$kind, stringify!($field))?),+
|
||||
})
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> impl Iterator<Item = &T> {
|
||||
[$( &self.$field ),+].into_iter()
|
||||
}
|
||||
|
||||
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> {
|
||||
let Self { $($field),+ } = self;
|
||||
[$($field),+].into_iter()
|
||||
}
|
||||
|
||||
pub fn iter_typed(&self) -> impl Iterator<Item = (OpReturnKind, &T)> {
|
||||
[$( (OpReturnKind::$kind, &self.$field) ),+].into_iter()
|
||||
}
|
||||
|
||||
pub fn iter_typed_mut(
|
||||
&mut self,
|
||||
) -> impl Iterator<Item = (OpReturnKind, &mut T)> {
|
||||
let Self { $($field),+ } = self;
|
||||
[$( (OpReturnKind::$kind, $field) ),+].into_iter()
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
define_by_kind! {
|
||||
runes => Runes,
|
||||
veri_block => VeriBlock,
|
||||
omni => Omni,
|
||||
stacks => Stacks,
|
||||
blockstack => Blockstack,
|
||||
colu => Colu,
|
||||
open_assets => OpenAssets,
|
||||
komodo => Komodo,
|
||||
coin_spark => CoinSpark,
|
||||
poet => Poet,
|
||||
docproof => Docproof,
|
||||
open_timestamps => OpenTimestamps,
|
||||
factom => Factom,
|
||||
eternity_wall => EternityWall,
|
||||
memo => Memo,
|
||||
bitproof => Bitproof,
|
||||
ascribe => Ascribe,
|
||||
stampery => Stampery,
|
||||
epobc => Epobc,
|
||||
bare_hash => BareHash,
|
||||
text => Text,
|
||||
empty => Empty,
|
||||
unknown => Unknown,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::convert::Infallible;
|
||||
|
||||
use brk_types::OpReturnKind;
|
||||
|
||||
use super::ByKind;
|
||||
|
||||
#[test]
|
||||
fn covers_every_kind_in_discriminant_order() {
|
||||
let by_kind =
|
||||
ByKind::try_new(|kind, _| Ok::<_, Infallible>(kind)).expect("infallible constructor");
|
||||
let kinds: Vec<_> = by_kind.iter_typed().collect();
|
||||
|
||||
assert_eq!(kinds.len(), OpReturnKind::Unknown as usize + 1);
|
||||
for (index, (kind, value)) in kinds.into_iter().enumerate() {
|
||||
assert_eq!(kind as usize, index);
|
||||
assert_eq!(kind, *value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,319 @@
|
||||
use brk_error::Result;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_types::{BasisPoints16, Height, OpReturnKind, StoredU64, VSize};
|
||||
use vecdb::{AnyVec, Exit, ReadableVec, VecIndex};
|
||||
|
||||
use super::{Breakdown, Vecs, vecs::Totals};
|
||||
use crate::{
|
||||
blocks,
|
||||
internal::{PercentPerBlock, RatioU64Bp16},
|
||||
};
|
||||
|
||||
const KIND_COUNT: usize = OpReturnKind::Unknown as usize + 1;
|
||||
const OLD_STANDARD_MAX_POST_OP_RETURN_BYTES: u64 = 82;
|
||||
const WRITE_INTERVAL: usize = 10_000;
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct PolicyTotals {
|
||||
pre_v30_standard: Totals,
|
||||
pre_v30_nonstandard: Totals,
|
||||
oversized: Totals,
|
||||
multiple: Totals,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct Carrier {
|
||||
kinds: u32,
|
||||
output_count: u64,
|
||||
data_bytes: u64,
|
||||
oversized_output_count: u64,
|
||||
oversized_data_bytes: u64,
|
||||
vsize: VSize,
|
||||
}
|
||||
|
||||
impl Carrier {
|
||||
fn add_output(&mut self, kind: OpReturnKind, data_bytes: u64) {
|
||||
self.kinds |= kind_bit(kind);
|
||||
self.output_count += 1;
|
||||
self.data_bytes += data_bytes;
|
||||
if data_bytes > OLD_STANDARD_MAX_POST_OP_RETURN_BYTES {
|
||||
self.oversized_output_count += 1;
|
||||
self.oversized_data_bytes += data_bytes;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
blocks: &blocks::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
self.db.sync_bg_tasks()?;
|
||||
|
||||
let starting_lengths = indexer.safe_lengths();
|
||||
let raw = &indexer.vecs.op_return;
|
||||
let txs = &indexer.vecs.transactions;
|
||||
let version = raw.first_index.version()
|
||||
+ raw.to_tx_index.version()
|
||||
+ raw.kind.version()
|
||||
+ raw.post_op_return_bytes.version()
|
||||
+ txs.weight.version();
|
||||
|
||||
self.validate_and_truncate(version, starting_lengths.height)?;
|
||||
|
||||
let skip = self.min_len();
|
||||
let end = raw.first_index.len();
|
||||
if skip < end {
|
||||
self.truncate_if_needed_at(skip)?;
|
||||
|
||||
let op_return_len = raw.to_tx_index.len();
|
||||
let mut tx_cursor = raw.to_tx_index.cursor();
|
||||
let mut kind_cursor = raw.kind.cursor();
|
||||
let mut post_op_return_bytes = raw.post_op_return_bytes.cursor();
|
||||
let mut first_index_cursor = raw.first_index.cursor();
|
||||
let mut weight_cursor = txs.weight.cursor();
|
||||
first_index_cursor.advance(skip);
|
||||
let mut start = first_index_cursor.next().unwrap().to_usize();
|
||||
|
||||
for height in skip..end {
|
||||
let block_end = if height + 1 < end {
|
||||
first_index_cursor.next().unwrap().to_usize()
|
||||
} else {
|
||||
op_return_len
|
||||
};
|
||||
|
||||
tx_cursor.advance(start - tx_cursor.position());
|
||||
kind_cursor.advance(start - kind_cursor.position());
|
||||
post_op_return_bytes.advance(start - post_op_return_bytes.position());
|
||||
|
||||
let mut total = Totals::default();
|
||||
let mut by_kind = [Totals::default(); KIND_COUNT];
|
||||
let mut policy = PolicyTotals::default();
|
||||
let mut current_tx = None;
|
||||
let mut carrier = Carrier::default();
|
||||
|
||||
for _ in start..block_end {
|
||||
let tx_index = tx_cursor.next().unwrap();
|
||||
let kind = kind_cursor.next().unwrap();
|
||||
let bytes = u32::from(post_op_return_bytes.next().unwrap()) as u64;
|
||||
let kind_index = kind as usize;
|
||||
|
||||
if current_tx != Some(tx_index) {
|
||||
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
|
||||
current_tx = Some(tx_index);
|
||||
carrier = Carrier::default();
|
||||
|
||||
let tx_position = tx_index.to_usize();
|
||||
weight_cursor.advance(tx_position - weight_cursor.position());
|
||||
carrier.vsize = VSize::from(weight_cursor.next().unwrap());
|
||||
}
|
||||
|
||||
total.data_bytes += bytes;
|
||||
by_kind[kind_index].output_count += 1;
|
||||
by_kind[kind_index].data_bytes += bytes;
|
||||
carrier.add_output(kind, bytes);
|
||||
}
|
||||
|
||||
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
|
||||
|
||||
self.total.push(total);
|
||||
for (kind, metrics) in self.by_kind.iter_typed_mut() {
|
||||
metrics.push(by_kind[kind as usize]);
|
||||
}
|
||||
self.policy.pre_v30_standard.push(policy.pre_v30_standard);
|
||||
self.policy
|
||||
.pre_v30_nonstandard
|
||||
.push(policy.pre_v30_nonstandard);
|
||||
self.policy.oversized.push(policy.oversized);
|
||||
self.policy.multiple.push(policy.multiple);
|
||||
|
||||
if (height + 1).is_multiple_of(WRITE_INTERVAL) {
|
||||
let _lock = exit.lock();
|
||||
self.write()?;
|
||||
}
|
||||
start = block_end;
|
||||
}
|
||||
|
||||
let _lock = exit.lock();
|
||||
self.write()?;
|
||||
}
|
||||
|
||||
self.compute_cumulative(starting_lengths.height, exit)?;
|
||||
let block_size = &blocks.size.size.cumulative.height;
|
||||
compute_data_share(
|
||||
starting_lengths.height,
|
||||
&mut self.total.chain_share,
|
||||
&self.total.metrics.data_bytes.cumulative.height,
|
||||
block_size,
|
||||
exit,
|
||||
)?;
|
||||
let total_data = &self.total.metrics.data_bytes.cumulative.height;
|
||||
for breakdown in self.by_kind.iter_mut() {
|
||||
compute_breakdown_data_shares(
|
||||
starting_lengths.height,
|
||||
breakdown,
|
||||
total_data,
|
||||
block_size,
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
for policy in self.policy.iter_mut() {
|
||||
compute_breakdown_data_shares(
|
||||
starting_lengths.height,
|
||||
policy,
|
||||
total_data,
|
||||
block_size,
|
||||
exit,
|
||||
)?;
|
||||
}
|
||||
|
||||
let exit = exit.clone();
|
||||
self.db.run_bg(move |db| {
|
||||
let _lock = exit.lock();
|
||||
db.compact_deferred_default()
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn compute_breakdown_data_shares(
|
||||
max_from: Height,
|
||||
breakdown: &mut Breakdown,
|
||||
total_data: &impl ReadableVec<Height, StoredU64>,
|
||||
block_size: &impl ReadableVec<Height, StoredU64>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let data = &breakdown.metrics.total.data_bytes.cumulative.height;
|
||||
compute_data_share(max_from, &mut breakdown.chain_share, data, block_size, exit)?;
|
||||
compute_data_share(max_from, &mut breakdown.data_share, data, total_data, exit)
|
||||
}
|
||||
|
||||
fn compute_data_share(
|
||||
max_from: Height,
|
||||
target: &mut PercentPerBlock<BasisPoints16>,
|
||||
data: &impl ReadableVec<Height, StoredU64>,
|
||||
block_size: &impl ReadableVec<Height, StoredU64>,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
target.compute_binary::<StoredU64, StoredU64, RatioU64Bp16>(max_from, data, block_size, exit)
|
||||
}
|
||||
|
||||
fn finalize_transaction(
|
||||
total: &mut Totals,
|
||||
by_kind: &mut [Totals; KIND_COUNT],
|
||||
policy: &mut PolicyTotals,
|
||||
carrier: Carrier,
|
||||
) {
|
||||
if carrier.output_count == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
add_carrier(total, carrier.vsize);
|
||||
let mut kinds = carrier.kinds;
|
||||
while kinds != 0 {
|
||||
let kind_index = kinds.trailing_zeros() as usize;
|
||||
add_carrier(&mut by_kind[kind_index], carrier.vsize);
|
||||
kinds &= kinds - 1;
|
||||
}
|
||||
|
||||
if carrier.oversized_output_count > 0 {
|
||||
policy.oversized.output_count += carrier.oversized_output_count;
|
||||
policy.oversized.data_bytes += carrier.oversized_data_bytes;
|
||||
add_carrier(&mut policy.oversized, carrier.vsize);
|
||||
}
|
||||
|
||||
if carrier.output_count > 1 {
|
||||
policy.multiple.output_count += carrier.output_count;
|
||||
policy.multiple.data_bytes += carrier.data_bytes;
|
||||
add_carrier(&mut policy.multiple, carrier.vsize);
|
||||
}
|
||||
|
||||
if carrier.oversized_output_count > 0 || carrier.output_count > 1 {
|
||||
policy.pre_v30_nonstandard.output_count += carrier.output_count;
|
||||
policy.pre_v30_nonstandard.data_bytes += carrier.data_bytes;
|
||||
add_carrier(&mut policy.pre_v30_nonstandard, carrier.vsize);
|
||||
} else {
|
||||
policy.pre_v30_standard.output_count += carrier.output_count;
|
||||
policy.pre_v30_standard.data_bytes += carrier.data_bytes;
|
||||
add_carrier(&mut policy.pre_v30_standard, carrier.vsize);
|
||||
}
|
||||
}
|
||||
|
||||
fn add_carrier(metrics: &mut Totals, vsize: VSize) {
|
||||
metrics.tx_count += 1;
|
||||
metrics.tx_vsize += vsize;
|
||||
}
|
||||
|
||||
const fn kind_bit(kind: OpReturnKind) -> u32 {
|
||||
1_u32 << kind as u8
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn multiple_kinds_count_one_total_carrier() {
|
||||
let mut total = Totals::default();
|
||||
let mut by_kind = [Totals::default(); KIND_COUNT];
|
||||
let mut policy = PolicyTotals::default();
|
||||
let mut carrier = Carrier {
|
||||
vsize: VSize::new(100),
|
||||
..Carrier::default()
|
||||
};
|
||||
carrier.add_output(OpReturnKind::Runes, 15);
|
||||
carrier.add_output(OpReturnKind::Omni, 15);
|
||||
|
||||
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
|
||||
|
||||
assert_eq!(total.tx_count, 1);
|
||||
assert_eq!(by_kind[OpReturnKind::Runes as usize].tx_count, 1);
|
||||
assert_eq!(by_kind[OpReturnKind::Omni as usize].tx_count, 1);
|
||||
assert_eq!(policy.multiple.output_count, 2);
|
||||
assert_eq!(policy.pre_v30_nonstandard.tx_count, 1);
|
||||
assert_eq!(policy.oversized.tx_count, 0);
|
||||
assert_eq!(policy.pre_v30_standard.tx_count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_output_marks_pre_v30_nonstandard_once() {
|
||||
let mut total = Totals::default();
|
||||
let mut by_kind = [Totals::default(); KIND_COUNT];
|
||||
let mut policy = PolicyTotals::default();
|
||||
let mut carrier = Carrier {
|
||||
vsize: VSize::new(120),
|
||||
..Carrier::default()
|
||||
};
|
||||
carrier.add_output(OpReturnKind::Unknown, 83);
|
||||
|
||||
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
|
||||
|
||||
assert_eq!(policy.oversized.output_count, 1);
|
||||
assert_eq!(policy.oversized.tx_vsize, VSize::new(120));
|
||||
assert_eq!(policy.pre_v30_nonstandard.tx_count, 1);
|
||||
assert_eq!(policy.multiple.tx_count, 0);
|
||||
assert_eq!(policy.pre_v30_standard.tx_count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_output_is_recorded_directly() {
|
||||
let mut total = Totals::default();
|
||||
let mut by_kind = [Totals::default(); KIND_COUNT];
|
||||
let mut policy = PolicyTotals::default();
|
||||
let mut carrier = Carrier {
|
||||
vsize: VSize::new(100),
|
||||
..Carrier::default()
|
||||
};
|
||||
carrier.add_output(OpReturnKind::Runes, 15);
|
||||
|
||||
finalize_transaction(&mut total, &mut by_kind, &mut policy, carrier);
|
||||
|
||||
assert_eq!(policy.pre_v30_standard.output_count, 1);
|
||||
assert_eq!(policy.pre_v30_standard.data_bytes, 15);
|
||||
assert_eq!(policy.pre_v30_standard.tx_count, 1);
|
||||
assert_eq!(policy.pre_v30_standard.tx_vsize, VSize::new(100));
|
||||
assert_eq!(policy.pre_v30_nonstandard.tx_count, 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
use std::path::Path;
|
||||
|
||||
use brk_error::Result;
|
||||
use brk_types::Version;
|
||||
|
||||
use super::{Breakdown, ByKind, Policy, Total, Vecs};
|
||||
use crate::{
|
||||
indexes,
|
||||
internal::{
|
||||
WindowStartVec, Windows,
|
||||
db_utils::{finalize_db, open_db},
|
||||
},
|
||||
};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn forced_import(
|
||||
parent_path: &Path,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
cached_starts: &Windows<&WindowStartVec>,
|
||||
) -> Result<Self> {
|
||||
let db = open_db(parent_path, super::DB_NAME, 1_000_000)?;
|
||||
let total = Total::forced_import(&db, "op_return", version, indexes, cached_starts)?;
|
||||
let by_kind = ByKind::try_new(|_, name| {
|
||||
Breakdown::forced_import(
|
||||
&db,
|
||||
&format!("op_return_{name}"),
|
||||
version,
|
||||
indexes,
|
||||
cached_starts,
|
||||
)
|
||||
})?;
|
||||
let policy = Policy::forced_import(&db, version, indexes, cached_starts)?;
|
||||
|
||||
let this = Self {
|
||||
db,
|
||||
total,
|
||||
by_kind,
|
||||
policy,
|
||||
};
|
||||
finalize_db(&this.db, &this)?;
|
||||
Ok(this)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
mod by_kind;
|
||||
mod compute;
|
||||
mod import;
|
||||
mod vecs;
|
||||
|
||||
pub use by_kind::ByKind;
|
||||
pub use vecs::{Breakdown, Policy, Total, Vecs};
|
||||
|
||||
pub const DB_NAME: &str = "op_return";
|
||||
@@ -0,0 +1,355 @@
|
||||
use brk_error::Result;
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{BasisPoints16, Height, StoredU64, VSize, Version};
|
||||
use derive_more::{Deref, DerefMut};
|
||||
use vecdb::{AnyStoredVec, AnyVec, Database, Exit, Rw, StorageMode, WritableVec};
|
||||
|
||||
use super::ByKind;
|
||||
use crate::{
|
||||
indexes,
|
||||
internal::{PerBlockCumulativeRolling, PercentPerBlock, WindowStartVec, Windows},
|
||||
};
|
||||
|
||||
pub type Series<T, M = Rw> = PerBlockCumulativeRolling<T, T, M>;
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
pub(super) struct Totals {
|
||||
pub output_count: u64,
|
||||
pub data_bytes: u64,
|
||||
pub tx_count: u64,
|
||||
pub tx_vsize: VSize,
|
||||
}
|
||||
|
||||
#[derive(Traversable)]
|
||||
pub struct TotalMetrics<M: StorageMode = Rw> {
|
||||
pub data_bytes: Series<StoredU64, M>,
|
||||
pub tx_count: Series<StoredU64, M>,
|
||||
pub tx_vsize: Series<VSize, M>,
|
||||
}
|
||||
|
||||
impl TotalMetrics {
|
||||
pub(crate) fn forced_import(
|
||||
db: &Database,
|
||||
prefix: &str,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
cached_starts: &Windows<&WindowStartVec>,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
data_bytes: Series::forced_import(
|
||||
db,
|
||||
&format!("{prefix}_data_bytes"),
|
||||
version,
|
||||
indexes,
|
||||
cached_starts,
|
||||
)?,
|
||||
tx_count: Series::forced_import(
|
||||
db,
|
||||
&format!("{prefix}_tx_count"),
|
||||
version,
|
||||
indexes,
|
||||
cached_starts,
|
||||
)?,
|
||||
tx_vsize: Series::forced_import(
|
||||
db,
|
||||
&format!("{prefix}_tx_vsize"),
|
||||
version,
|
||||
indexes,
|
||||
cached_starts,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.data_bytes
|
||||
.block
|
||||
.len()
|
||||
.min(self.tx_count.block.len())
|
||||
.min(self.tx_vsize.block.len())
|
||||
}
|
||||
|
||||
pub(super) fn push(&mut self, block: Totals) {
|
||||
self.data_bytes.block.push(block.data_bytes.into());
|
||||
self.tx_count.block.push(block.tx_count.into());
|
||||
self.tx_vsize.block.push(block.tx_vsize);
|
||||
}
|
||||
|
||||
fn validate_and_truncate(&mut self, version: Version, height: Height) -> Result<()> {
|
||||
self.data_bytes
|
||||
.block
|
||||
.validate_and_truncate(version, height)?;
|
||||
self.tx_count.block.validate_and_truncate(version, height)?;
|
||||
self.tx_vsize.block.validate_and_truncate(version, height)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> {
|
||||
self.data_bytes.block.truncate_if_needed_at(len)?;
|
||||
self.tx_count.block.truncate_if_needed_at(len)?;
|
||||
self.tx_vsize.block.truncate_if_needed_at(len)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write(&mut self) -> Result<()> {
|
||||
self.data_bytes.block.write()?;
|
||||
self.tx_count.block.write()?;
|
||||
self.tx_vsize.block.write()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compute_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
|
||||
self.data_bytes.compute_rest(max_from, exit)?;
|
||||
self.tx_count.compute_rest(max_from, exit)?;
|
||||
self.tx_vsize.compute_rest(max_from, exit)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deref, DerefMut, Traversable)]
|
||||
pub struct Total<M: StorageMode = Rw> {
|
||||
#[deref]
|
||||
#[deref_mut]
|
||||
#[traversable(flatten)]
|
||||
pub metrics: TotalMetrics<M>,
|
||||
pub chain_share: PercentPerBlock<BasisPoints16, M>,
|
||||
}
|
||||
|
||||
impl Total {
|
||||
pub(crate) fn forced_import(
|
||||
db: &Database,
|
||||
prefix: &str,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
cached_starts: &Windows<&WindowStartVec>,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
metrics: TotalMetrics::forced_import(db, prefix, version, indexes, cached_starts)?,
|
||||
chain_share: PercentPerBlock::forced_import(
|
||||
db,
|
||||
&format!("{prefix}_chain_share"),
|
||||
version,
|
||||
indexes,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Traversable)]
|
||||
pub struct Metrics<M: StorageMode = Rw> {
|
||||
pub output_count: Series<StoredU64, M>,
|
||||
#[traversable(flatten)]
|
||||
pub total: TotalMetrics<M>,
|
||||
}
|
||||
|
||||
impl Metrics {
|
||||
pub(crate) fn forced_import(
|
||||
db: &Database,
|
||||
prefix: &str,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
cached_starts: &Windows<&WindowStartVec>,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
output_count: Series::forced_import(
|
||||
db,
|
||||
&format!("{prefix}_output_count"),
|
||||
version,
|
||||
indexes,
|
||||
cached_starts,
|
||||
)?,
|
||||
total: TotalMetrics::forced_import(db, prefix, version, indexes, cached_starts)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.output_count.block.len().min(self.total.len())
|
||||
}
|
||||
|
||||
pub(super) fn push(&mut self, block: Totals) {
|
||||
self.output_count.block.push(block.output_count.into());
|
||||
self.total.push(block);
|
||||
}
|
||||
|
||||
fn validate_and_truncate(&mut self, version: Version, height: Height) -> Result<()> {
|
||||
self.output_count
|
||||
.block
|
||||
.validate_and_truncate(version, height)?;
|
||||
self.total.validate_and_truncate(version, height)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> {
|
||||
self.output_count.block.truncate_if_needed_at(len)?;
|
||||
self.total.truncate_if_needed_at(len)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write(&mut self) -> Result<()> {
|
||||
self.output_count.block.write()?;
|
||||
self.total.write()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compute_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
|
||||
self.output_count.compute_rest(max_from, exit)?;
|
||||
self.total.compute_cumulative(max_from, exit)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deref, DerefMut, Traversable)]
|
||||
pub struct Breakdown<M: StorageMode = Rw> {
|
||||
#[deref]
|
||||
#[deref_mut]
|
||||
#[traversable(flatten)]
|
||||
pub metrics: Metrics<M>,
|
||||
pub data_share: PercentPerBlock<BasisPoints16, M>,
|
||||
pub chain_share: PercentPerBlock<BasisPoints16, M>,
|
||||
}
|
||||
|
||||
impl Breakdown {
|
||||
pub(crate) fn forced_import(
|
||||
db: &Database,
|
||||
prefix: &str,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
cached_starts: &Windows<&WindowStartVec>,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
metrics: Metrics::forced_import(db, prefix, version, indexes, cached_starts)?,
|
||||
data_share: PercentPerBlock::forced_import(
|
||||
db,
|
||||
&format!("{prefix}_data_share"),
|
||||
version,
|
||||
indexes,
|
||||
)?,
|
||||
chain_share: PercentPerBlock::forced_import(
|
||||
db,
|
||||
&format!("{prefix}_chain_share"),
|
||||
version,
|
||||
indexes,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Traversable)]
|
||||
pub struct Policy<M: StorageMode = Rw> {
|
||||
pub pre_v30_standard: Breakdown<M>,
|
||||
pub pre_v30_nonstandard: Breakdown<M>,
|
||||
pub oversized: Breakdown<M>,
|
||||
pub multiple: Breakdown<M>,
|
||||
}
|
||||
|
||||
impl Policy {
|
||||
pub(crate) fn forced_import(
|
||||
db: &Database,
|
||||
version: Version,
|
||||
indexes: &indexes::Vecs,
|
||||
cached_starts: &Windows<&WindowStartVec>,
|
||||
) -> Result<Self> {
|
||||
let import = |name| {
|
||||
Breakdown::forced_import(
|
||||
db,
|
||||
&format!("op_return_policy_{name}"),
|
||||
version,
|
||||
indexes,
|
||||
cached_starts,
|
||||
)
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
pre_v30_standard: import("pre_v30_standard")?,
|
||||
pre_v30_nonstandard: import("pre_v30_nonstandard")?,
|
||||
oversized: import("oversized")?,
|
||||
multiple: import("multiple")?,
|
||||
})
|
||||
}
|
||||
|
||||
fn iter(&self) -> impl Iterator<Item = &Breakdown> {
|
||||
[
|
||||
&self.pre_v30_standard,
|
||||
&self.pre_v30_nonstandard,
|
||||
&self.oversized,
|
||||
&self.multiple,
|
||||
]
|
||||
.into_iter()
|
||||
}
|
||||
|
||||
pub(super) fn iter_mut(&mut self) -> impl Iterator<Item = &mut Breakdown> {
|
||||
[
|
||||
&mut self.pre_v30_standard,
|
||||
&mut self.pre_v30_nonstandard,
|
||||
&mut self.oversized,
|
||||
&mut self.multiple,
|
||||
]
|
||||
.into_iter()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Traversable)]
|
||||
pub struct Vecs<M: StorageMode = Rw> {
|
||||
#[traversable(skip)]
|
||||
pub(crate) db: Database,
|
||||
pub total: Total<M>,
|
||||
pub by_kind: ByKind<Breakdown<M>>,
|
||||
pub policy: Policy<M>,
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn min_len(&self) -> usize {
|
||||
let len = self
|
||||
.by_kind
|
||||
.iter()
|
||||
.map(|metrics| metrics.len())
|
||||
.fold(self.total.len(), usize::min);
|
||||
self.policy
|
||||
.iter()
|
||||
.map(|metrics| metrics.len())
|
||||
.fold(len, usize::min)
|
||||
}
|
||||
|
||||
pub(crate) fn validate_and_truncate(&mut self, version: Version, height: Height) -> Result<()> {
|
||||
self.total.validate_and_truncate(version, height)?;
|
||||
for metrics in self.by_kind.iter_mut() {
|
||||
metrics.validate_and_truncate(version, height)?;
|
||||
}
|
||||
for metrics in self.policy.iter_mut() {
|
||||
metrics.validate_and_truncate(version, height)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn truncate_if_needed_at(&mut self, len: usize) -> Result<()> {
|
||||
self.total.truncate_if_needed_at(len)?;
|
||||
for metrics in self.by_kind.iter_mut() {
|
||||
metrics.truncate_if_needed_at(len)?;
|
||||
}
|
||||
for metrics in self.policy.iter_mut() {
|
||||
metrics.truncate_if_needed_at(len)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn write(&mut self) -> Result<()> {
|
||||
self.total.write()?;
|
||||
for metrics in self.by_kind.iter_mut() {
|
||||
metrics.write()?;
|
||||
}
|
||||
for metrics in self.policy.iter_mut() {
|
||||
metrics.write()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn compute_cumulative(&mut self, max_from: Height, exit: &Exit) -> Result<()> {
|
||||
self.total.compute_cumulative(max_from, exit)?;
|
||||
for metrics in self.by_kind.iter_mut() {
|
||||
metrics.compute_cumulative(max_from, exit)?;
|
||||
}
|
||||
for metrics in self.policy.iter_mut() {
|
||||
metrics.compute_cumulative(max_from, exit)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,8 @@ use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
|
||||
use super::{Vecs, WithOutputTypes};
|
||||
use crate::internal::{CoinbasePolicy, PerBlockCumulativeRolling, walk_blocks};
|
||||
|
||||
const WRITE_INTERVAL: usize = 10_000;
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
|
||||
let starting_lengths = indexer.safe_lengths();
|
||||
@@ -44,6 +46,7 @@ impl Vecs {
|
||||
|
||||
let mut otype_cursor = indexer.vecs.outputs.output_type.cursor();
|
||||
let mut fo_cursor = indexer.vecs.transactions.first_txout_index.cursor();
|
||||
let mut height = skip;
|
||||
|
||||
walk_blocks(
|
||||
&fi_batch,
|
||||
@@ -77,7 +80,8 @@ impl Vecs {
|
||||
.block
|
||||
.push(StoredU64::from(spendable_total));
|
||||
|
||||
if self.output_count.all.block.batch_limit_reached() {
|
||||
height += 1;
|
||||
if height.is_multiple_of(WRITE_INTERVAL) {
|
||||
let _lock = exit.lock();
|
||||
self.output_count.write()?;
|
||||
self.spendable_output_count.block.write()?;
|
||||
@@ -93,14 +97,14 @@ impl Vecs {
|
||||
self.spendable_output_count.block.write()?;
|
||||
self.tx_count.write()?;
|
||||
}
|
||||
|
||||
self.output_count
|
||||
.compute_rest(starting_lengths.height, exit)?;
|
||||
self.spendable_output_count
|
||||
.compute_rest(starting_lengths.height, exit)?;
|
||||
self.tx_count.compute_rest(starting_lengths.height, exit)?;
|
||||
}
|
||||
|
||||
self.output_count
|
||||
.compute_rest(starting_lengths.height, exit)?;
|
||||
self.spendable_output_count
|
||||
.compute_rest(starting_lengths.height, exit)?;
|
||||
self.tx_count.compute_rest(starting_lengths.height, exit)?;
|
||||
|
||||
for (otype, source) in self.output_count.by_type.iter_typed() {
|
||||
self.output_share.get_mut(otype).compute_count_ratio(
|
||||
source,
|
||||
|
||||
@@ -3,14 +3,13 @@ use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{blocks, indexes, inputs, price};
|
||||
use crate::{blocks, inputs, price};
|
||||
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
inputs: &inputs::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
prices: &price::Vecs,
|
||||
@@ -20,7 +19,7 @@ impl Vecs {
|
||||
|
||||
let starting_lengths = indexer.safe_lengths();
|
||||
|
||||
self.count.compute(indexer, indexes, blocks, exit)?;
|
||||
self.count.compute(indexer, blocks, exit)?;
|
||||
self.per_sec.compute(&self.count, &starting_lengths, exit)?;
|
||||
self.value.compute(indexer, prices, exit)?;
|
||||
self.by_type.compute(indexer, exit)?;
|
||||
|
||||
@@ -3,27 +3,26 @@ use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{blocks, indexes};
|
||||
use crate::blocks;
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
blocks: &blocks::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let starting_height = indexer.safe_lengths().height;
|
||||
let window_starts = blocks.lookback.window_starts();
|
||||
self.total.compute(
|
||||
|
||||
self.total.sum.compute_count_from_indexes(
|
||||
starting_height,
|
||||
&indexes.tx_index.output_count,
|
||||
&indexer.vecs.transactions.first_tx_index,
|
||||
&indexes.height.tx_index_count,
|
||||
&window_starts,
|
||||
&indexer.vecs.outputs.first_txout_index,
|
||||
&indexer.vecs.outputs.value,
|
||||
exit,
|
||||
0,
|
||||
)?;
|
||||
self.total
|
||||
.compute_rest(starting_height, &window_starts, exit)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::ops::Range;
|
||||
use brk_error::Result;
|
||||
use brk_indexer::{Indexer, Lengths};
|
||||
use brk_oracle::{
|
||||
bin_to_cents, cents_to_bin, Config, Oracle, PaymentFilter, START_HEIGHT_FAST, START_HEIGHT_SLOW,
|
||||
Config, Oracle, PaymentFilter, START_HEIGHT_FAST, START_HEIGHT_SLOW, bin_to_cents, cents_to_bin,
|
||||
};
|
||||
use brk_types::{Cents, OutputType, Sats, TxIndex, TxOutIndex};
|
||||
use tracing::info;
|
||||
|
||||
@@ -19,7 +19,7 @@ impl Vecs {
|
||||
self.db.sync_bg_tasks()?;
|
||||
|
||||
let (r1, (r2, r3)) = rayon::join(
|
||||
|| self.count.compute(indexer, &blocks.lookback, exit),
|
||||
|| self.count.compute(indexer, indexes, &blocks.lookback, exit),
|
||||
|| {
|
||||
rayon::join(
|
||||
|| self.versions.compute(indexer, exit),
|
||||
@@ -34,6 +34,11 @@ impl Vecs {
|
||||
self.fees
|
||||
.compute(indexer, indexes, &inputs.spent, &self.size, exit)?;
|
||||
|
||||
self.patterns
|
||||
.compute(indexer, indexes, &inputs.spent, exit)?;
|
||||
|
||||
self.policy.compute(indexer, indexes, &self.fees, exit)?;
|
||||
|
||||
self.volume
|
||||
.compute(indexer, indexes, prices, &self.count, &self.fees, exit)?;
|
||||
|
||||
|
||||
@@ -3,12 +3,13 @@ use brk_indexer::Indexer;
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
use crate::blocks;
|
||||
use crate::{blocks, indexes};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
lookback: &blocks::LookbackVecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
@@ -17,10 +18,10 @@ impl Vecs {
|
||||
let window_starts = lookback.window_starts();
|
||||
self.total
|
||||
.compute(starting_height, &window_starts, exit, |height| {
|
||||
Ok(height.compute_count_from_indexes(
|
||||
Ok(height.compute_transform(
|
||||
starting_height,
|
||||
&indexer.vecs.transactions.first_tx_index,
|
||||
&indexer.vecs.transactions.txid,
|
||||
&indexes.height.tx_index_count,
|
||||
|(height, count, ..)| (height, count),
|
||||
exit,
|
||||
)?)
|
||||
})?;
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
use brk_error::Result;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_types::{FeeRate, OutPoint, Sats, TxInIndex, TxIndex, VSize};
|
||||
use brk_types::{
|
||||
ChunkInput, CpfpClusterTxIndex, FeeRate, OutPoint, Sats, StoredBool, StoredU64, TxInIndex,
|
||||
TxIndex, VSize, linearize,
|
||||
};
|
||||
use smallvec::SmallVec;
|
||||
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec, unlikely};
|
||||
|
||||
use super::super::size;
|
||||
@@ -70,7 +74,11 @@ impl Vecs {
|
||||
|
||||
let dep_version = self.input_value.version()
|
||||
+ self.output_value.version()
|
||||
+ size_vecs.vsize.tx_index.version();
|
||||
+ size_vecs.vsize.tx_index.version()
|
||||
+ indexer.vecs.inputs.outpoint.version()
|
||||
+ indexer.vecs.transactions.first_tx_index.version()
|
||||
+ indexer.vecs.transactions.first_txin_index.version()
|
||||
+ indexes.height.tx_index_count.version();
|
||||
|
||||
self.fee
|
||||
.tx_index
|
||||
@@ -80,92 +88,128 @@ impl Vecs {
|
||||
self.effective_fee_rate
|
||||
.tx_index
|
||||
.validate_computed_version_or_reset(dep_version)?;
|
||||
self.is_cpfp_parent
|
||||
.validate_computed_version_or_reset(dep_version)?;
|
||||
self.is_cpfp_child
|
||||
.validate_computed_version_or_reset(dep_version)?;
|
||||
self.count
|
||||
.cpfp_parent
|
||||
.validate_computed_version_or_reset(dep_version)?;
|
||||
self.count
|
||||
.cpfp_child
|
||||
.validate_computed_version_or_reset(dep_version)?;
|
||||
|
||||
let target = self
|
||||
.input_value
|
||||
.len()
|
||||
.min(self.output_value.len())
|
||||
.min(size_vecs.vsize.tx_index.len());
|
||||
let min = self
|
||||
let tx_len = self
|
||||
.fee
|
||||
.tx_index
|
||||
.len()
|
||||
.min(self.fee_rate.len())
|
||||
.min(self.effective_fee_rate.tx_index.len())
|
||||
.min(self.is_cpfp_parent.len())
|
||||
.min(self.is_cpfp_child.len())
|
||||
.min(starting_lengths.tx_index.to_usize());
|
||||
|
||||
if min >= target {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.fee
|
||||
.tx_index
|
||||
.truncate_if_needed(starting_lengths.tx_index)?;
|
||||
self.fee_rate
|
||||
.truncate_if_needed(starting_lengths.tx_index)?;
|
||||
self.effective_fee_rate
|
||||
.tx_index
|
||||
.truncate_if_needed(starting_lengths.tx_index)?;
|
||||
|
||||
let start_tx = self.fee.tx_index.len();
|
||||
let max_height = indexer.vecs.transactions.first_tx_index.len();
|
||||
|
||||
let start_height = if start_tx == 0 {
|
||||
0
|
||||
let max_height = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_tx_index
|
||||
.len()
|
||||
.min(indexes.height.tx_index_count.len());
|
||||
let next_height = if tx_len >= target {
|
||||
max_height
|
||||
} else {
|
||||
indexes
|
||||
.tx_heights
|
||||
.get_shared(TxIndex::from(start_tx))
|
||||
.get_shared(TxIndex::from(tx_len))
|
||||
.unwrap()
|
||||
.to_usize()
|
||||
};
|
||||
let count_len = self
|
||||
.count
|
||||
.cpfp_parent
|
||||
.len()
|
||||
.min(self.count.cpfp_child.len())
|
||||
.min(max_height);
|
||||
let start_height = count_len.min(next_height);
|
||||
if start_height >= max_height {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let start_tx = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_tx_index
|
||||
.collect_one_at(start_height)
|
||||
.unwrap()
|
||||
.to_usize();
|
||||
self.fee
|
||||
.tx_index
|
||||
.truncate_if_needed(TxIndex::from(start_tx))?;
|
||||
self.fee_rate.truncate_if_needed(TxIndex::from(start_tx))?;
|
||||
self.effective_fee_rate
|
||||
.tx_index
|
||||
.truncate_if_needed(TxIndex::from(start_tx))?;
|
||||
self.is_cpfp_parent
|
||||
.truncate_if_needed(TxIndex::from(start_tx))?;
|
||||
self.is_cpfp_child
|
||||
.truncate_if_needed(TxIndex::from(start_tx))?;
|
||||
self.count.cpfp_parent.truncate_if_needed_at(start_height)?;
|
||||
self.count.cpfp_child.truncate_if_needed_at(start_height)?;
|
||||
|
||||
let mut tx_count = indexes.height.tx_index_count.cursor();
|
||||
let mut next_block_input = indexer.vecs.inputs.first_txin_index.cursor();
|
||||
tx_count.advance(start_height);
|
||||
next_block_input.advance(start_height + 1);
|
||||
|
||||
let mut input_values = Vec::new();
|
||||
let mut output_values = Vec::new();
|
||||
let mut vsizes = Vec::new();
|
||||
let mut txin_starts = Vec::new();
|
||||
let mut outpoints = Vec::new();
|
||||
let mut fees = Vec::new();
|
||||
let mut cluster = Cluster::default();
|
||||
let mut first_tx = start_tx;
|
||||
|
||||
for h in start_height..max_height {
|
||||
let first_tx: usize = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_tx_index
|
||||
.collect_one_at(h)
|
||||
.unwrap()
|
||||
.to_usize();
|
||||
let n = *indexes.height.tx_index_count.collect_one_at(h).unwrap() as usize;
|
||||
let n = u64::from(tx_count.next().unwrap()) as usize;
|
||||
|
||||
if first_tx + n > target {
|
||||
break;
|
||||
}
|
||||
|
||||
// Batch read all per-tx data for this block
|
||||
let input_values = self.input_value.collect_range_at(first_tx, first_tx + n);
|
||||
let output_values = self.output_value.collect_range_at(first_tx, first_tx + n);
|
||||
let vsizes: Vec<VSize> = size_vecs
|
||||
self.input_value
|
||||
.collect_range_into_at(first_tx, first_tx + n, &mut input_values);
|
||||
self.output_value
|
||||
.collect_range_into_at(first_tx, first_tx + n, &mut output_values);
|
||||
size_vecs
|
||||
.vsize
|
||||
.tx_index
|
||||
.collect_range_at(first_tx, first_tx + n);
|
||||
let txin_starts: Vec<TxInIndex> = indexer
|
||||
.collect_range_into_at(first_tx, first_tx + n, &mut vsizes);
|
||||
indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txin_index
|
||||
.collect_range_at(first_tx, first_tx + n);
|
||||
.collect_range_into_at(first_tx, first_tx + n, &mut txin_starts);
|
||||
let input_begin = txin_starts[0].to_usize();
|
||||
let input_end = if h + 1 < max_height {
|
||||
indexer
|
||||
.vecs
|
||||
.inputs
|
||||
.first_txin_index
|
||||
.collect_one_at(h + 1)
|
||||
.unwrap()
|
||||
.to_usize()
|
||||
next_block_input.next().unwrap().to_usize()
|
||||
} else {
|
||||
indexer.vecs.inputs.outpoint.len()
|
||||
};
|
||||
let outpoints: Vec<OutPoint> = indexer
|
||||
.vecs
|
||||
.inputs
|
||||
.outpoint
|
||||
.collect_range_at(input_begin, input_end);
|
||||
indexer.vecs.inputs.outpoint.collect_range_into_at(
|
||||
input_begin,
|
||||
input_end,
|
||||
&mut outpoints,
|
||||
);
|
||||
|
||||
// Compute fee + fee_rate per tx
|
||||
let mut fees = Vec::with_capacity(n);
|
||||
fees.clear();
|
||||
fees.reserve(n);
|
||||
for j in 0..n {
|
||||
let fee = if unlikely(input_values[j].is_max()) {
|
||||
Sats::ZERO
|
||||
@@ -178,36 +222,56 @@ impl Vecs {
|
||||
}
|
||||
|
||||
// Effective fee rate via same-block CPFP clustering
|
||||
let effective = cluster_fee_rates(
|
||||
cluster_fee_rates(
|
||||
&txin_starts,
|
||||
&outpoints,
|
||||
input_begin,
|
||||
first_tx,
|
||||
&fees,
|
||||
&vsizes,
|
||||
&mut cluster,
|
||||
);
|
||||
for rate in effective {
|
||||
self.effective_fee_rate.tx_index.push(rate);
|
||||
let mut parent_count = 0;
|
||||
let mut child_count = 0;
|
||||
for ((&effective, &fee), &vsize) in cluster.rates.iter().zip(&fees).zip(&vsizes) {
|
||||
let (is_parent, is_child) = cpfp_roles(effective, FeeRate::from((fee, vsize)));
|
||||
parent_count += is_parent as u64;
|
||||
child_count += is_child as u64;
|
||||
self.effective_fee_rate.tx_index.push(effective);
|
||||
self.is_cpfp_parent.push(StoredBool::from(is_parent));
|
||||
self.is_cpfp_child.push(StoredBool::from(is_child));
|
||||
}
|
||||
self.count.cpfp_parent.push(StoredU64::from(parent_count));
|
||||
self.count.cpfp_child.push(StoredU64::from(child_count));
|
||||
|
||||
if h % 1_000 == 0 {
|
||||
let _lock = exit.lock();
|
||||
self.fee.tx_index.write()?;
|
||||
self.fee_rate.write()?;
|
||||
self.effective_fee_rate.tx_index.write()?;
|
||||
self.is_cpfp_parent.write()?;
|
||||
self.is_cpfp_child.write()?;
|
||||
self.count.cpfp_parent.write()?;
|
||||
self.count.cpfp_child.write()?;
|
||||
}
|
||||
|
||||
first_tx += n;
|
||||
}
|
||||
|
||||
let _lock = exit.lock();
|
||||
self.fee.tx_index.write()?;
|
||||
self.fee_rate.write()?;
|
||||
self.effective_fee_rate.tx_index.write()?;
|
||||
self.is_cpfp_parent.write()?;
|
||||
self.is_cpfp_child.write()?;
|
||||
self.count.cpfp_parent.write()?;
|
||||
self.count.cpfp_child.write()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Clusters same-block parent-child txs and computes effective fee rate per cluster.
|
||||
/// Computes SFL chunk rates for each same-block dependency component.
|
||||
fn cluster_fee_rates(
|
||||
txin_starts: &[TxInIndex],
|
||||
outpoints: &[OutPoint],
|
||||
@@ -215,57 +279,254 @@ fn cluster_fee_rates(
|
||||
first_tx: usize,
|
||||
fees: &[Sats],
|
||||
vsizes: &[VSize],
|
||||
) -> Vec<FeeRate> {
|
||||
cluster: &mut Cluster,
|
||||
) {
|
||||
let n = fees.len();
|
||||
let mut parent: Vec<usize> = (0..n).collect();
|
||||
cluster.rates.clear();
|
||||
cluster.rates.extend(
|
||||
fees.iter()
|
||||
.zip(vsizes)
|
||||
.map(|(&fee, &vsize)| FeeRate::from((fee, vsize))),
|
||||
);
|
||||
cluster.parents.clear();
|
||||
cluster.parents.resize_with(n, SmallVec::new);
|
||||
cluster.roots.clear();
|
||||
cluster.roots.extend(0..n);
|
||||
cluster.members.clear();
|
||||
cluster.local_index.clear();
|
||||
cluster.local_index.resize(n, usize::MAX);
|
||||
|
||||
for j in 1..n {
|
||||
let start = txin_starts[j].to_usize() - outpoint_base;
|
||||
let end = if j + 1 < txin_starts.len() {
|
||||
txin_starts[j + 1].to_usize() - outpoint_base
|
||||
} else {
|
||||
outpoints.len()
|
||||
};
|
||||
for child in 0..n {
|
||||
let mut parents: SmallVec<[usize; 2]> =
|
||||
same_block_parents(child, txin_starts, outpoints, outpoint_base, first_tx, n).collect();
|
||||
parents.sort_unstable();
|
||||
parents.dedup();
|
||||
for &parent in &parents {
|
||||
union(&mut cluster.roots, child, parent);
|
||||
}
|
||||
cluster.parents[child] = parents;
|
||||
}
|
||||
|
||||
for op in &outpoints[start..end] {
|
||||
if op.is_coinbase() {
|
||||
continue;
|
||||
}
|
||||
let parent_tx = op.tx_index().to_usize();
|
||||
if parent_tx >= first_tx && parent_tx < first_tx + n {
|
||||
union(&mut parent, j, parent_tx - first_tx);
|
||||
}
|
||||
for tx in 0..n {
|
||||
cluster.members.push((root(&mut cluster.roots, tx), tx));
|
||||
}
|
||||
cluster.members.sort_unstable();
|
||||
|
||||
let mut start = 0;
|
||||
while start < n {
|
||||
let component_root = cluster.members[start].0;
|
||||
let end = cluster.members[start..]
|
||||
.partition_point(|&(candidate, _)| candidate == component_root)
|
||||
+ start;
|
||||
if end - start > 1 {
|
||||
linearize_component(
|
||||
&cluster.members[start..end],
|
||||
&cluster.parents,
|
||||
fees,
|
||||
vsizes,
|
||||
&mut cluster.rates,
|
||||
&mut cluster.local_index,
|
||||
&mut cluster.local_parents,
|
||||
);
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn linearize_component(
|
||||
members: &[(usize, usize)],
|
||||
parents: &[SmallVec<[usize; 2]>],
|
||||
fees: &[Sats],
|
||||
vsizes: &[VSize],
|
||||
rates: &mut [FeeRate],
|
||||
local_index: &mut [usize],
|
||||
local_parents: &mut Vec<SmallVec<[CpfpClusterTxIndex; 2]>>,
|
||||
) {
|
||||
for (local, &(_, tx)) in members.iter().enumerate() {
|
||||
local_index[tx] = local;
|
||||
}
|
||||
|
||||
local_parents.clear();
|
||||
local_parents.extend(members.iter().map(|&(_, tx)| {
|
||||
parents[tx]
|
||||
.iter()
|
||||
.map(|&parent| CpfpClusterTxIndex::from(local_index[parent] as u32))
|
||||
.collect()
|
||||
}));
|
||||
|
||||
let inputs: Vec<ChunkInput<'_>> = members
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(local, &(_, tx))| ChunkInput {
|
||||
fee: fees[tx],
|
||||
vsize: vsizes[tx],
|
||||
parents: local_parents[local].as_slice(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
for chunk in linearize(&inputs) {
|
||||
for local in chunk.txs {
|
||||
rates[members[u32::from(local) as usize].1] = chunk.feerate;
|
||||
}
|
||||
}
|
||||
|
||||
let mut cluster_fee = vec![Sats::ZERO; n];
|
||||
let mut cluster_vsize = vec![VSize::from(0u64); n];
|
||||
for j in 0..n {
|
||||
let root = find(&mut parent, j);
|
||||
cluster_fee[root] += fees[j];
|
||||
cluster_vsize[root] += vsizes[j];
|
||||
}
|
||||
|
||||
(0..n)
|
||||
.map(|j| {
|
||||
let root = find(&mut parent, j);
|
||||
FeeRate::from((cluster_fee[root], cluster_vsize[root]))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn find(parent: &mut [usize], mut i: usize) -> usize {
|
||||
while parent[i] != i {
|
||||
parent[i] = parent[parent[i]];
|
||||
i = parent[i];
|
||||
fn union(roots: &mut [usize], left: usize, right: usize) {
|
||||
let left = root(roots, left);
|
||||
let right = root(roots, right);
|
||||
if left != right {
|
||||
roots[right] = left;
|
||||
}
|
||||
i
|
||||
}
|
||||
|
||||
fn union(parent: &mut [usize], a: usize, b: usize) {
|
||||
let ra = find(parent, a);
|
||||
let rb = find(parent, b);
|
||||
if ra != rb {
|
||||
parent[ra] = rb;
|
||||
fn root(roots: &mut [usize], node: usize) -> usize {
|
||||
let mut root = node;
|
||||
while roots[root] != root {
|
||||
root = roots[root];
|
||||
}
|
||||
|
||||
let mut current = node;
|
||||
while roots[current] != current {
|
||||
let next = roots[current];
|
||||
roots[current] = root;
|
||||
current = next;
|
||||
}
|
||||
root
|
||||
}
|
||||
|
||||
fn same_block_parents<'a>(
|
||||
tx: usize,
|
||||
txin_starts: &'a [TxInIndex],
|
||||
outpoints: &'a [OutPoint],
|
||||
outpoint_base: usize,
|
||||
first_tx: usize,
|
||||
tx_count: usize,
|
||||
) -> impl Iterator<Item = usize> + 'a {
|
||||
let start = txin_starts[tx].to_usize() - outpoint_base;
|
||||
let end = txin_starts
|
||||
.get(tx + 1)
|
||||
.map_or(outpoints.len(), |index| index.to_usize() - outpoint_base);
|
||||
|
||||
outpoints[start..end].iter().filter_map(move |outpoint| {
|
||||
let parent = outpoint.tx_index().to_usize();
|
||||
(parent >= first_tx && parent < first_tx + tx_count).then(|| parent - first_tx)
|
||||
})
|
||||
}
|
||||
|
||||
fn cpfp_roles(effective: FeeRate, raw: FeeRate) -> (bool, bool) {
|
||||
(effective > raw, effective < raw)
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Cluster {
|
||||
rates: Vec<FeeRate>,
|
||||
parents: Vec<SmallVec<[usize; 2]>>,
|
||||
roots: Vec<usize>,
|
||||
members: Vec<(usize, usize)>,
|
||||
local_index: Vec<usize>,
|
||||
local_parents: Vec<SmallVec<[CpfpClusterTxIndex; 2]>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use brk_types::{FeeRate, OutPoint, Sats, TxInIndex, TxIndex, VSize, Vout};
|
||||
|
||||
use super::{Cluster, cluster_fee_rates, cpfp_roles};
|
||||
|
||||
#[test]
|
||||
fn marks_actual_cpfp_roles() {
|
||||
let mut cluster = Cluster::default();
|
||||
cluster_fee_rates(
|
||||
&[TxInIndex::from(0usize), TxInIndex::from(1usize)],
|
||||
&[
|
||||
OutPoint::COINBASE,
|
||||
OutPoint::new(TxIndex::from(10usize), Vout::ZERO),
|
||||
],
|
||||
0,
|
||||
10,
|
||||
&[Sats::new(100), Sats::new(200)],
|
||||
&[VSize::new(100), VSize::new(100)],
|
||||
&mut cluster,
|
||||
);
|
||||
|
||||
assert_eq!(cluster.rates, [FeeRate::new(1.5), FeeRate::new(1.5)]);
|
||||
assert_eq!(
|
||||
[
|
||||
cpfp_roles(cluster.rates[0], FeeRate::new(1.0)),
|
||||
cpfp_roles(cluster.rates[1], FeeRate::new(2.0)),
|
||||
],
|
||||
[(true, false), (false, true)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_independent_transaction_rates_separate() {
|
||||
let mut cluster = Cluster::default();
|
||||
cluster_fee_rates(
|
||||
&[TxInIndex::from(0usize), TxInIndex::from(1usize)],
|
||||
&[
|
||||
OutPoint::COINBASE,
|
||||
OutPoint::new(TxIndex::from(9usize), Vout::ZERO),
|
||||
],
|
||||
0,
|
||||
10,
|
||||
&[Sats::new(100), Sats::new(300)],
|
||||
&[VSize::new(100), VSize::new(100)],
|
||||
&mut cluster,
|
||||
);
|
||||
|
||||
assert_eq!(cluster.rates, [FeeRate::new(1.0), FeeRate::new(3.0)]);
|
||||
assert_eq!(
|
||||
[
|
||||
cpfp_roles(cluster.rates[0], FeeRate::new(1.0)),
|
||||
cpfp_roles(cluster.rates[1], FeeRate::new(3.0)),
|
||||
],
|
||||
[(false, false), (false, false)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn linearizes_shared_parent_branches_independently_of_sibling_order() {
|
||||
let txin_starts = [
|
||||
TxInIndex::from(0usize),
|
||||
TxInIndex::from(1usize),
|
||||
TxInIndex::from(2usize),
|
||||
];
|
||||
let outpoints = [
|
||||
OutPoint::COINBASE,
|
||||
OutPoint::new(TxIndex::from(10usize), Vout::ZERO),
|
||||
OutPoint::new(TxIndex::from(10usize), Vout::ZERO),
|
||||
];
|
||||
let vsizes = [VSize::new(100); 3];
|
||||
let mut cluster = Cluster::default();
|
||||
|
||||
cluster_fee_rates(
|
||||
&txin_starts,
|
||||
&outpoints,
|
||||
0,
|
||||
10,
|
||||
&[Sats::ZERO, Sats::ZERO, Sats::new(3_000)],
|
||||
&vsizes,
|
||||
&mut cluster,
|
||||
);
|
||||
assert_eq!(
|
||||
cluster.rates,
|
||||
[FeeRate::new(15.0), FeeRate::new(0.0), FeeRate::new(15.0)]
|
||||
);
|
||||
|
||||
cluster_fee_rates(
|
||||
&txin_starts,
|
||||
&outpoints,
|
||||
0,
|
||||
10,
|
||||
&[Sats::ZERO, Sats::new(3_000), Sats::ZERO],
|
||||
&vsizes,
|
||||
&mut cluster,
|
||||
);
|
||||
assert_eq!(
|
||||
cluster.rates,
|
||||
[FeeRate::new(15.0), FeeRate::new(15.0), FeeRate::new(0.0)]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ use brk_error::Result;
|
||||
use brk_types::Version;
|
||||
use vecdb::{Database, EagerVec, ImportableVec};
|
||||
|
||||
use super::Vecs;
|
||||
use super::{CountVecs, Vecs};
|
||||
use crate::{indexes, internal::PerTxDistribution};
|
||||
|
||||
/// Bump this when fee/feerate aggregation logic changes (e.g., skip coinbase, skip zero-fee).
|
||||
@@ -16,6 +16,10 @@ impl Vecs {
|
||||
) -> Result<Self> {
|
||||
let v = version + VERSION;
|
||||
Ok(Self {
|
||||
count: CountVecs {
|
||||
cpfp_parent: EagerVec::forced_import(db, "cpfp_parent_count", version)?,
|
||||
cpfp_child: EagerVec::forced_import(db, "cpfp_child_count", version)?,
|
||||
},
|
||||
input_value: EagerVec::forced_import(db, "input_value", version)?,
|
||||
output_value: EagerVec::forced_import(db, "output_value", version)?,
|
||||
fee: PerTxDistribution::forced_import(db, "fee", v, indexes)?,
|
||||
@@ -26,6 +30,8 @@ impl Vecs {
|
||||
v,
|
||||
indexes,
|
||||
)?,
|
||||
is_cpfp_parent: EagerVec::forced_import(db, "is_cpfp_parent", version)?,
|
||||
is_cpfp_child: EagerVec::forced_import(db, "is_cpfp_child", version)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,4 +2,4 @@ mod compute;
|
||||
mod import;
|
||||
mod vecs;
|
||||
|
||||
pub use vecs::Vecs;
|
||||
pub use vecs::{CountVecs, Vecs};
|
||||
|
||||
@@ -1,14 +1,23 @@
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{FeeRate, Sats, TxIndex};
|
||||
use brk_types::{FeeRate, Height, Sats, StoredBool, StoredU64, TxIndex};
|
||||
use vecdb::{EagerVec, PcoVec, Rw, StorageMode};
|
||||
|
||||
use crate::internal::PerTxDistribution;
|
||||
|
||||
#[derive(Traversable)]
|
||||
pub struct CountVecs<M: StorageMode = Rw> {
|
||||
pub cpfp_parent: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
|
||||
pub cpfp_child: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
|
||||
}
|
||||
|
||||
#[derive(Traversable)]
|
||||
pub struct Vecs<M: StorageMode = Rw> {
|
||||
pub count: CountVecs<M>,
|
||||
pub input_value: M::Stored<EagerVec<PcoVec<TxIndex, Sats>>>,
|
||||
pub output_value: M::Stored<EagerVec<PcoVec<TxIndex, Sats>>>,
|
||||
pub fee: PerTxDistribution<Sats, M>,
|
||||
pub fee_rate: M::Stored<EagerVec<PcoVec<TxIndex, FeeRate>>>,
|
||||
pub effective_fee_rate: PerTxDistribution<FeeRate, M>,
|
||||
pub is_cpfp_parent: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
|
||||
pub is_cpfp_child: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
|
||||
}
|
||||
|
||||
@@ -12,7 +12,9 @@ use crate::{
|
||||
},
|
||||
};
|
||||
|
||||
use super::{CountVecs, FeesVecs, SizeVecs, Vecs, VersionsVecs, VolumeVecs};
|
||||
use super::{
|
||||
CountVecs, FeesVecs, PatternsVecs, PolicyVecs, SizeVecs, Vecs, VersionsVecs, VolumeVecs,
|
||||
};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn forced_import(
|
||||
@@ -28,6 +30,8 @@ impl Vecs {
|
||||
let count = CountVecs::forced_import(&db, version, indexes, cached_starts)?;
|
||||
let size = SizeVecs::forced_import(&db, version, indexer, indexes)?;
|
||||
let fees = FeesVecs::forced_import(&db, version, indexes)?;
|
||||
let patterns = PatternsVecs::forced_import(&db, version)?;
|
||||
let policy = PolicyVecs::forced_import(&db, version)?;
|
||||
let versions = VersionsVecs::forced_import(&db, version, indexes, cached_starts)?;
|
||||
let volume = VolumeVecs::forced_import(&db, version, indexes, cached_starts)?;
|
||||
|
||||
@@ -36,6 +40,8 @@ impl Vecs {
|
||||
count,
|
||||
size,
|
||||
fees,
|
||||
patterns,
|
||||
policy,
|
||||
versions,
|
||||
volume,
|
||||
};
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
pub mod count;
|
||||
pub mod fees;
|
||||
pub mod patterns;
|
||||
pub mod policy;
|
||||
pub mod size;
|
||||
pub mod versions;
|
||||
pub mod volume;
|
||||
@@ -12,6 +14,8 @@ use vecdb::{Database, Rw, StorageMode};
|
||||
|
||||
pub use count::Vecs as CountVecs;
|
||||
pub use fees::Vecs as FeesVecs;
|
||||
pub use patterns::Vecs as PatternsVecs;
|
||||
pub use policy::Vecs as PolicyVecs;
|
||||
pub use size::Vecs as SizeVecs;
|
||||
pub use versions::Vecs as VersionsVecs;
|
||||
pub use volume::Vecs as VolumeVecs;
|
||||
@@ -26,6 +30,8 @@ pub struct Vecs<M: StorageMode = Rw> {
|
||||
pub count: CountVecs<M>,
|
||||
pub size: SizeVecs<M>,
|
||||
pub fees: FeesVecs<M>,
|
||||
pub patterns: PatternsVecs<M>,
|
||||
pub policy: PolicyVecs<M>,
|
||||
pub versions: VersionsVecs<M>,
|
||||
pub volume: VolumeVecs<M>,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
use brk_types::{OutputType, Sats, TypeIndex};
|
||||
use rustc_hash::FxHashSet;
|
||||
|
||||
type Address = (OutputType, TypeIndex);
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct Candidate {
|
||||
values: FxHashSet<Sats>,
|
||||
input_addresses: FxHashSet<Address>,
|
||||
output_addresses: FxHashSet<Address>,
|
||||
zero_values: usize,
|
||||
address_reuse: bool,
|
||||
}
|
||||
|
||||
impl Candidate {
|
||||
pub(super) fn clear(&mut self) {
|
||||
self.values.clear();
|
||||
self.input_addresses.clear();
|
||||
self.output_addresses.clear();
|
||||
self.zero_values = 0;
|
||||
self.address_reuse = false;
|
||||
}
|
||||
|
||||
pub(super) fn add_input(
|
||||
&mut self,
|
||||
value: Sats,
|
||||
output_type: OutputType,
|
||||
type_index: TypeIndex,
|
||||
) {
|
||||
self.add_value(value);
|
||||
if has_script_address(output_type) {
|
||||
self.address_reuse |= !self.input_addresses.insert((output_type, type_index));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn add_output(
|
||||
&mut self,
|
||||
value: Sats,
|
||||
output_type: OutputType,
|
||||
type_index: TypeIndex,
|
||||
) {
|
||||
self.add_value(value);
|
||||
if has_script_address(output_type) {
|
||||
let address = (output_type, type_index);
|
||||
self.address_reuse |=
|
||||
self.input_addresses.contains(&address) || !self.output_addresses.insert(address);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_match(&self, input_count: usize, output_count: usize) -> bool {
|
||||
!self.address_reuse
|
||||
&& self.values.len() + self.zero_values <= (input_count + output_count) / 2
|
||||
}
|
||||
|
||||
fn add_value(&mut self, value: Sats) {
|
||||
if value.is_zero() {
|
||||
self.zero_values += 1;
|
||||
} else {
|
||||
self.values.insert(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn has_script_address(output_type: OutputType) -> bool {
|
||||
matches!(
|
||||
output_type,
|
||||
OutputType::P2PK65
|
||||
| OutputType::P2PK33
|
||||
| OutputType::P2PKH
|
||||
| OutputType::P2SH
|
||||
| OutputType::P2WPKH
|
||||
| OutputType::P2WSH
|
||||
| OutputType::P2TR
|
||||
| OutputType::P2A
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use brk_types::{OutputType, Sats, TypeIndex};
|
||||
|
||||
use super::Candidate;
|
||||
|
||||
#[test]
|
||||
fn repeated_values_match_without_address_reuse() {
|
||||
let mut candidate = Candidate::default();
|
||||
for index in 0usize..5 {
|
||||
candidate.add_input(
|
||||
Sats::new(10_000),
|
||||
OutputType::P2WPKH,
|
||||
TypeIndex::from(index),
|
||||
);
|
||||
candidate.add_output(
|
||||
Sats::new(9_000),
|
||||
OutputType::P2WPKH,
|
||||
TypeIndex::from(index + 10),
|
||||
);
|
||||
}
|
||||
assert!(candidate.is_match(5, 5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reused_output_address_does_not_match() {
|
||||
let mut candidate = Candidate::default();
|
||||
for _ in 0..5 {
|
||||
candidate.add_output(
|
||||
Sats::new(9_000),
|
||||
OutputType::P2WPKH,
|
||||
TypeIndex::from(1usize),
|
||||
);
|
||||
}
|
||||
assert!(!candidate.is_match(5, 5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reused_input_address_does_not_match() {
|
||||
let mut candidate = Candidate::default();
|
||||
for _ in 0..5 {
|
||||
candidate.add_input(
|
||||
Sats::new(10_000),
|
||||
OutputType::P2PK33,
|
||||
TypeIndex::from(1usize),
|
||||
);
|
||||
}
|
||||
assert!(!candidate.is_match(5, 5));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
use brk_error::Result;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_types::{StoredBool, StoredU64, TxIndex};
|
||||
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
|
||||
|
||||
use super::{Vecs, coinjoin::Candidate};
|
||||
use crate::{indexes, inputs};
|
||||
|
||||
const WRITE_INTERVAL: usize = 10_000;
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
spent: &inputs::SpentVecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let features = &indexer.vecs.transaction_features;
|
||||
let version = indexes.tx_index.input_count.version()
|
||||
+ indexes.tx_index.output_count.version()
|
||||
+ indexer.vecs.transactions.first_tx_index.version()
|
||||
+ indexer.vecs.transactions.first_txin_index.version()
|
||||
+ indexer.vecs.transactions.first_txout_index.version()
|
||||
+ spent.value.version()
|
||||
+ indexer.vecs.inputs.output_type.version()
|
||||
+ indexer.vecs.inputs.type_index.version()
|
||||
+ indexer.vecs.outputs.value.version()
|
||||
+ indexer.vecs.outputs.output_type.version()
|
||||
+ indexer.vecs.outputs.type_index.version()
|
||||
+ features.has_op_return.version()
|
||||
+ features.has_inscription.version()
|
||||
+ indexes.height.tx_index_count.version();
|
||||
|
||||
for vec in [
|
||||
&mut self.is_coinjoin,
|
||||
&mut self.is_consolidation,
|
||||
&mut self.is_batch_payout,
|
||||
] {
|
||||
vec.validate_computed_version_or_reset(version)?;
|
||||
}
|
||||
for vec in [
|
||||
&mut self.count.coinjoin,
|
||||
&mut self.count.consolidation,
|
||||
&mut self.count.batch_payout,
|
||||
] {
|
||||
vec.validate_computed_version_or_reset(version)?;
|
||||
}
|
||||
|
||||
let starting_lengths = indexer.safe_lengths();
|
||||
let target_tx = indexes.tx_index.input_count.len();
|
||||
let target_height = indexes.height.tx_index_count.len();
|
||||
let tx_len = self
|
||||
.is_coinjoin
|
||||
.len()
|
||||
.min(self.is_consolidation.len())
|
||||
.min(self.is_batch_payout.len())
|
||||
.min(starting_lengths.tx_index.to_usize());
|
||||
let count_len = self
|
||||
.count
|
||||
.coinjoin
|
||||
.len()
|
||||
.min(self.count.consolidation.len())
|
||||
.min(self.count.batch_payout.len())
|
||||
.min(starting_lengths.height.to_usize());
|
||||
let start_height = count_len.min(next_height(indexes, tx_len, target_tx, target_height));
|
||||
if start_height >= target_height {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let first_tx = &indexer.vecs.transactions.first_tx_index;
|
||||
let start_tx = first_tx.collect_one_at(start_height).unwrap().to_usize();
|
||||
self.is_coinjoin.truncate_if_needed_at(start_tx)?;
|
||||
self.is_consolidation.truncate_if_needed_at(start_tx)?;
|
||||
self.is_batch_payout.truncate_if_needed_at(start_tx)?;
|
||||
self.count.coinjoin.truncate_if_needed_at(start_height)?;
|
||||
self.count
|
||||
.consolidation
|
||||
.truncate_if_needed_at(start_height)?;
|
||||
self.count
|
||||
.batch_payout
|
||||
.truncate_if_needed_at(start_height)?;
|
||||
|
||||
let first_txin = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txin_index
|
||||
.collect_one_at(start_tx)
|
||||
.unwrap()
|
||||
.to_usize();
|
||||
let first_txout = indexer
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txout_index
|
||||
.collect_one_at(start_tx)
|
||||
.unwrap()
|
||||
.to_usize();
|
||||
|
||||
let mut input_count = indexes.tx_index.input_count.cursor();
|
||||
let mut output_count = indexes.tx_index.output_count.cursor();
|
||||
let mut input_value = spent.value.cursor();
|
||||
let mut input_type = indexer.vecs.inputs.output_type.cursor();
|
||||
let mut input_type_index = indexer.vecs.inputs.type_index.cursor();
|
||||
let mut output_value = indexer.vecs.outputs.value.cursor();
|
||||
let mut output_type = indexer.vecs.outputs.output_type.cursor();
|
||||
let mut output_type_index = indexer.vecs.outputs.type_index.cursor();
|
||||
let mut has_op_return = features.has_op_return.cursor();
|
||||
let mut has_inscription = features.has_inscription.cursor();
|
||||
let mut tx_count = indexes.height.tx_index_count.cursor();
|
||||
|
||||
input_count.advance(start_tx);
|
||||
output_count.advance(start_tx);
|
||||
input_value.advance(first_txin);
|
||||
input_type.advance(first_txin);
|
||||
input_type_index.advance(first_txin);
|
||||
output_value.advance(first_txout);
|
||||
output_type.advance(first_txout);
|
||||
output_type_index.advance(first_txout);
|
||||
has_op_return.advance(start_tx);
|
||||
has_inscription.advance(start_tx);
|
||||
tx_count.advance(start_height);
|
||||
|
||||
let mut candidate = Candidate::default();
|
||||
let mut block_start = start_tx;
|
||||
for height in start_height..target_height {
|
||||
let block_end = block_start + u64::from(tx_count.next().unwrap()) as usize;
|
||||
let mut coinjoin_count = 0;
|
||||
let mut consolidation_count = 0;
|
||||
let mut batch_payout_count = 0;
|
||||
|
||||
for tx_index in block_start..block_end {
|
||||
let inputs = usize::from(input_count.next().unwrap());
|
||||
let outputs = usize::from(output_count.next().unwrap());
|
||||
let op_return = has_op_return.next().unwrap().is_true();
|
||||
let inscription = has_inscription.next().unwrap().is_true();
|
||||
let token_related = op_return || inscription;
|
||||
|
||||
let consolidation = is_consolidation(inputs, outputs);
|
||||
let batch_payout = is_batch_payout(inputs, outputs, tx_index == block_start);
|
||||
let coinjoin_candidate = is_coinjoin_candidate(inputs, outputs, token_related);
|
||||
|
||||
let coinjoin = if coinjoin_candidate {
|
||||
candidate.clear();
|
||||
for _ in 0..inputs {
|
||||
candidate.add_input(
|
||||
input_value.next().unwrap(),
|
||||
input_type.next().unwrap(),
|
||||
input_type_index.next().unwrap(),
|
||||
);
|
||||
}
|
||||
for _ in 0..outputs {
|
||||
candidate.add_output(
|
||||
output_value.next().unwrap(),
|
||||
output_type.next().unwrap(),
|
||||
output_type_index.next().unwrap(),
|
||||
);
|
||||
}
|
||||
candidate.is_match(inputs, outputs)
|
||||
} else {
|
||||
input_value.advance(inputs);
|
||||
input_type.advance(inputs);
|
||||
input_type_index.advance(inputs);
|
||||
output_value.advance(outputs);
|
||||
output_type.advance(outputs);
|
||||
output_type_index.advance(outputs);
|
||||
false
|
||||
};
|
||||
|
||||
coinjoin_count += coinjoin as u64;
|
||||
consolidation_count += consolidation as u64;
|
||||
batch_payout_count += batch_payout as u64;
|
||||
self.is_coinjoin.push(StoredBool::from(coinjoin));
|
||||
self.is_consolidation.push(StoredBool::from(consolidation));
|
||||
self.is_batch_payout.push(StoredBool::from(batch_payout));
|
||||
}
|
||||
|
||||
self.count.coinjoin.push(StoredU64::from(coinjoin_count));
|
||||
self.count
|
||||
.consolidation
|
||||
.push(StoredU64::from(consolidation_count));
|
||||
self.count
|
||||
.batch_payout
|
||||
.push(StoredU64::from(batch_payout_count));
|
||||
|
||||
if (height + 1).is_multiple_of(WRITE_INTERVAL) {
|
||||
let _lock = exit.lock();
|
||||
self.write()?;
|
||||
}
|
||||
|
||||
block_start = block_end;
|
||||
}
|
||||
|
||||
let _lock = exit.lock();
|
||||
self.write()
|
||||
}
|
||||
|
||||
fn write(&mut self) -> Result<()> {
|
||||
self.is_coinjoin.write()?;
|
||||
self.is_consolidation.write()?;
|
||||
self.is_batch_payout.write()?;
|
||||
self.count.coinjoin.write()?;
|
||||
self.count.consolidation.write()?;
|
||||
self.count.batch_payout.write()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn next_height(
|
||||
indexes: &indexes::Vecs,
|
||||
tx_len: usize,
|
||||
target_tx: usize,
|
||||
target_height: usize,
|
||||
) -> usize {
|
||||
if tx_len >= target_tx {
|
||||
target_height
|
||||
} else {
|
||||
indexes
|
||||
.tx_heights
|
||||
.get_shared(TxIndex::from(tx_len))
|
||||
.unwrap()
|
||||
.to_usize()
|
||||
}
|
||||
}
|
||||
|
||||
fn is_consolidation(inputs: usize, outputs: usize) -> bool {
|
||||
inputs >= outputs * 5
|
||||
}
|
||||
|
||||
fn is_batch_payout(inputs: usize, outputs: usize, is_coinbase: bool) -> bool {
|
||||
!is_coinbase && outputs >= inputs * 5
|
||||
}
|
||||
|
||||
fn is_coinjoin_candidate(inputs: usize, outputs: usize, token_related: bool) -> bool {
|
||||
inputs >= 5 && outputs >= 5 && inputs < outputs * 5 && outputs < inputs * 5 && !token_related
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{is_batch_payout, is_coinjoin_candidate, is_consolidation};
|
||||
|
||||
#[test]
|
||||
fn ratio_boundaries_match_filter_semantics() {
|
||||
assert!(is_consolidation(25, 5));
|
||||
assert!(is_batch_payout(5, 25, false));
|
||||
assert!(!is_batch_payout(1, 5, true));
|
||||
assert!(!is_coinjoin_candidate(25, 5, false));
|
||||
assert!(!is_coinjoin_candidate(5, 25, false));
|
||||
assert!(is_coinjoin_candidate(5, 5, false));
|
||||
assert!(!is_coinjoin_candidate(5, 5, true));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
use brk_error::Result;
|
||||
use brk_types::Version;
|
||||
use vecdb::{Database, EagerVec, ImportableVec};
|
||||
|
||||
use super::{CountVecs, Vecs};
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn forced_import(db: &Database, version: Version) -> Result<Self> {
|
||||
Ok(Self {
|
||||
count: CountVecs {
|
||||
coinjoin: EagerVec::forced_import(db, "coinjoin_count", version)?,
|
||||
consolidation: EagerVec::forced_import(db, "consolidation_count", version)?,
|
||||
batch_payout: EagerVec::forced_import(db, "batch_payout_count", version)?,
|
||||
},
|
||||
is_coinjoin: EagerVec::forced_import(db, "is_coinjoin", version)?,
|
||||
is_consolidation: EagerVec::forced_import(db, "is_consolidation", version)?,
|
||||
is_batch_payout: EagerVec::forced_import(db, "is_batch_payout", version)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
mod coinjoin;
|
||||
mod compute;
|
||||
mod import;
|
||||
mod vecs;
|
||||
|
||||
pub use vecs::{CountVecs, Vecs};
|
||||
@@ -0,0 +1,18 @@
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Height, StoredBool, StoredU64, TxIndex};
|
||||
use vecdb::{EagerVec, PcoVec, Rw, StorageMode};
|
||||
|
||||
#[derive(Traversable)]
|
||||
pub struct CountVecs<M: StorageMode = Rw> {
|
||||
pub coinjoin: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
|
||||
pub consolidation: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
|
||||
pub batch_payout: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
|
||||
}
|
||||
|
||||
#[derive(Traversable)]
|
||||
pub struct Vecs<M: StorageMode = Rw> {
|
||||
pub count: CountVecs<M>,
|
||||
pub is_coinjoin: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
|
||||
pub is_consolidation: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
|
||||
pub is_batch_payout: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
use brk_error::Result;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_types::{Sats, StoredBool, StoredU64, TxIndex};
|
||||
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{indexes, transactions::fees};
|
||||
|
||||
const FIRST_EPHEMERAL_DUST_HEIGHT: usize = 905_000;
|
||||
const WRITE_INTERVAL: usize = 10_000;
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fees: &fees::Vecs,
|
||||
exit: &Exit,
|
||||
) -> Result<()> {
|
||||
let features = &indexer.vecs.transaction_features;
|
||||
let version = features.is_unconditionally_nonstandard.version()
|
||||
+ features.has_dust_output.version()
|
||||
+ fees.fee.tx_index.version()
|
||||
+ indexer.vecs.transactions.first_tx_index.version()
|
||||
+ indexes.height.tx_index_count.version();
|
||||
self.is_nonstandard
|
||||
.validate_computed_version_or_reset(version)?;
|
||||
self.count.validate_computed_version_or_reset(version)?;
|
||||
|
||||
let starting_lengths = indexer.safe_lengths();
|
||||
let target_tx = fees.fee.tx_index.len();
|
||||
let target_height = indexes.height.tx_index_count.len();
|
||||
let tx_len = self
|
||||
.is_nonstandard
|
||||
.len()
|
||||
.min(starting_lengths.tx_index.to_usize());
|
||||
let count_len = self.count.len().min(starting_lengths.height.to_usize());
|
||||
let next_height = if tx_len >= target_tx {
|
||||
target_height
|
||||
} else {
|
||||
indexes
|
||||
.tx_heights
|
||||
.get_shared(TxIndex::from(tx_len))
|
||||
.unwrap()
|
||||
.to_usize()
|
||||
};
|
||||
let start_height = count_len.min(next_height);
|
||||
if start_height >= target_height {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let first_tx = &indexer.vecs.transactions.first_tx_index;
|
||||
let start_tx = first_tx.collect_one_at(start_height).unwrap().to_usize();
|
||||
self.is_nonstandard.truncate_if_needed_at(start_tx)?;
|
||||
self.count.truncate_if_needed_at(start_height)?;
|
||||
|
||||
let mut unconditional = features.is_unconditionally_nonstandard.cursor();
|
||||
let mut has_dust = features.has_dust_output.cursor();
|
||||
let mut fee = fees.fee.tx_index.cursor();
|
||||
let mut tx_count = indexes.height.tx_index_count.cursor();
|
||||
unconditional.advance(start_tx);
|
||||
has_dust.advance(start_tx);
|
||||
fee.advance(start_tx);
|
||||
tx_count.advance(start_height);
|
||||
|
||||
let mut block_start = start_tx;
|
||||
for height in start_height..target_height {
|
||||
let block_end =
|
||||
(block_start + u64::from(tx_count.next().unwrap()) as usize).min(target_tx);
|
||||
let mut count = 0;
|
||||
|
||||
for _ in block_start..block_end {
|
||||
let raw = unconditional.next().unwrap().is_true();
|
||||
let dust = has_dust.next().unwrap().is_true();
|
||||
let nonstandard = if raw {
|
||||
fee.advance(1);
|
||||
true
|
||||
} else if dust {
|
||||
dust_is_nonstandard(height, fee.next().unwrap())
|
||||
} else {
|
||||
fee.advance(1);
|
||||
false
|
||||
};
|
||||
count += nonstandard as u64;
|
||||
self.is_nonstandard.push(StoredBool::from(nonstandard));
|
||||
}
|
||||
self.count.push(StoredU64::from(count));
|
||||
|
||||
if (height + 1).is_multiple_of(WRITE_INTERVAL) {
|
||||
let _lock = exit.lock();
|
||||
self.is_nonstandard.write()?;
|
||||
self.count.write()?;
|
||||
}
|
||||
|
||||
block_start = block_end;
|
||||
}
|
||||
|
||||
let _lock = exit.lock();
|
||||
self.is_nonstandard.write()?;
|
||||
self.count.write()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn dust_is_nonstandard(height: usize, fee: Sats) -> bool {
|
||||
height < FIRST_EPHEMERAL_DUST_HEIGHT || fee != Sats::ZERO
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use brk_types::Sats;
|
||||
|
||||
use super::{FIRST_EPHEMERAL_DUST_HEIGHT, dust_is_nonstandard};
|
||||
|
||||
#[test]
|
||||
fn zero_fee_ephemeral_dust_starts_at_activation() {
|
||||
assert!(dust_is_nonstandard(
|
||||
FIRST_EPHEMERAL_DUST_HEIGHT - 1,
|
||||
Sats::ZERO
|
||||
));
|
||||
assert!(!dust_is_nonstandard(
|
||||
FIRST_EPHEMERAL_DUST_HEIGHT,
|
||||
Sats::ZERO
|
||||
));
|
||||
assert!(dust_is_nonstandard(
|
||||
FIRST_EPHEMERAL_DUST_HEIGHT,
|
||||
Sats::new(1)
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
use brk_error::Result;
|
||||
use brk_types::Version;
|
||||
use vecdb::{Database, EagerVec, ImportableVec};
|
||||
|
||||
use super::Vecs;
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn forced_import(db: &Database, version: Version) -> Result<Self> {
|
||||
Ok(Self {
|
||||
count: EagerVec::forced_import(db, "nonstandard_count", version)?,
|
||||
is_nonstandard: EagerVec::forced_import(db, "is_nonstandard", version)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
mod compute;
|
||||
mod import;
|
||||
mod vecs;
|
||||
|
||||
pub use vecs::Vecs;
|
||||
@@ -0,0 +1,9 @@
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{Height, StoredBool, StoredU64, TxIndex};
|
||||
use vecdb::{EagerVec, PcoVec, Rw, StorageMode};
|
||||
|
||||
#[derive(Traversable)]
|
||||
pub struct Vecs<M: StorageMode = Rw> {
|
||||
pub count: M::Stored<EagerVec<PcoVec<Height, StoredU64>>>,
|
||||
pub is_nonstandard: M::Stored<EagerVec<PcoVec<TxIndex, StoredBool>>>,
|
||||
}
|
||||
@@ -14,8 +14,13 @@ impl Vecs {
|
||||
) -> Result<()> {
|
||||
let starting_lengths = indexer.safe_lengths();
|
||||
|
||||
self.vsize
|
||||
.derive_from(indexer, indexes, &starting_lengths, exit)?;
|
||||
self.weight.derive_from(
|
||||
indexer,
|
||||
indexes,
|
||||
&starting_lengths,
|
||||
&indexer.vecs.transactions.weight,
|
||||
exit,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
use brk_error::Result;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_types::{TxIndex, VSize, Version, Weight};
|
||||
use vecdb::{Database, LazyVecFrom2, ReadableCloneableVec};
|
||||
use brk_types::Version;
|
||||
use vecdb::{Database, LazyVecFrom1, ReadableCloneableVec};
|
||||
|
||||
use super::Vecs;
|
||||
use crate::{
|
||||
indexes,
|
||||
internal::{LazyPerTxDistribution, LazyPerTxDistributionTransformed, VSizeToWeight},
|
||||
internal::{LazyPerTxDistributionTransformed, TxDerivedDistribution, WeightToVSize},
|
||||
};
|
||||
|
||||
impl Vecs {
|
||||
@@ -16,37 +16,19 @@ impl Vecs {
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
) -> Result<Self> {
|
||||
let tx_index_to_vsize = LazyVecFrom2::init(
|
||||
let weight = TxDerivedDistribution::forced_import(db, "tx_weight", version, indexes)?;
|
||||
|
||||
let tx_index_to_vsize = LazyVecFrom1::transformed::<WeightToVSize>(
|
||||
"tx_vsize",
|
||||
version,
|
||||
indexer.vecs.transactions.base_size.read_only_boxed_clone(),
|
||||
indexer.vecs.transactions.total_size.read_only_boxed_clone(),
|
||||
|_index: TxIndex, base_size, total_size| {
|
||||
VSize::from(Weight::from_sizes(*base_size, *total_size))
|
||||
},
|
||||
indexer.vecs.transactions.weight.read_only_boxed_clone(),
|
||||
);
|
||||
|
||||
let vsize = LazyPerTxDistribution::forced_import(
|
||||
db,
|
||||
let vsize = LazyPerTxDistributionTransformed::new::<WeightToVSize>(
|
||||
"tx_vsize",
|
||||
version,
|
||||
indexes,
|
||||
tx_index_to_vsize,
|
||||
)?;
|
||||
|
||||
let tx_index_to_weight = LazyVecFrom2::init(
|
||||
"tx_weight",
|
||||
version,
|
||||
indexer.vecs.transactions.base_size.read_only_boxed_clone(),
|
||||
indexer.vecs.transactions.total_size.read_only_boxed_clone(),
|
||||
|_index: TxIndex, base_size, total_size| Weight::from_sizes(*base_size, *total_size),
|
||||
);
|
||||
|
||||
let weight = LazyPerTxDistributionTransformed::new::<VSizeToWeight>(
|
||||
"tx_weight",
|
||||
version,
|
||||
tx_index_to_weight,
|
||||
&vsize.distribution,
|
||||
&weight,
|
||||
);
|
||||
|
||||
Ok(Self { vsize, weight })
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
use brk_traversable::Traversable;
|
||||
use brk_types::{StoredU32, VSize, Weight};
|
||||
use brk_types::{VSize, Weight};
|
||||
use vecdb::{Rw, StorageMode};
|
||||
|
||||
use crate::internal::{LazyPerTxDistribution, LazyPerTxDistributionTransformed};
|
||||
use crate::internal::{LazyPerTxDistributionTransformed, TxDerivedDistribution};
|
||||
|
||||
#[derive(Traversable)]
|
||||
pub struct Vecs<M: StorageMode = Rw> {
|
||||
pub vsize: LazyPerTxDistribution<VSize, StoredU32, StoredU32, M>,
|
||||
pub weight: LazyPerTxDistributionTransformed<Weight, StoredU32, StoredU32, VSize>,
|
||||
pub vsize: LazyPerTxDistributionTransformed<VSize, Weight, Weight>,
|
||||
pub weight: TxDerivedDistribution<Weight, M>,
|
||||
}
|
||||
|
||||
@@ -1,90 +1,31 @@
|
||||
use brk_error::Result;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_types::{StoredU64, TxVersion};
|
||||
use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecIndex, WritableVec};
|
||||
use vecdb::Exit;
|
||||
|
||||
use super::Vecs;
|
||||
|
||||
impl Vecs {
|
||||
pub(crate) fn compute(&mut self, indexer: &Indexer, exit: &Exit) -> Result<()> {
|
||||
let starting_height = indexer.safe_lengths().height;
|
||||
let lengths = indexer.safe_lengths();
|
||||
let starting_height = lengths.height;
|
||||
let counts = &indexer.vecs.transaction_features.count;
|
||||
|
||||
let dep_version = indexer.vecs.transactions.tx_version.version()
|
||||
+ indexer.vecs.transactions.first_tx_index.version()
|
||||
+ indexer.vecs.transactions.txid.version();
|
||||
|
||||
for vec in [&mut self.v1.block, &mut self.v2.block, &mut self.v3.block] {
|
||||
vec.validate_and_truncate(dep_version, starting_height)?;
|
||||
for (metrics, source) in [
|
||||
(&mut self.v1, &counts.v1),
|
||||
(&mut self.v2, &counts.v2),
|
||||
(&mut self.v3, &counts.v3),
|
||||
(&mut self.other, &counts.other_version),
|
||||
] {
|
||||
metrics.compute(starting_height, exit, |block| {
|
||||
Ok(block.compute_transform(
|
||||
starting_height,
|
||||
source,
|
||||
|(height, count, ..)| (height, count),
|
||||
exit,
|
||||
)?)
|
||||
})?;
|
||||
}
|
||||
|
||||
let skip = self
|
||||
.v1
|
||||
.block
|
||||
.len()
|
||||
.min(self.v2.block.len())
|
||||
.min(self.v3.block.len());
|
||||
|
||||
let first_tx_index = &indexer.vecs.transactions.first_tx_index;
|
||||
let end = first_tx_index.len();
|
||||
if skip >= end {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Truncate all 3 to skip, then push (no per-element bounds checks).
|
||||
self.v1.block.truncate_if_needed_at(skip)?;
|
||||
self.v2.block.truncate_if_needed_at(skip)?;
|
||||
self.v3.block.truncate_if_needed_at(skip)?;
|
||||
|
||||
// Single cursor over tx_version — scanned once for all 3 version counts.
|
||||
let mut cursor = indexer.vecs.transactions.tx_version.cursor();
|
||||
let fi_batch = first_tx_index.collect_range_at(skip, end);
|
||||
let txid_len = indexer.vecs.transactions.txid.len();
|
||||
|
||||
for (j, first_index) in fi_batch.iter().enumerate() {
|
||||
let next_first = fi_batch
|
||||
.get(j + 1)
|
||||
.map(|fi| fi.to_usize())
|
||||
.unwrap_or(txid_len);
|
||||
|
||||
let mut c1: usize = 0;
|
||||
let mut c2: usize = 0;
|
||||
let mut c3: usize = 0;
|
||||
|
||||
let fi = first_index.to_usize();
|
||||
cursor.advance(fi - cursor.position());
|
||||
for _ in fi..next_first {
|
||||
match cursor.next().unwrap() {
|
||||
TxVersion::ONE => c1 += 1,
|
||||
TxVersion::TWO => c2 += 1,
|
||||
TxVersion::THREE => c3 += 1,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
self.v1.block.push(StoredU64::from(c1 as u64));
|
||||
self.v2.block.push(StoredU64::from(c2 as u64));
|
||||
self.v3.block.push(StoredU64::from(c3 as u64));
|
||||
|
||||
if self.v1.block.batch_limit_reached() {
|
||||
let _lock = exit.lock();
|
||||
self.v1.block.write()?;
|
||||
self.v2.block.write()?;
|
||||
self.v3.block.write()?;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let _lock = exit.lock();
|
||||
self.v1.block.write()?;
|
||||
self.v2.block.write()?;
|
||||
self.v3.block.write()?;
|
||||
}
|
||||
|
||||
// Derive cumulative + sums from base
|
||||
self.v1.compute_rest(starting_height, exit)?;
|
||||
self.v2.compute_rest(starting_height, exit)?;
|
||||
self.v3.compute_rest(starting_height, exit)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,13 @@ impl Vecs {
|
||||
indexes,
|
||||
cached_starts,
|
||||
)?,
|
||||
other: PerBlockCumulativeRolling::forced_import(
|
||||
db,
|
||||
"tx_other_version",
|
||||
version,
|
||||
indexes,
|
||||
cached_starts,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,4 +9,5 @@ pub struct Vecs<M: StorageMode = Rw> {
|
||||
pub v1: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
|
||||
pub v2: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
|
||||
pub v3: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
|
||||
pub other: PerBlockCumulativeRolling<StoredU64, StoredU64, M>,
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use brk_types::{TxIndex, Txid, TxidPrefix, Version};
|
||||
|
||||
// One version for all data sources
|
||||
// Increment on **change _OR_ addition**
|
||||
pub const VERSION: Version = Version::new(26);
|
||||
pub const VERSION: Version = Version::new(27);
|
||||
pub const SNAPSHOT_BLOCK_RANGE: usize = 1_000;
|
||||
|
||||
/// Known duplicate Bitcoin transactions (BIP30)
|
||||
|
||||
@@ -93,8 +93,7 @@ impl Lengths {
|
||||
.p2ms
|
||||
.first_index
|
||||
.checked_push(height, self.p2ms_output_index)?;
|
||||
vecs.scripts
|
||||
.op_return
|
||||
vecs.op_return
|
||||
.first_index
|
||||
.checked_push(height, self.op_return_index)?;
|
||||
vecs.addrs
|
||||
@@ -177,8 +176,8 @@ impl Lengths {
|
||||
height,
|
||||
)?,
|
||||
op_return_index: next_index(
|
||||
&vecs.scripts.op_return.first_index,
|
||||
&vecs.scripts.op_return.to_tx_index,
|
||||
&vecs.op_return.first_index,
|
||||
&vecs.op_return.to_tx_index,
|
||||
height,
|
||||
)?,
|
||||
p2pk33_addr_index: next_index(
|
||||
|
||||
@@ -37,6 +37,7 @@ pub struct Indexer<M: StorageMode = Rw> {
|
||||
path: PathBuf,
|
||||
pub vecs: Vecs<M>,
|
||||
pub stores: Stores,
|
||||
buffers: BlockBuffers,
|
||||
safe_lengths: SafeLengths,
|
||||
}
|
||||
|
||||
@@ -53,7 +54,12 @@ impl<M: StorageMode> Indexer<M> {
|
||||
/// actually queryable.
|
||||
pub fn tip_blockhash(&self) -> BlockHash {
|
||||
match self.safe_lengths().height.decremented() {
|
||||
Some(h) => self.vecs.blocks.blockhash.collect_one(h).unwrap_or_default(),
|
||||
Some(h) => self
|
||||
.vecs
|
||||
.blocks
|
||||
.blockhash
|
||||
.collect_one(h)
|
||||
.unwrap_or_default(),
|
||||
None => BlockHash::default(),
|
||||
}
|
||||
}
|
||||
@@ -102,6 +108,7 @@ impl Indexer {
|
||||
path: indexed_path.clone(),
|
||||
vecs,
|
||||
stores,
|
||||
buffers: BlockBuffers::default(),
|
||||
safe_lengths,
|
||||
})
|
||||
};
|
||||
@@ -128,6 +135,7 @@ impl Indexer {
|
||||
/// record that gets replayed on every recovery), this cleanly recreates.
|
||||
fn full_reset(&mut self) -> Result<()> {
|
||||
info!("Full reset...");
|
||||
self.buffers.reset();
|
||||
self.safe_lengths.reset();
|
||||
self.vecs.reset()?;
|
||||
let stores_path = self.path.join("stores");
|
||||
@@ -196,6 +204,8 @@ impl Indexer {
|
||||
debug!("Rollback vecs done.");
|
||||
drop(lock);
|
||||
|
||||
self.buffers.continue_from(prev_hash);
|
||||
|
||||
let mut lengths = starting_lengths;
|
||||
|
||||
let is_export_height =
|
||||
@@ -227,10 +237,10 @@ impl Indexer {
|
||||
};
|
||||
|
||||
let mut readers = Readers::new(&self.vecs);
|
||||
let mut buffers = BlockBuffers::default();
|
||||
|
||||
let vecs = &mut self.vecs;
|
||||
let stores = &mut self.stores;
|
||||
let buffers = &mut self.buffers;
|
||||
|
||||
for block in reader.after(prev_hash)?.iter() {
|
||||
let block = match block {
|
||||
@@ -273,12 +283,11 @@ impl Indexer {
|
||||
processor.process_block_metadata()?;
|
||||
|
||||
let txs = processor.compute_txids()?;
|
||||
|
||||
processor.push_block_size_and_weight(&txs)?;
|
||||
|
||||
let (txins_result, txouts_result) = rayon::join(
|
||||
|| processor.process_inputs(&txs, &mut buffers.txid_prefix_map),
|
||||
|| processor.process_outputs(),
|
||||
|| processor.process_inputs(&txs, &mut buffers.inputs),
|
||||
|| processor.process_outputs(&mut buffers.addresses),
|
||||
);
|
||||
let txins = txins_result?;
|
||||
let txouts = txouts_result?;
|
||||
@@ -287,28 +296,17 @@ impl Indexer {
|
||||
let input_count = txins.len();
|
||||
let output_count = txouts.len();
|
||||
|
||||
BlockProcessor::collect_same_block_spent_outpoints(
|
||||
&txins,
|
||||
&mut buffers.same_block_spent,
|
||||
);
|
||||
|
||||
processor.check_txid_collisions(&txs)?;
|
||||
|
||||
let sigops = processor.compute_sigops(&txins, &txouts);
|
||||
|
||||
processor.finalize_and_store_metadata(
|
||||
processor.analyze_and_finalize_transactions(
|
||||
txs,
|
||||
txouts,
|
||||
txins,
|
||||
sigops,
|
||||
&buffers.same_block_spent,
|
||||
&mut buffers.already_added_addrs,
|
||||
&mut buffers.same_block_output_info,
|
||||
&mut buffers.addresses,
|
||||
)?;
|
||||
|
||||
processor
|
||||
.lengths
|
||||
.add_block(tx_count, input_count, output_count);
|
||||
buffers.finish_block(*block.hash());
|
||||
|
||||
if is_export_height(height) {
|
||||
drop(readers);
|
||||
@@ -389,6 +387,7 @@ impl ReadOnlyClone for Indexer {
|
||||
path: self.path.clone(),
|
||||
vecs: self.vecs.read_only_clone(),
|
||||
stores: self.stores.clone(),
|
||||
buffers: BlockBuffers::default(),
|
||||
safe_lengths: self.safe_lengths.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
+12
-11
@@ -1,21 +1,22 @@
|
||||
use brk_error::{Error, Result};
|
||||
use brk_types::{BlockHashPrefix, Timestamp};
|
||||
use tracing::error;
|
||||
use vecdb::WritableVec;
|
||||
use vecdb::{WritableVec, unlikely};
|
||||
|
||||
use super::{BlockProcessor, ComputedTx};
|
||||
use super::{BlockProcessor, transaction::ComputedTx};
|
||||
|
||||
impl BlockProcessor<'_> {
|
||||
pub fn process_block_metadata(&mut self) -> Result<()> {
|
||||
pub(crate) fn process_block_metadata(&mut self) -> Result<()> {
|
||||
let height = self.height;
|
||||
let blockhash = self.block.hash();
|
||||
let blockhash_prefix = BlockHashPrefix::from(blockhash);
|
||||
|
||||
if self
|
||||
.stores
|
||||
.blockhash_prefix_to_height
|
||||
.get(&blockhash_prefix)?
|
||||
.is_some_and(|prev_height| *prev_height != height)
|
||||
if unlikely(self.check_collisions)
|
||||
&& self
|
||||
.stores
|
||||
.blockhash_prefix_to_height
|
||||
.get(&blockhash_prefix)?
|
||||
.is_some_and(|prev_height| *prev_height != height)
|
||||
{
|
||||
error!("BlockHash: {blockhash}");
|
||||
return Err(Error::Internal("BlockHash prefix collision"));
|
||||
@@ -51,13 +52,13 @@ impl BlockProcessor<'_> {
|
||||
|
||||
/// Push block total_size and weight, reusing per-tx sizes already computed in ComputedTx.
|
||||
/// This avoids redundant tx serialization (base_size + total_size were already computed).
|
||||
pub fn push_block_size_and_weight(&mut self, txs: &[ComputedTx]) -> Result<()> {
|
||||
pub(crate) fn push_block_size_and_weight(&mut self, txs: &[ComputedTx]) -> Result<()> {
|
||||
let overhead = bitcoin::block::Header::SIZE + bitcoin::VarInt::from(txs.len()).size();
|
||||
let mut total_size = overhead;
|
||||
let mut weight = overhead * 4;
|
||||
let mut weight = bitcoin::Weight::from_non_witness_data_size(overhead as u64);
|
||||
let mut sw_txs = 0u32;
|
||||
let mut sw_size = 0usize;
|
||||
let mut sw_weight = 0usize;
|
||||
let mut sw_weight = bitcoin::Weight::ZERO;
|
||||
|
||||
for (i, tx) in txs.iter().enumerate() {
|
||||
total_size += tx.total_size as usize;
|
||||
@@ -0,0 +1,29 @@
|
||||
use brk_types::BlockHash;
|
||||
|
||||
use super::{txin::InputResolver, txout::BlockAddresses};
|
||||
|
||||
/// Reusable buffers cleared and refilled each block to avoid allocation churn.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct BlockBuffers {
|
||||
pub(crate) inputs: InputResolver,
|
||||
pub(crate) addresses: BlockAddresses,
|
||||
tip: Option<BlockHash>,
|
||||
}
|
||||
|
||||
impl BlockBuffers {
|
||||
pub(crate) fn continue_from(&mut self, parent: Option<BlockHash>) {
|
||||
if self.tip != parent {
|
||||
self.addresses.clear_cache();
|
||||
}
|
||||
self.tip = parent;
|
||||
}
|
||||
|
||||
pub(crate) fn finish_block(&mut self, blockhash: BlockHash) {
|
||||
self.tip = Some(blockhash);
|
||||
}
|
||||
|
||||
pub(crate) fn reset(&mut self) {
|
||||
self.addresses.clear_cache();
|
||||
self.tip = None;
|
||||
}
|
||||
}
|
||||
@@ -1,89 +1,22 @@
|
||||
mod metadata;
|
||||
mod sigops;
|
||||
mod tx;
|
||||
mod block;
|
||||
mod buffer;
|
||||
mod transaction;
|
||||
mod txin;
|
||||
mod txout;
|
||||
mod types;
|
||||
|
||||
pub use types::*;
|
||||
pub(crate) use buffer::BlockBuffers;
|
||||
|
||||
use brk_cohort::ByAddrType;
|
||||
use brk_error::Result;
|
||||
use brk_types::{AddrHash, Block, Height, OutPoint, SigOps, TxInIndex, TypeIndex};
|
||||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
use brk_types::{Block, Height};
|
||||
|
||||
use crate::{Lengths, Readers, Stores, Vecs};
|
||||
|
||||
/// Processes a single block, extracting and storing all indexed data.
|
||||
pub struct BlockProcessor<'a> {
|
||||
pub block: &'a Block,
|
||||
pub height: Height,
|
||||
pub check_collisions: bool,
|
||||
pub lengths: &'a mut Lengths,
|
||||
pub vecs: &'a mut Vecs,
|
||||
pub stores: &'a mut Stores,
|
||||
pub readers: &'a Readers,
|
||||
}
|
||||
|
||||
impl BlockProcessor<'_> {
|
||||
/// Finalizes outputs/inputs in parallel with storing tx metadata.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn finalize_and_store_metadata(
|
||||
&mut self,
|
||||
txs: Vec<ComputedTx>,
|
||||
txouts: Vec<ProcessedOutput>,
|
||||
txins: Vec<(TxInIndex, InputSource)>,
|
||||
sigops: Vec<SigOps>,
|
||||
same_block_spent_outpoints: &FxHashSet<OutPoint>,
|
||||
already_added: &mut ByAddrType<FxHashMap<AddrHash, TypeIndex>>,
|
||||
same_block_info: &mut FxHashMap<OutPoint, SameBlockOutputInfo>,
|
||||
) -> Result<()> {
|
||||
let lengths = &mut *self.lengths;
|
||||
|
||||
// Split transactions vecs: finalize needs first_txout_index/first_txin_index, metadata needs the rest
|
||||
let (first_txout_index, first_txin_index, mut tx_metadata) =
|
||||
self.vecs.transactions.split_for_finalize();
|
||||
|
||||
let outputs = &mut self.vecs.outputs;
|
||||
let inputs = &mut self.vecs.inputs;
|
||||
let addrs = &mut self.vecs.addrs;
|
||||
let scripts = &mut self.vecs.scripts;
|
||||
|
||||
let addr_hash_stores = &mut self.stores.addr_type_to_addr_hash_to_addr_index;
|
||||
let addr_tx_index_stores = &mut self.stores.addr_type_to_addr_index_and_tx_index;
|
||||
let addr_outpoint_stores = &mut self.stores.addr_type_to_addr_index_and_unspent_outpoint;
|
||||
let txid_prefix_store = &mut self.stores.txid_prefix_to_tx_index;
|
||||
|
||||
let (finalize_result, metadata_result) = rayon::join(
|
||||
|| -> Result<()> {
|
||||
txout::finalize_outputs(
|
||||
lengths,
|
||||
first_txout_index,
|
||||
outputs,
|
||||
addrs,
|
||||
scripts,
|
||||
addr_hash_stores,
|
||||
addr_tx_index_stores,
|
||||
addr_outpoint_stores,
|
||||
txouts,
|
||||
same_block_spent_outpoints,
|
||||
already_added,
|
||||
same_block_info,
|
||||
)?;
|
||||
txin::finalize_inputs(
|
||||
first_txin_index,
|
||||
inputs,
|
||||
addr_tx_index_stores,
|
||||
addr_outpoint_stores,
|
||||
txins,
|
||||
same_block_info,
|
||||
)
|
||||
},
|
||||
|| tx::store_tx_metadata(txs, sigops, txid_prefix_store, &mut tx_metadata),
|
||||
);
|
||||
|
||||
finalize_result?;
|
||||
metadata_result?;
|
||||
Ok(())
|
||||
}
|
||||
pub(crate) struct BlockProcessor<'a> {
|
||||
pub(crate) block: &'a Block,
|
||||
pub(crate) height: Height,
|
||||
pub(crate) check_collisions: bool,
|
||||
pub(crate) lengths: &'a mut Lengths,
|
||||
pub(crate) vecs: &'a mut Vecs,
|
||||
pub(crate) stores: &'a mut Stores,
|
||||
pub(crate) readers: &'a Readers,
|
||||
}
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
use bitcoin::{Script, script::Instruction};
|
||||
use brk_types::{OutputType, SigOps, TxInIndex};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use super::{BlockProcessor, InputSource, ProcessedOutput};
|
||||
|
||||
impl BlockProcessor<'_> {
|
||||
/// BIP-141 sigop cost per tx in the block. Mirrors
|
||||
/// `bitcoin::Transaction::total_sigop_cost` but dispatches on each
|
||||
/// input's prevout `OutputType` and each output's `OutputType`
|
||||
/// (already resolved by `process_inputs`/`process_outputs`) instead
|
||||
/// of round-tripping through bitcoin's closure API with
|
||||
/// synthetic-prevout `ScriptBuf` allocations. The legacy-sigop walk
|
||||
/// is short-circuited by `OutputType` for every script with a
|
||||
/// canonical shape (~99% of outputs and ~95% of inputs on mainnet);
|
||||
/// only `OpReturn`/`Unknown` outputs and non-segwit/non-P2SH inputs
|
||||
/// fall back to a real script walk.
|
||||
pub fn compute_sigops(
|
||||
&self,
|
||||
txins: &[(TxInIndex, InputSource)],
|
||||
txouts: &[ProcessedOutput<'_>],
|
||||
) -> Vec<SigOps> {
|
||||
let txdata = &self.block.txdata;
|
||||
let base_tx_index = u32::from(self.lengths.tx_index);
|
||||
|
||||
let mut tx_input_offsets = Vec::with_capacity(txdata.len());
|
||||
let mut tx_output_offsets = Vec::with_capacity(txdata.len());
|
||||
let mut input_offset = 0usize;
|
||||
let mut output_offset = 0usize;
|
||||
for tx in txdata {
|
||||
tx_input_offsets.push(input_offset);
|
||||
input_offset += tx.input.len();
|
||||
tx_output_offsets.push(output_offset);
|
||||
output_offset += tx.output.len();
|
||||
}
|
||||
|
||||
txdata
|
||||
.par_iter()
|
||||
.enumerate()
|
||||
.map(|(i, tx)| {
|
||||
if tx.is_coinbase() {
|
||||
return SigOps::ZERO;
|
||||
}
|
||||
let in_start = tx_input_offsets[i];
|
||||
let tx_inputs = &txins[in_start..in_start + tx.input.len()];
|
||||
let out_start = tx_output_offsets[i];
|
||||
let tx_outputs = &txouts[out_start..out_start + tx.output.len()];
|
||||
|
||||
let mut legacy: usize = 0;
|
||||
let mut redeem: usize = 0;
|
||||
let mut witness: usize = 0;
|
||||
|
||||
for (input, (_, source)) in tx.input.iter().zip(tx_inputs.iter()) {
|
||||
let prev_kind = match source {
|
||||
InputSource::PreviousBlock { output_type, .. } => *output_type,
|
||||
InputSource::SameBlock { outpoint, .. } => {
|
||||
let local = (u32::from(outpoint.tx_index()) - base_tx_index) as usize;
|
||||
let vout = u32::from(outpoint.vout()) as usize;
|
||||
txouts[tx_output_offsets[local] + vout].output_type
|
||||
}
|
||||
};
|
||||
|
||||
// Single match per input: legacy script_sig sigops AND the
|
||||
// redeem/witness contribution. Consensus enforces a
|
||||
// push-only or empty script_sig in the four cases below
|
||||
// (BIP-16 for P2SH from block 173805 onwards; BIP-141 /
|
||||
// BIP-341 for segwit/taproot from activation), so legacy
|
||||
// sigops are guaranteed 0 there. Everything else falls
|
||||
// through to a real `count_sigops_legacy` walk.
|
||||
match prev_kind {
|
||||
OutputType::P2SH => {
|
||||
// Faithful to bitcoin's count_p2sh_sigops + the
|
||||
// nested-segwit branch of count_witness_sigops in
|
||||
// a single script walk: redeem sigops use
|
||||
// last_pushdata (no push-only check), wrapped
|
||||
// witness sigops require both push-only and
|
||||
// last_pushdata.
|
||||
let (last_push, is_push_only) =
|
||||
last_push_and_push_only(&input.script_sig);
|
||||
let Some(redeem_bytes) = last_push else {
|
||||
continue;
|
||||
};
|
||||
let rs = Script::from_bytes(redeem_bytes);
|
||||
redeem = redeem.saturating_add(rs.count_sigops());
|
||||
if !is_push_only {
|
||||
continue;
|
||||
}
|
||||
if rs.is_p2wpkh() {
|
||||
witness = witness.saturating_add(1);
|
||||
} else if rs.is_p2wsh()
|
||||
&& let Some(last) = input.witness.last()
|
||||
{
|
||||
witness =
|
||||
witness.saturating_add(Script::from_bytes(last).count_sigops());
|
||||
}
|
||||
}
|
||||
OutputType::P2WPKH => {
|
||||
witness = witness.saturating_add(1);
|
||||
}
|
||||
OutputType::P2WSH => {
|
||||
if let Some(last) = input.witness.last() {
|
||||
witness =
|
||||
witness.saturating_add(Script::from_bytes(last).count_sigops());
|
||||
}
|
||||
}
|
||||
OutputType::P2TR => {}
|
||||
_ => {
|
||||
legacy = legacy.saturating_add(input.script_sig.count_sigops_legacy());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for processed in tx_outputs {
|
||||
legacy = legacy.saturating_add(legacy_sigops_for_output(
|
||||
processed.output_type,
|
||||
&processed.txout.script_pubkey,
|
||||
));
|
||||
}
|
||||
|
||||
SigOps::from(
|
||||
legacy
|
||||
.saturating_mul(4)
|
||||
.saturating_add(redeem.saturating_mul(4))
|
||||
.saturating_add(witness),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Legacy sigop count of a script_pubkey, dispatched on `OutputType`.
|
||||
/// Every variant except `OpReturn` and `Unknown` has a canonical shape
|
||||
/// recognised by `OutputType::from`'s exact byte-pattern matchers, so
|
||||
/// the legacy sigop count is fixed: P2PKH and P2PK both end in a
|
||||
/// single OP_CHECKSIG (1), P2MS contains one OP_CHECKMULTISIG counted
|
||||
/// as 20 in legacy mode, and P2SH/P2WPKH/P2WSH/P2TR/P2A/Empty contain
|
||||
/// no CHECKSIG-class opcodes outside their pushdata. `OpReturn`
|
||||
/// payloads can include 0xac/0xae bytes outside a push, and `Unknown`
|
||||
/// can be anything, so both fall back to a real script walk.
|
||||
#[inline]
|
||||
fn legacy_sigops_for_output(output_type: OutputType, script_pubkey: &Script) -> usize {
|
||||
match output_type {
|
||||
OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65 => 1,
|
||||
OutputType::P2MS => 20,
|
||||
OutputType::P2SH
|
||||
| OutputType::P2WPKH
|
||||
| OutputType::P2WSH
|
||||
| OutputType::P2TR
|
||||
| OutputType::P2A
|
||||
| OutputType::Empty => 0,
|
||||
OutputType::OpReturn | OutputType::Unknown => script_pubkey.count_sigops_legacy(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Single-pass equivalent of bitcoin's private `last_pushdata()` plus the
|
||||
/// public `is_push_only()`: returns the bytes of the script's last
|
||||
/// `Instruction::PushBytes` (only when it is the *last* instruction)
|
||||
/// alongside whether every instruction was a push (per Core,
|
||||
/// `OP_RESERVED` and `OP_PUSHNUM_1..16` count as pushes too).
|
||||
fn last_push_and_push_only(script: &Script) -> (Option<&[u8]>, bool) {
|
||||
let mut last: Option<&[u8]> = None;
|
||||
let mut push_only = true;
|
||||
for inst in script.instructions() {
|
||||
match inst {
|
||||
Ok(Instruction::PushBytes(b)) => {
|
||||
last = Some(b.as_bytes());
|
||||
}
|
||||
Ok(Instruction::Op(op)) => {
|
||||
last = None;
|
||||
if op.to_u8() > 0x60 {
|
||||
push_only = false;
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
last = None;
|
||||
push_only = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
(last, push_only)
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
mod sighash;
|
||||
|
||||
use crate::TxFeatureFlags;
|
||||
|
||||
pub(super) fn scan_ecdsa_signature(bytes: &[u8], flags: &mut TxFeatureFlags) {
|
||||
sighash::scan_ecdsa_signature(bytes, flags);
|
||||
}
|
||||
|
||||
pub(super) fn scan_taproot_signature(bytes: &[u8], flags: &mut TxFeatureFlags) {
|
||||
sighash::scan_taproot_signature(bytes, flags);
|
||||
}
|
||||
|
||||
pub(super) fn record_validated_ecdsa_sighash(bytes: &[u8], flags: &mut TxFeatureFlags) {
|
||||
sighash::record_validated_ecdsa_sighash(bytes, flags);
|
||||
}
|
||||
|
||||
pub(super) fn record_validated_taproot_sighash(bytes: &[u8], flags: &mut TxFeatureFlags) {
|
||||
sighash::record_validated_taproot_sighash(bytes, flags);
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
use bitcoin::{
|
||||
secp256k1::{ecdsa::Signature as EcdsaSignature, schnorr::Signature as SchnorrSignature},
|
||||
sighash::{EcdsaSighashType, TapSighashType},
|
||||
};
|
||||
|
||||
use crate::TxFeatureFlags;
|
||||
|
||||
const MIN_ECDSA_SIGNATURE_BYTES: usize = 9;
|
||||
const MAX_ECDSA_SIGNATURE_BYTES: usize = 73;
|
||||
|
||||
pub(super) fn scan_ecdsa_signature(bytes: &[u8], flags: &mut TxFeatureFlags) {
|
||||
if !is_ecdsa_signature_candidate(bytes) {
|
||||
return;
|
||||
}
|
||||
|
||||
let (sighash_byte, der) = bytes.split_last().unwrap();
|
||||
let Ok(sighash_type) = EcdsaSighashType::from_standard(u32::from(*sighash_byte)) else {
|
||||
return;
|
||||
};
|
||||
let candidate_flags = ecdsa_feature_flags(sighash_type);
|
||||
if flags.contains_all(candidate_flags) {
|
||||
return;
|
||||
}
|
||||
|
||||
if EcdsaSignature::from_der(der).is_err() {
|
||||
return;
|
||||
}
|
||||
flags.insert(candidate_flags);
|
||||
}
|
||||
|
||||
pub(super) fn scan_taproot_signature(bytes: &[u8], flags: &mut TxFeatureFlags) {
|
||||
let (sighash_type, signature) = match bytes.len() {
|
||||
64 => (TapSighashType::Default, bytes),
|
||||
65 => {
|
||||
let Ok(sighash_type) = TapSighashType::from_consensus_u8(bytes[64]) else {
|
||||
return;
|
||||
};
|
||||
(sighash_type, &bytes[..64])
|
||||
}
|
||||
_ => return,
|
||||
};
|
||||
let candidate_flags = taproot_feature_flags(sighash_type);
|
||||
if flags.contains_all(candidate_flags) {
|
||||
return;
|
||||
}
|
||||
|
||||
if SchnorrSignature::from_slice(signature).is_err() {
|
||||
return;
|
||||
}
|
||||
flags.insert(candidate_flags);
|
||||
}
|
||||
|
||||
pub(super) fn record_validated_ecdsa_sighash(bytes: &[u8], flags: &mut TxFeatureFlags) {
|
||||
let Ok(sighash_type) = EcdsaSighashType::from_standard(u32::from(*bytes.last().unwrap()))
|
||||
else {
|
||||
return;
|
||||
};
|
||||
flags.insert(ecdsa_feature_flags(sighash_type));
|
||||
}
|
||||
|
||||
pub(super) fn record_validated_taproot_sighash(bytes: &[u8], flags: &mut TxFeatureFlags) {
|
||||
let sighash_type = match bytes.len() {
|
||||
64 => TapSighashType::Default,
|
||||
65 => {
|
||||
let Ok(sighash_type) = TapSighashType::from_consensus_u8(bytes[64]) else {
|
||||
return;
|
||||
};
|
||||
sighash_type
|
||||
}
|
||||
_ => return,
|
||||
};
|
||||
flags.insert(taproot_feature_flags(sighash_type));
|
||||
}
|
||||
|
||||
fn ecdsa_feature_flags(sighash_type: EcdsaSighashType) -> u32 {
|
||||
let mut flags = match sighash_type {
|
||||
EcdsaSighashType::All | EcdsaSighashType::AllPlusAnyoneCanPay => {
|
||||
TxFeatureFlags::SIGHASH_ALL
|
||||
}
|
||||
EcdsaSighashType::None | EcdsaSighashType::NonePlusAnyoneCanPay => {
|
||||
TxFeatureFlags::SIGHASH_NONE
|
||||
}
|
||||
EcdsaSighashType::Single | EcdsaSighashType::SinglePlusAnyoneCanPay => {
|
||||
TxFeatureFlags::SIGHASH_SINGLE
|
||||
}
|
||||
};
|
||||
if sighash_type.to_u32() & 0x80 != 0 {
|
||||
flags |= TxFeatureFlags::SIGHASH_ANYONE_CAN_PAY;
|
||||
}
|
||||
flags
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_ecdsa_signature_candidate(bytes: &[u8]) -> bool {
|
||||
(MIN_ECDSA_SIGNATURE_BYTES..=MAX_ECDSA_SIGNATURE_BYTES).contains(&bytes.len())
|
||||
&& bytes.first() == Some(&0x30)
|
||||
}
|
||||
|
||||
fn taproot_feature_flags(sighash_type: TapSighashType) -> u32 {
|
||||
let mut flags = match sighash_type {
|
||||
TapSighashType::Default => TxFeatureFlags::SIGHASH_DEFAULT,
|
||||
TapSighashType::All | TapSighashType::AllPlusAnyoneCanPay => TxFeatureFlags::SIGHASH_ALL,
|
||||
TapSighashType::None | TapSighashType::NonePlusAnyoneCanPay => TxFeatureFlags::SIGHASH_NONE,
|
||||
TapSighashType::Single | TapSighashType::SinglePlusAnyoneCanPay => {
|
||||
TxFeatureFlags::SIGHASH_SINGLE
|
||||
}
|
||||
};
|
||||
if matches!(
|
||||
sighash_type,
|
||||
TapSighashType::AllPlusAnyoneCanPay
|
||||
| TapSighashType::NonePlusAnyoneCanPay
|
||||
| TapSighashType::SinglePlusAnyoneCanPay
|
||||
) {
|
||||
flags |= TxFeatureFlags::SIGHASH_ANYONE_CAN_PAY;
|
||||
}
|
||||
flags
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::is_ecdsa_signature_candidate;
|
||||
|
||||
#[test]
|
||||
fn rejects_impossible_ecdsa_signature_encodings() {
|
||||
assert!(!is_ecdsa_signature_candidate(&[0x30; 8]));
|
||||
assert!(is_ecdsa_signature_candidate(&[0x30; 9]));
|
||||
assert!(is_ecdsa_signature_candidate(&[0x30; 73]));
|
||||
assert!(!is_ecdsa_signature_candidate(&[0x30; 74]));
|
||||
assert!(!is_ecdsa_signature_candidate(&[0x02; 33]));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
mod redeem;
|
||||
mod script_sig;
|
||||
mod witness;
|
||||
|
||||
use bitcoin::{TxIn, taproot::LeafVersion};
|
||||
use brk_types::OutputType;
|
||||
|
||||
use crate::TxFeatureFlags;
|
||||
|
||||
pub(super) struct Facts<'a> {
|
||||
pub(super) script_sig: ScriptSigFacts<'a>,
|
||||
redeem: redeem::Facts,
|
||||
pub(super) witness: WitnessFacts<'a>,
|
||||
}
|
||||
|
||||
impl Facts<'_> {
|
||||
#[inline]
|
||||
pub(super) fn redeem_sigops(&self) -> Option<usize> {
|
||||
self.redeem.sigops()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(super) fn redeem_is_p2wpkh(&self) -> bool {
|
||||
self.redeem.is_p2wpkh()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(super) fn redeem_is_p2wsh(&self) -> bool {
|
||||
self.redeem.is_p2wsh()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(super) fn redeem_is_witness_program(&self) -> bool {
|
||||
self.redeem.is_witness_program()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct ScriptSigFacts<'a> {
|
||||
pub(super) accurate_sigops: usize,
|
||||
pub(super) last_push: Option<&'a [u8]>,
|
||||
pub(super) legacy_sigops: usize,
|
||||
pub(super) push_only: bool,
|
||||
}
|
||||
|
||||
impl ScriptSigFacts<'_> {
|
||||
const EMPTY: Self = Self {
|
||||
accurate_sigops: 0,
|
||||
last_push: None,
|
||||
legacy_sigops: 0,
|
||||
push_only: true,
|
||||
};
|
||||
}
|
||||
|
||||
pub(super) struct WitnessFacts<'a> {
|
||||
pub(super) has_annex: bool,
|
||||
pub(super) last: Option<&'a [u8]>,
|
||||
pub(super) leaf_version: Option<LeafVersion>,
|
||||
pub(super) max_argument_bytes: usize,
|
||||
pub(super) stack_items: usize,
|
||||
}
|
||||
|
||||
pub(super) fn analyze<'a>(
|
||||
input: &'a TxIn,
|
||||
output_type: OutputType,
|
||||
flags: &mut TxFeatureFlags,
|
||||
) -> Facts<'a> {
|
||||
flags.insert_type(output_type);
|
||||
|
||||
let script_sig = if input.script_sig.is_empty() {
|
||||
ScriptSigFacts::EMPTY
|
||||
} else {
|
||||
script_sig::analyze(
|
||||
&input.script_sig,
|
||||
output_type,
|
||||
input.witness.is_empty(),
|
||||
flags,
|
||||
)
|
||||
};
|
||||
let redeem = redeem::Facts::analyze(script_sig.last_push, output_type);
|
||||
let witness = witness::analyze(
|
||||
&input.witness,
|
||||
redeem.effective_output_type(output_type, input.witness.len()),
|
||||
flags,
|
||||
);
|
||||
|
||||
Facts {
|
||||
script_sig,
|
||||
redeem,
|
||||
witness,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use bitcoin::TxIn;
|
||||
use brk_types::OutputType;
|
||||
|
||||
use super::analyze;
|
||||
use crate::TxFeatureFlags;
|
||||
|
||||
#[test]
|
||||
fn empty_script_sig_has_empty_facts() {
|
||||
let input = TxIn::default();
|
||||
let facts = analyze(&input, OutputType::Unknown, &mut TxFeatureFlags::default());
|
||||
|
||||
assert_eq!(facts.script_sig.accurate_sigops, 0);
|
||||
assert_eq!(facts.script_sig.last_push, None);
|
||||
assert_eq!(facts.script_sig.legacy_sigops, 0);
|
||||
assert!(facts.script_sig.push_only);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
use bitcoin::{Script, WitnessVersion};
|
||||
use brk_types::OutputType;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Kind {
|
||||
None,
|
||||
P2WPKH,
|
||||
P2WSH,
|
||||
Witness,
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct Facts {
|
||||
kind: Kind,
|
||||
sigops: usize,
|
||||
}
|
||||
|
||||
impl Facts {
|
||||
pub(super) fn analyze(last_push: Option<&[u8]>, output_type: OutputType) -> Self {
|
||||
if output_type != OutputType::P2SH {
|
||||
return Self::NONE;
|
||||
}
|
||||
|
||||
let Some(bytes) = last_push else {
|
||||
return Self::NONE;
|
||||
};
|
||||
let script = Script::from_bytes(bytes);
|
||||
let kind = match script.witness_version() {
|
||||
Some(WitnessVersion::V0) if script.len() == 22 => Kind::P2WPKH,
|
||||
Some(WitnessVersion::V0) if script.len() == 34 => Kind::P2WSH,
|
||||
Some(_) => Kind::Witness,
|
||||
None => Kind::Other,
|
||||
};
|
||||
|
||||
Self {
|
||||
kind,
|
||||
sigops: script.count_sigops(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn effective_output_type(
|
||||
self,
|
||||
output_type: OutputType,
|
||||
witness_items: usize,
|
||||
) -> OutputType {
|
||||
match self.kind {
|
||||
Kind::P2WPKH if witness_items == 2 => OutputType::P2WPKH,
|
||||
_ => output_type,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_present(self) -> bool {
|
||||
self.kind != Kind::None
|
||||
}
|
||||
|
||||
pub(super) fn is_p2wpkh(self) -> bool {
|
||||
self.kind == Kind::P2WPKH
|
||||
}
|
||||
|
||||
pub(super) fn is_p2wsh(self) -> bool {
|
||||
self.kind == Kind::P2WSH
|
||||
}
|
||||
|
||||
pub(super) fn is_witness_program(self) -> bool {
|
||||
matches!(self.kind, Kind::P2WPKH | Kind::P2WSH | Kind::Witness)
|
||||
}
|
||||
|
||||
pub(super) fn sigops(self) -> Option<usize> {
|
||||
self.is_present().then_some(self.sigops)
|
||||
}
|
||||
|
||||
const NONE: Self = Self {
|
||||
kind: Kind::None,
|
||||
sigops: 0,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use bitcoin::ScriptBuf;
|
||||
use brk_types::OutputType;
|
||||
|
||||
use super::{Facts, Kind};
|
||||
|
||||
fn classify(hex: &str) -> Facts {
|
||||
let script = ScriptBuf::from_hex(hex).unwrap();
|
||||
Facts::analyze(Some(script.as_bytes()), OutputType::P2SH)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_redeem_scripts_once() {
|
||||
assert_eq!(
|
||||
classify("00140000000000000000000000000000000000000000").kind,
|
||||
Kind::P2WPKH
|
||||
);
|
||||
assert_eq!(
|
||||
classify("00200000000000000000000000000000000000000000000000000000000000000000").kind,
|
||||
Kind::P2WSH
|
||||
);
|
||||
assert_eq!(
|
||||
classify("51200000000000000000000000000000000000000000000000000000000000000000").kind,
|
||||
Kind::Witness
|
||||
);
|
||||
assert_eq!(classify("ac").kind, Kind::Other);
|
||||
assert_eq!(classify("ac").sigops(), Some(1));
|
||||
assert_eq!(Facts::analyze(None, OutputType::P2SH).kind, Kind::None);
|
||||
assert_eq!(
|
||||
Facts::analyze(Some(&[0xac]), OutputType::P2PKH).kind,
|
||||
Kind::None
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
use bitcoin::{
|
||||
Script,
|
||||
opcodes::all::{OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY, OP_CHECKSIG, OP_CHECKSIGVERIFY},
|
||||
script::Instruction,
|
||||
};
|
||||
use brk_types::OutputType;
|
||||
|
||||
use super::{super::features, ScriptSigFacts};
|
||||
use crate::TxFeatureFlags;
|
||||
|
||||
pub(super) fn analyze<'a>(
|
||||
script: &'a Script,
|
||||
output_type: OutputType,
|
||||
scan_signatures: bool,
|
||||
flags: &mut TxFeatureFlags,
|
||||
) -> ScriptSigFacts<'a> {
|
||||
if scan_signatures && let Some((signature, last_push)) = direct_push_spend(script, output_type)
|
||||
{
|
||||
features::record_validated_ecdsa_sighash(signature, flags);
|
||||
return ScriptSigFacts {
|
||||
accurate_sigops: 0,
|
||||
last_push: Some(last_push),
|
||||
legacy_sigops: 0,
|
||||
push_only: true,
|
||||
};
|
||||
}
|
||||
|
||||
let mut accurate_sigops = 0;
|
||||
let mut first_push = None;
|
||||
let mut last_push = None;
|
||||
let mut legacy_sigops = 0;
|
||||
let mut only_push_bytes = true;
|
||||
let mut push_count = 0;
|
||||
let mut push_only = true;
|
||||
let mut pushnum = None;
|
||||
let validated_shape = scan_signatures
|
||||
&& matches!(
|
||||
output_type,
|
||||
OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65
|
||||
);
|
||||
|
||||
for instruction in script.instructions() {
|
||||
match instruction {
|
||||
Ok(Instruction::PushBytes(bytes)) => {
|
||||
let bytes = bytes.as_bytes();
|
||||
first_push.get_or_insert(bytes);
|
||||
last_push = Some(bytes);
|
||||
push_count += 1;
|
||||
pushnum = None;
|
||||
if scan_signatures && !validated_shape {
|
||||
features::scan_ecdsa_signature(bytes, flags);
|
||||
}
|
||||
}
|
||||
Ok(Instruction::Op(opcode)) => {
|
||||
only_push_bytes = false;
|
||||
last_push = None;
|
||||
if opcode.to_u8() > 0x60 {
|
||||
push_only = false;
|
||||
}
|
||||
match opcode {
|
||||
OP_CHECKSIG | OP_CHECKSIGVERIFY => {
|
||||
accurate_sigops += 1;
|
||||
legacy_sigops += 1;
|
||||
}
|
||||
OP_CHECKMULTISIG | OP_CHECKMULTISIGVERIFY => {
|
||||
accurate_sigops += pushnum.unwrap_or(20);
|
||||
legacy_sigops += 20;
|
||||
}
|
||||
_ => pushnum = decode_pushnum(opcode.to_u8()),
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
only_push_bytes = false;
|
||||
last_push = None;
|
||||
push_only = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if validated_shape {
|
||||
let expected_pushes = match output_type {
|
||||
OutputType::P2PKH => 2,
|
||||
OutputType::P2PK33 | OutputType::P2PK65 => 1,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
if only_push_bytes && push_count == expected_pushes {
|
||||
features::record_validated_ecdsa_sighash(first_push.unwrap(), flags);
|
||||
} else {
|
||||
scan_ecdsa_signatures(script, flags);
|
||||
}
|
||||
}
|
||||
|
||||
ScriptSigFacts {
|
||||
accurate_sigops,
|
||||
last_push,
|
||||
legacy_sigops,
|
||||
push_only,
|
||||
}
|
||||
}
|
||||
|
||||
fn direct_push_spend(script: &Script, output_type: OutputType) -> Option<(&[u8], &[u8])> {
|
||||
let bytes = script.as_bytes();
|
||||
let signature_len = usize::from(*bytes.first()?);
|
||||
if !(1..=75).contains(&signature_len) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let signature_end = 1 + signature_len;
|
||||
let signature = bytes.get(1..signature_end)?;
|
||||
|
||||
match output_type {
|
||||
OutputType::P2PK33 | OutputType::P2PK65 => {
|
||||
(signature_end == bytes.len()).then_some((signature, signature))
|
||||
}
|
||||
OutputType::P2PKH => {
|
||||
let public_key_len = usize::from(*bytes.get(signature_end)?);
|
||||
if !matches!(public_key_len, 33 | 65) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let public_key_start = signature_end + 1;
|
||||
let public_key_end = public_key_start + public_key_len;
|
||||
(public_key_end == bytes.len())
|
||||
.then(|| (signature, &bytes[public_key_start..public_key_end]))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_ecdsa_signatures(script: &Script, flags: &mut TxFeatureFlags) {
|
||||
for instruction in script.instructions() {
|
||||
if let Ok(Instruction::PushBytes(bytes)) = instruction {
|
||||
features::scan_ecdsa_signature(bytes.as_bytes(), flags);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn decode_pushnum(opcode: u8) -> Option<usize> {
|
||||
(0x51..=0x60)
|
||||
.contains(&opcode)
|
||||
.then(|| usize::from(opcode - 0x50))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use bitcoin::ScriptBuf;
|
||||
use brk_types::OutputType;
|
||||
|
||||
use super::analyze;
|
||||
use crate::TxFeatureFlags;
|
||||
|
||||
fn analyze_script(script: &ScriptBuf) -> super::ScriptSigFacts<'_> {
|
||||
analyze(
|
||||
script,
|
||||
OutputType::Unknown,
|
||||
false,
|
||||
&mut TxFeatureFlags::default(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analyzes_last_push_and_push_only_together() {
|
||||
let pushed = ScriptBuf::from_hex("03616263").unwrap();
|
||||
let facts = analyze_script(&pushed);
|
||||
assert_eq!(facts.last_push, Some(b"abc".as_slice()));
|
||||
assert!(facts.push_only);
|
||||
|
||||
let followed_by_checksig = ScriptBuf::from_hex("03616263ac").unwrap();
|
||||
let facts = analyze_script(&followed_by_checksig);
|
||||
assert_eq!(facts.last_push, None);
|
||||
assert!(!facts.push_only);
|
||||
|
||||
let push_number = ScriptBuf::from_hex("51").unwrap();
|
||||
let facts = analyze_script(&push_number);
|
||||
assert_eq!(facts.last_push, None);
|
||||
assert!(facts.push_only);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counts_legacy_and_accurate_sigops_together() {
|
||||
let script = ScriptBuf::from_hex("52aeac").unwrap();
|
||||
let facts = analyze_script(&script);
|
||||
|
||||
assert_eq!(facts.legacy_sigops, 21);
|
||||
assert_eq!(facts.accurate_sigops, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn records_validated_p2pkh_and_p2pk_sighashes() {
|
||||
let signature = [0x30, 0x06, 0x02, 0x01, 0x01, 0x02, 0x01, 0x01, 0x01];
|
||||
let compressed_key = [0x02; 33];
|
||||
let uncompressed_key = [0x04; 65];
|
||||
let mut scripts = Vec::new();
|
||||
|
||||
for public_key in [compressed_key.as_slice(), uncompressed_key.as_slice()] {
|
||||
let mut bytes = Vec::with_capacity(signature.len() + public_key.len() + 2);
|
||||
bytes.push(signature.len() as u8);
|
||||
bytes.extend(signature);
|
||||
bytes.push(public_key.len() as u8);
|
||||
bytes.extend(public_key);
|
||||
scripts.push((ScriptBuf::from_bytes(bytes), OutputType::P2PKH, public_key));
|
||||
}
|
||||
|
||||
let mut p2pk = Vec::with_capacity(signature.len() + 1);
|
||||
p2pk.push(signature.len() as u8);
|
||||
p2pk.extend(signature);
|
||||
scripts.push((
|
||||
ScriptBuf::from_bytes(p2pk),
|
||||
OutputType::P2PK33,
|
||||
signature.as_slice(),
|
||||
));
|
||||
|
||||
for (script, output_type, last_push) in scripts {
|
||||
let mut flags = TxFeatureFlags::default();
|
||||
let facts = analyze(&script, output_type, true, &mut flags);
|
||||
|
||||
assert_eq!(facts.accurate_sigops, 0);
|
||||
assert_eq!(facts.last_push, Some(last_push));
|
||||
assert_eq!(facts.legacy_sigops, 0);
|
||||
assert!(facts.push_only);
|
||||
assert!(flags.is_set(TxFeatureFlags::SIGHASH_ALL));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_for_noncanonical_p2pkh_shape() {
|
||||
let script = ScriptBuf::from_hex(
|
||||
"000930060201010201010121020000000000000000000000000000000000000000000000000000000000000000",
|
||||
)
|
||||
.unwrap();
|
||||
let mut flags = TxFeatureFlags::default();
|
||||
|
||||
analyze(&script, OutputType::P2PKH, true, &mut flags);
|
||||
|
||||
assert!(flags.is_set(TxFeatureFlags::SIGHASH_ALL));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
use bitcoin::{
|
||||
Script, Witness,
|
||||
opcodes::all::OP_IF,
|
||||
script::Instruction,
|
||||
taproot::{LeafVersion, TAPROOT_ANNEX_PREFIX, TAPROOT_CONTROL_BASE_SIZE, TAPROOT_LEAF_MASK},
|
||||
};
|
||||
use brk_types::OutputType;
|
||||
|
||||
use super::{super::features, WitnessFacts};
|
||||
use crate::TxFeatureFlags;
|
||||
|
||||
pub(super) fn analyze<'a>(
|
||||
witness: &'a Witness,
|
||||
output_type: OutputType,
|
||||
flags: &mut TxFeatureFlags,
|
||||
) -> WitnessFacts<'a> {
|
||||
match output_type {
|
||||
OutputType::P2TR => analyze_taproot(witness, flags),
|
||||
OutputType::P2WPKH => analyze_p2wpkh(witness, flags),
|
||||
_ => analyze_ecdsa(witness, flags),
|
||||
}
|
||||
}
|
||||
|
||||
fn analyze_p2wpkh<'a>(witness: &'a Witness, flags: &mut TxFeatureFlags) -> WitnessFacts<'a> {
|
||||
let mut stack = witness.iter();
|
||||
let signature = stack.next().unwrap();
|
||||
let public_key = stack.next().unwrap();
|
||||
debug_assert!(stack.next().is_none());
|
||||
features::record_validated_ecdsa_sighash(signature, flags);
|
||||
|
||||
WitnessFacts {
|
||||
has_annex: false,
|
||||
last: Some(public_key),
|
||||
leaf_version: None,
|
||||
max_argument_bytes: signature.len(),
|
||||
stack_items: 2,
|
||||
}
|
||||
}
|
||||
|
||||
fn analyze_ecdsa<'a>(witness: &'a Witness, flags: &mut TxFeatureFlags) -> WitnessFacts<'a> {
|
||||
let stack_items = witness.len();
|
||||
let mut last = None;
|
||||
let mut max_argument_bytes = 0;
|
||||
|
||||
for (index, item) in witness.iter().enumerate() {
|
||||
features::scan_ecdsa_signature(item, flags);
|
||||
if index + 1 < stack_items {
|
||||
max_argument_bytes = max_argument_bytes.max(item.len());
|
||||
}
|
||||
last = Some(item);
|
||||
}
|
||||
|
||||
WitnessFacts {
|
||||
has_annex: false,
|
||||
last,
|
||||
leaf_version: None,
|
||||
max_argument_bytes,
|
||||
stack_items,
|
||||
}
|
||||
}
|
||||
|
||||
fn analyze_taproot<'a>(witness: &'a Witness, flags: &mut TxFeatureFlags) -> WitnessFacts<'a> {
|
||||
let len = witness.len();
|
||||
if len == 1 {
|
||||
let signature = witness.last().unwrap();
|
||||
features::record_validated_taproot_sighash(signature, flags);
|
||||
return WitnessFacts {
|
||||
has_annex: false,
|
||||
last: Some(signature),
|
||||
leaf_version: None,
|
||||
max_argument_bytes: signature.len(),
|
||||
stack_items: 1,
|
||||
};
|
||||
}
|
||||
|
||||
let last = witness.last();
|
||||
let has_annex = len > 1 && last.is_some_and(|item| item.first() == Some(&TAPROOT_ANNEX_PREFIX));
|
||||
if has_annex {
|
||||
flags.insert(TxFeatureFlags::ANNEX);
|
||||
}
|
||||
|
||||
let stack_items = len - usize::from(has_annex);
|
||||
let argument_count = if stack_items == 1 {
|
||||
1
|
||||
} else {
|
||||
stack_items.saturating_sub(2)
|
||||
};
|
||||
let script_index = (stack_items >= 2).then(|| stack_items - 2);
|
||||
let control_index = (stack_items >= 2).then(|| stack_items - 1);
|
||||
let is_key_path = stack_items == 1;
|
||||
let mut leaf_version = None;
|
||||
let mut max_argument_bytes = 0;
|
||||
|
||||
for (index, item) in witness.iter().enumerate() {
|
||||
if index < argument_count {
|
||||
if is_key_path {
|
||||
features::record_validated_taproot_sighash(item, flags);
|
||||
} else {
|
||||
features::scan_taproot_signature(item, flags);
|
||||
}
|
||||
max_argument_bytes = max_argument_bytes.max(item.len());
|
||||
} else if Some(index) == script_index {
|
||||
if has_inscription_envelope(Script::from_bytes(item)) {
|
||||
flags.insert(TxFeatureFlags::INSCRIPTION);
|
||||
}
|
||||
} else if Some(index) == control_index && item.len() >= TAPROOT_CONTROL_BASE_SIZE {
|
||||
leaf_version = LeafVersion::from_consensus(item[0] & TAPROOT_LEAF_MASK).ok();
|
||||
}
|
||||
}
|
||||
|
||||
WitnessFacts {
|
||||
has_annex,
|
||||
last,
|
||||
leaf_version,
|
||||
max_argument_bytes,
|
||||
stack_items,
|
||||
}
|
||||
}
|
||||
|
||||
fn has_inscription_envelope(script: &Script) -> bool {
|
||||
let mut state = 0;
|
||||
for instruction in script.instructions() {
|
||||
state = match (state, instruction) {
|
||||
(0, Ok(Instruction::PushBytes(bytes))) if bytes.is_empty() => 1,
|
||||
(1, Ok(Instruction::Op(OP_IF))) => 2,
|
||||
(2, Ok(Instruction::PushBytes(bytes))) if bytes.as_bytes() == b"ord" => return true,
|
||||
(_, Ok(Instruction::PushBytes(bytes))) if bytes.is_empty() => 1,
|
||||
_ => 0,
|
||||
};
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use bitcoin::{ScriptBuf, TxIn, Witness, taproot::TAPROOT_ANNEX_PREFIX};
|
||||
use brk_types::OutputType;
|
||||
|
||||
use super::{analyze, has_inscription_envelope};
|
||||
use crate::TxFeatureFlags;
|
||||
|
||||
#[test]
|
||||
fn recognizes_only_ord_envelopes() {
|
||||
let inscription = ScriptBuf::from_hex("0063036f726468").unwrap();
|
||||
let generic_envelope = ScriptBuf::from_hex("006303666f6f68").unwrap();
|
||||
|
||||
assert!(has_inscription_envelope(&inscription));
|
||||
assert!(!has_inscription_envelope(&generic_envelope));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_tapscript_before_control_block_and_annex() {
|
||||
let script = ScriptBuf::from_hex("0063036f726468").unwrap();
|
||||
let control_block = [0xc0; 33];
|
||||
let annex = [TAPROOT_ANNEX_PREFIX, 0x01];
|
||||
let input = TxIn {
|
||||
witness: Witness::from_slice(&[
|
||||
[0x01].as_slice(),
|
||||
script.as_bytes(),
|
||||
control_block.as_slice(),
|
||||
annex.as_slice(),
|
||||
]),
|
||||
..TxIn::default()
|
||||
};
|
||||
let mut flags = TxFeatureFlags::default();
|
||||
|
||||
let facts = analyze(&input.witness, OutputType::P2TR, &mut flags);
|
||||
|
||||
assert!(facts.has_annex);
|
||||
assert!(flags.is_set(TxFeatureFlags::ANNEX));
|
||||
assert!(flags.is_set(TxFeatureFlags::INSCRIPTION));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_fixed_shape_p2wpkh_once() {
|
||||
let signature = [0x01; 71];
|
||||
let public_key = [0x02; 33];
|
||||
let witness = Witness::from_slice(&[signature.as_slice(), public_key.as_slice()]);
|
||||
let mut flags = TxFeatureFlags::default();
|
||||
|
||||
let facts = analyze(&witness, OutputType::P2WPKH, &mut flags);
|
||||
|
||||
assert_eq!(facts.last, Some(public_key.as_slice()));
|
||||
assert_eq!(facts.max_argument_bytes, signature.len());
|
||||
assert_eq!(facts.stack_items, 2);
|
||||
assert!(flags.is_set(TxFeatureFlags::SIGHASH_ALL));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_taproot_key_path_once() {
|
||||
let signature = [0x00; 64];
|
||||
let witness = Witness::from_slice(&[signature]);
|
||||
let mut flags = TxFeatureFlags::default();
|
||||
|
||||
let facts = analyze(&witness, OutputType::P2TR, &mut flags);
|
||||
|
||||
assert_eq!(facts.last, Some(signature.as_slice()));
|
||||
assert_eq!(facts.max_argument_bytes, signature.len());
|
||||
assert_eq!(facts.stack_items, 1);
|
||||
assert!(flags.is_set(TxFeatureFlags::SIGHASH_DEFAULT));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
mod features;
|
||||
mod input;
|
||||
mod output;
|
||||
mod policy;
|
||||
mod sigops;
|
||||
|
||||
use brk_types::{OutputType, SigOps};
|
||||
use rayon::prelude::*;
|
||||
use vecdb::{likely, unlikely};
|
||||
|
||||
use super::ComputedTx;
|
||||
use crate::TxFeatureFlags;
|
||||
use crate::processor::{BlockProcessor, txin::InputSource, txout::ProcessedOutput};
|
||||
|
||||
pub(super) struct TransactionAnalysis {
|
||||
pub(super) total_sigop_cost: SigOps,
|
||||
pub(super) explicitly_rbf: bool,
|
||||
pub(super) features: TxFeatureFlags,
|
||||
}
|
||||
|
||||
impl BlockProcessor<'_> {
|
||||
#[inline]
|
||||
pub(crate) fn tracks_executed_legacy_sigops(&self) -> bool {
|
||||
policy::tracks_executed_legacy_sigops(self.height)
|
||||
}
|
||||
|
||||
pub(super) fn analyze_transactions(
|
||||
&self,
|
||||
txs: &[ComputedTx<'_>],
|
||||
txins: &[InputSource],
|
||||
txouts: &[ProcessedOutput],
|
||||
) -> Vec<TransactionAnalysis> {
|
||||
let block_first_tx_index = self.lengths.tx_index;
|
||||
let track_executed_legacy = self.tracks_executed_legacy_sigops();
|
||||
|
||||
txs.par_iter()
|
||||
.map(|tx| {
|
||||
let tx_inputs = tx.inputs(txins);
|
||||
let tx_outputs = tx.outputs(txouts);
|
||||
let is_coinbase = tx.is_coinbase(block_first_tx_index);
|
||||
|
||||
let mut sigops = sigops::Accumulator::new(track_executed_legacy);
|
||||
let mut flags = TxFeatureFlags::default();
|
||||
let mut explicitly_rbf = false;
|
||||
let mut output_scanner = output::Scanner::default();
|
||||
let mut policy = policy::Accumulator::new(self.height);
|
||||
|
||||
if unlikely(is_coinbase) {
|
||||
let input = &tx.tx.input[0];
|
||||
explicitly_rbf = input.sequence.is_rbf();
|
||||
sigops.scan_coinbase_input(input);
|
||||
} else {
|
||||
for (input, source) in tx.tx.input.iter().zip(tx_inputs) {
|
||||
explicitly_rbf |= input.sequence.is_rbf();
|
||||
let (output_type, legacy_sigops) = resolved_output_facts(source, txouts);
|
||||
let facts = input::analyze(input, output_type, &mut flags);
|
||||
sigops.scan_input(output_type, legacy_sigops, &facts);
|
||||
policy.scan_input(input, output_type, &facts);
|
||||
}
|
||||
}
|
||||
|
||||
for (txout, output) in tx.tx.output.iter().zip(tx_outputs) {
|
||||
output_scanner.scan(txout, output, &mut flags);
|
||||
sigops.scan_output(txout, output);
|
||||
if likely(!is_coinbase) {
|
||||
policy.scan_output(txout, output);
|
||||
}
|
||||
}
|
||||
|
||||
let sigops = sigops.finish();
|
||||
if likely(!is_coinbase) {
|
||||
policy.finish(tx, sigops, &mut flags);
|
||||
}
|
||||
|
||||
TransactionAnalysis {
|
||||
total_sigop_cost: sigops.total,
|
||||
explicitly_rbf,
|
||||
features: flags,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn resolved_output_facts(source: &InputSource, txouts: &[ProcessedOutput]) -> (OutputType, SigOps) {
|
||||
match source {
|
||||
InputSource::PreviousBlock {
|
||||
output_type,
|
||||
legacy_sigops,
|
||||
..
|
||||
} => (*output_type, *legacy_sigops),
|
||||
InputSource::SameBlock { txout_offset, .. } => {
|
||||
let output = &txouts[*txout_offset];
|
||||
(output.output_type, output.legacy_sigops)
|
||||
}
|
||||
InputSource::Coinbase => unreachable!(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
use crate::TxFeatureFlags;
|
||||
use crate::processor::txout::ProcessedOutput;
|
||||
use bitcoin::{PublicKey, TxOut, script::Instruction};
|
||||
use brk_types::OutputType;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct Scanner {
|
||||
p2wsh_count: usize,
|
||||
payload_size: usize,
|
||||
}
|
||||
|
||||
impl Scanner {
|
||||
pub(super) fn scan(
|
||||
&mut self,
|
||||
txout: &TxOut,
|
||||
output: &ProcessedOutput,
|
||||
flags: &mut TxFeatureFlags,
|
||||
) {
|
||||
flags.insert_type(output.output_type);
|
||||
|
||||
match output.output_type {
|
||||
OutputType::P2PK33 | OutputType::P2PK65 => {
|
||||
if txout.script_pubkey.p2pk_public_key().is_none() {
|
||||
flags.insert(TxFeatureFlags::FAKE_PUBKEY);
|
||||
}
|
||||
self.p2wsh_count = 0;
|
||||
}
|
||||
OutputType::P2MS => {
|
||||
if has_fake_multisig_key(&txout.script_pubkey) {
|
||||
flags.insert(TxFeatureFlags::FAKE_PUBKEY);
|
||||
}
|
||||
self.p2wsh_count = 0;
|
||||
}
|
||||
OutputType::P2WSH => {
|
||||
let program = &txout.script_pubkey.as_bytes()[2..];
|
||||
if self.p2wsh_count == 0 {
|
||||
self.payload_size = u16::from_be_bytes([program[0], program[1]]) as usize;
|
||||
}
|
||||
self.p2wsh_count += 1;
|
||||
|
||||
let output_count = (self.payload_size + 33) / 32;
|
||||
if self.p2wsh_count == output_count {
|
||||
let padding = output_count * 32 - self.payload_size - 2;
|
||||
if program[32 - padding..].iter().all(|byte| *byte == 0) {
|
||||
flags.insert(TxFeatureFlags::FAKE_SCRIPTHASH);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => self.p2wsh_count = 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn has_fake_multisig_key(script: &bitcoin::Script) -> bool {
|
||||
script.instructions().any(|instruction| {
|
||||
let Ok(Instruction::PushBytes(bytes)) = instruction else {
|
||||
return false;
|
||||
};
|
||||
let bytes = bytes.as_bytes();
|
||||
is_burn_key(bytes) || PublicKey::from_slice(bytes).is_err()
|
||||
})
|
||||
}
|
||||
|
||||
fn is_burn_key(bytes: &[u8]) -> bool {
|
||||
bytes.len() == 33
|
||||
&& matches!(
|
||||
bytes,
|
||||
[0x02, rest @ ..]
|
||||
if rest.iter().all(|byte| *byte == 0x02)
|
||||
|| rest.iter().all(|byte| *byte == 0x22)
|
||||
)
|
||||
|| bytes.len() == 33
|
||||
&& matches!(
|
||||
bytes,
|
||||
[0x03, rest @ ..]
|
||||
if rest.iter().all(|byte| *byte == 0x03)
|
||||
|| rest.iter().all(|byte| *byte == 0x33)
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
use bitcoin::{
|
||||
Amount, Script, Transaction, TxIn, TxOut, WitnessVersion, policy::MAX_STANDARD_TX_SIGOPS_COST,
|
||||
taproot::LeafVersion,
|
||||
};
|
||||
use brk_types::{Height, OutputType, SigOps};
|
||||
|
||||
use super::super::ComputedTx;
|
||||
use super::{input, sigops::ComputedSigOps};
|
||||
use crate::TxFeatureFlags;
|
||||
use crate::processor::txout::ProcessedOutput;
|
||||
|
||||
// Deterministic policy snapshots, not consensus activation heights.
|
||||
const LAST_V2_POLICY_HEIGHT: u32 = 863_500;
|
||||
const FIRST_V29_POLICY_HEIGHT: u32 = 892_500;
|
||||
const FIRST_V30_POLICY_HEIGHT: u32 = 921_000;
|
||||
// rust-bitcoin 0.32 still exposes Core's previous 82-byte policy value.
|
||||
const MIN_STANDARD_TX_NONWITNESS_SIZE: u32 = 65;
|
||||
const MAX_EXECUTED_LEGACY_SIGOP_COST: u32 = 10_000;
|
||||
const MAX_SCRIPT_SIG_SIZE: usize = 1_650;
|
||||
const MAX_P2SH_SIGOPS: usize = 15;
|
||||
const MAX_V29_OP_RETURN_SCRIPT_BYTES: usize = 83;
|
||||
const MAX_V30_OP_RETURN_SCRIPT_BYTES: usize = 100_000;
|
||||
const MAX_P2WSH_SCRIPT_BYTES: usize = 3_600;
|
||||
const MAX_P2WSH_STACK_ITEMS: usize = 100;
|
||||
const MAX_WITNESS_STACK_ITEM_BYTES: usize = 80;
|
||||
const MAX_STANDARD_BARE_MULTISIG_SIGOP_COST: u32 =
|
||||
3 * bitcoin::constants::WITNESS_SCALE_FACTOR as u32;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct Accumulator {
|
||||
height: u32,
|
||||
nonstandard: bool,
|
||||
dust_output_count: usize,
|
||||
op_return_count: usize,
|
||||
op_return_script_bytes: usize,
|
||||
}
|
||||
|
||||
impl Accumulator {
|
||||
pub(super) fn new(height: Height) -> Self {
|
||||
Self {
|
||||
height: height.into(),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn scan_input(
|
||||
&mut self,
|
||||
input: &TxIn,
|
||||
output_type: OutputType,
|
||||
facts: &input::Facts<'_>,
|
||||
) {
|
||||
if input.script_sig.len() > MAX_SCRIPT_SIG_SIZE || !facts.script_sig.push_only {
|
||||
self.nonstandard = true;
|
||||
return;
|
||||
}
|
||||
|
||||
self.nonstandard |= match output_type {
|
||||
OutputType::P2SH => facts
|
||||
.redeem_sigops()
|
||||
.is_some_and(|count| count > MAX_P2SH_SIGOPS),
|
||||
OutputType::P2A => self.height <= LAST_V2_POLICY_HEIGHT,
|
||||
OutputType::P2TR => facts.witness.has_annex,
|
||||
OutputType::OpReturn | OutputType::Empty | OutputType::Unknown => true,
|
||||
_ => false,
|
||||
} || has_nonstandard_witness(output_type, facts);
|
||||
}
|
||||
|
||||
pub(super) fn scan_output(&mut self, txout: &TxOut, output: &ProcessedOutput) {
|
||||
let script = &txout.script_pubkey;
|
||||
self.nonstandard |= match output.output_type {
|
||||
OutputType::Empty => true,
|
||||
OutputType::Unknown => !is_standard_unknown_witness(script),
|
||||
OutputType::P2MS => has_too_many_bare_multisig_keys(output.legacy_sigops),
|
||||
OutputType::OpReturn => {
|
||||
self.op_return_count += 1;
|
||||
self.op_return_script_bytes += script.len();
|
||||
false
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
self.dust_output_count += is_dust(txout.value, output.output_type, script) as usize;
|
||||
}
|
||||
|
||||
pub(super) fn finish(
|
||||
mut self,
|
||||
tx: &ComputedTx<'_>,
|
||||
sigops: ComputedSigOps,
|
||||
flags: &mut TxFeatureFlags,
|
||||
) {
|
||||
self.nonstandard |= if self.height < FIRST_V30_POLICY_HEIGHT {
|
||||
self.op_return_count > 1 || self.op_return_script_bytes > MAX_V29_OP_RETURN_SCRIPT_BYTES
|
||||
} else {
|
||||
self.op_return_script_bytes > MAX_V30_OP_RETURN_SCRIPT_BYTES
|
||||
};
|
||||
self.nonstandard |=
|
||||
has_unconditionally_nonstandard_dust(self.height, self.dust_output_count);
|
||||
self.nonstandard |= has_nonstandard_header(tx, sigops, self.height);
|
||||
|
||||
if self.nonstandard {
|
||||
flags.insert(TxFeatureFlags::UNCONDITIONALLY_NONSTANDARD);
|
||||
}
|
||||
if self.dust_output_count > 0 {
|
||||
flags.insert(TxFeatureFlags::DUST_OUTPUT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn tracks_executed_legacy_sigops(height: Height) -> bool {
|
||||
u32::from(height) >= FIRST_V30_POLICY_HEIGHT
|
||||
}
|
||||
|
||||
fn has_nonstandard_header(tx: &ComputedTx, sigops: ComputedSigOps, height: u32) -> bool {
|
||||
has_nonstandard_version(tx.tx.version.0, height)
|
||||
|| tx.weight() > Transaction::MAX_STANDARD_WEIGHT
|
||||
|| tx.base_size < MIN_STANDARD_TX_NONWITNESS_SIZE
|
||||
|| u32::from(sigops.total) > MAX_STANDARD_TX_SIGOPS_COST
|
||||
|| height >= FIRST_V30_POLICY_HEIGHT
|
||||
&& u32::from(sigops.executed_legacy) > MAX_EXECUTED_LEGACY_SIGOP_COST
|
||||
}
|
||||
|
||||
fn has_unconditionally_nonstandard_dust(height: u32, dust_output_count: usize) -> bool {
|
||||
if height < FIRST_V29_POLICY_HEIGHT {
|
||||
dust_output_count > 0
|
||||
} else {
|
||||
dust_output_count > 1
|
||||
}
|
||||
}
|
||||
|
||||
fn is_dust(value: Amount, output_type: OutputType, script: &Script) -> bool {
|
||||
let threshold = match output_type {
|
||||
OutputType::P2PK65 => 672,
|
||||
OutputType::P2PK33 => 576,
|
||||
OutputType::P2PKH => 546,
|
||||
OutputType::P2MS | OutputType::Unknown => {
|
||||
return value < script.minimal_non_dust();
|
||||
}
|
||||
OutputType::P2SH => 540,
|
||||
OutputType::OpReturn => return false,
|
||||
OutputType::P2WPKH => 294,
|
||||
OutputType::P2WSH | OutputType::P2TR => 330,
|
||||
OutputType::P2A => 240,
|
||||
OutputType::Empty => 471,
|
||||
};
|
||||
|
||||
value.to_sat() < threshold
|
||||
}
|
||||
|
||||
fn has_nonstandard_witness(output_type: OutputType, facts: &input::Facts<'_>) -> bool {
|
||||
if facts.witness.stack_items == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
match output_type {
|
||||
OutputType::P2A => true,
|
||||
OutputType::P2WPKH => false,
|
||||
OutputType::P2WSH => has_nonstandard_p2wsh_witness(&facts.witness),
|
||||
OutputType::P2TR => {
|
||||
facts.witness.has_annex || has_nonstandard_taproot_witness(&facts.witness)
|
||||
}
|
||||
OutputType::P2SH => {
|
||||
if facts.redeem_sigops().is_none() {
|
||||
return true;
|
||||
}
|
||||
if facts.redeem_is_p2wsh() {
|
||||
has_nonstandard_p2wsh_witness(&facts.witness)
|
||||
} else {
|
||||
!facts.redeem_is_witness_program()
|
||||
}
|
||||
}
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
fn has_nonstandard_p2wsh_witness(witness: &input::WitnessFacts<'_>) -> bool {
|
||||
let stack_items = witness.stack_items - 1;
|
||||
witness.last.unwrap().len() > MAX_P2WSH_SCRIPT_BYTES
|
||||
|| stack_items > MAX_P2WSH_STACK_ITEMS
|
||||
|| witness.max_argument_bytes > MAX_WITNESS_STACK_ITEM_BYTES
|
||||
}
|
||||
|
||||
fn has_nonstandard_taproot_witness(witness: &input::WitnessFacts<'_>) -> bool {
|
||||
if witness.stack_items < 2 {
|
||||
return false;
|
||||
}
|
||||
|
||||
witness.leaf_version.is_none()
|
||||
|| witness.leaf_version == Some(LeafVersion::TapScript)
|
||||
&& witness.max_argument_bytes > MAX_WITNESS_STACK_ITEM_BYTES
|
||||
}
|
||||
|
||||
fn has_nonstandard_version(version: i32, height: u32) -> bool {
|
||||
let max = if height <= LAST_V2_POLICY_HEIGHT {
|
||||
2
|
||||
} else {
|
||||
3
|
||||
};
|
||||
!(1..=max).contains(&version)
|
||||
}
|
||||
|
||||
fn is_standard_unknown_witness(script: &Script) -> bool {
|
||||
script
|
||||
.witness_version()
|
||||
.is_some_and(|version| version != WitnessVersion::V0)
|
||||
}
|
||||
|
||||
fn has_too_many_bare_multisig_keys(sigops: SigOps) -> bool {
|
||||
u32::from(sigops) > MAX_STANDARD_BARE_MULTISIG_SIGOP_COST
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use bitcoin::{Amount, ScriptBuf, TxIn, Witness};
|
||||
use brk_types::{
|
||||
AddrBytes, Height, OutputType, P2ABytes, P2PK33Bytes, P2PK65Bytes, P2PKHBytes, P2SHBytes,
|
||||
P2TRBytes, P2WPKHBytes, P2WSHBytes, SigOps,
|
||||
};
|
||||
|
||||
use super::super::input;
|
||||
use super::{
|
||||
FIRST_V29_POLICY_HEIGHT, FIRST_V30_POLICY_HEIGHT, LAST_V2_POLICY_HEIGHT,
|
||||
has_nonstandard_p2wsh_witness, has_nonstandard_taproot_witness, has_nonstandard_version,
|
||||
has_nonstandard_witness, has_too_many_bare_multisig_keys,
|
||||
has_unconditionally_nonstandard_dust, is_dust, is_standard_unknown_witness,
|
||||
tracks_executed_legacy_sigops,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn tracks_executed_sigops_only_when_policy_uses_them() {
|
||||
assert!(!tracks_executed_legacy_sigops(Height::from(
|
||||
FIRST_V30_POLICY_HEIGHT - 1
|
||||
)));
|
||||
assert!(tracks_executed_legacy_sigops(Height::from(
|
||||
FIRST_V30_POLICY_HEIGHT
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_only_policy_transaction_versions() {
|
||||
assert!(has_nonstandard_version(0, LAST_V2_POLICY_HEIGHT));
|
||||
assert!(has_nonstandard_version(-1, LAST_V2_POLICY_HEIGHT));
|
||||
assert!(!has_nonstandard_version(1, LAST_V2_POLICY_HEIGHT));
|
||||
assert!(!has_nonstandard_version(2, LAST_V2_POLICY_HEIGHT));
|
||||
assert!(has_nonstandard_version(3, LAST_V2_POLICY_HEIGHT));
|
||||
assert!(!has_nonstandard_version(3, LAST_V2_POLICY_HEIGHT + 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn applies_ephemeral_dust_policy_from_v29() {
|
||||
assert!(!has_unconditionally_nonstandard_dust(
|
||||
FIRST_V29_POLICY_HEIGHT - 1,
|
||||
0
|
||||
));
|
||||
assert!(has_unconditionally_nonstandard_dust(
|
||||
FIRST_V29_POLICY_HEIGHT - 1,
|
||||
1
|
||||
));
|
||||
assert!(!has_unconditionally_nonstandard_dust(
|
||||
FIRST_V29_POLICY_HEIGHT,
|
||||
1
|
||||
));
|
||||
assert!(has_unconditionally_nonstandard_dust(
|
||||
FIRST_V29_POLICY_HEIGHT,
|
||||
2
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uses_exact_dust_thresholds_for_fixed_scripts() {
|
||||
let scripts = [
|
||||
AddrBytes::from(P2PK65Bytes::from(&[0; 65][..])).to_script_pubkey(),
|
||||
AddrBytes::from(P2PK33Bytes::from(&[0; 33][..])).to_script_pubkey(),
|
||||
AddrBytes::from(P2PKHBytes::from(&[0; 20][..])).to_script_pubkey(),
|
||||
AddrBytes::from(P2SHBytes::from(&[0; 20][..])).to_script_pubkey(),
|
||||
AddrBytes::from(P2WPKHBytes::from(&[0; 20][..])).to_script_pubkey(),
|
||||
AddrBytes::from(P2WSHBytes::from(&[0; 32][..])).to_script_pubkey(),
|
||||
AddrBytes::from(P2TRBytes::from(&[0; 32][..])).to_script_pubkey(),
|
||||
AddrBytes::from(P2ABytes::from(&[0; 2][..])).to_script_pubkey(),
|
||||
];
|
||||
let thresholds = [672, 576, 546, 540, 294, 330, 330, 240];
|
||||
|
||||
for ((output_type, script), threshold) in OutputType::ADDR_TYPES
|
||||
.into_iter()
|
||||
.zip(&scripts)
|
||||
.zip(thresholds)
|
||||
{
|
||||
assert_eq!(script.minimal_non_dust().to_sat(), threshold);
|
||||
assert!(is_dust(
|
||||
Amount::from_sat(threshold - 1),
|
||||
output_type,
|
||||
script
|
||||
));
|
||||
assert!(!is_dust(Amount::from_sat(threshold), output_type, script));
|
||||
}
|
||||
|
||||
let empty = ScriptBuf::new();
|
||||
assert_eq!(empty.minimal_non_dust().to_sat(), 471);
|
||||
assert!(is_dust(Amount::from_sat(470), OutputType::Empty, &empty));
|
||||
assert!(!is_dust(Amount::from_sat(471), OutputType::Empty, &empty));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_exact_dust_calculation_for_unknown_scripts() {
|
||||
let script = ScriptBuf::from_bytes(vec![0x61; 1_000]);
|
||||
let threshold = script.minimal_non_dust().to_sat();
|
||||
|
||||
assert!(is_dust(
|
||||
Amount::from_sat(threshold - 1),
|
||||
OutputType::Unknown,
|
||||
&script
|
||||
));
|
||||
assert!(!is_dust(
|
||||
Amount::from_sat(threshold),
|
||||
OutputType::Unknown,
|
||||
&script
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recognizes_future_witness_programs() {
|
||||
assert!(is_standard_unknown_witness(
|
||||
&ScriptBuf::from_hex(
|
||||
"52200000000000000000000000000000000000000000000000000000000000000000"
|
||||
)
|
||||
.unwrap()
|
||||
));
|
||||
assert!(!is_standard_unknown_witness(
|
||||
&ScriptBuf::from_hex(
|
||||
"00200000000000000000000000000000000000000000000000000000000000000000"
|
||||
)
|
||||
.unwrap()
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limits_standard_bare_multisig_to_three_keys() {
|
||||
assert!(!has_too_many_bare_multisig_keys(SigOps::new(12)));
|
||||
assert!(has_too_many_bare_multisig_keys(SigOps::new(16)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_p2a_witness_stuffing() {
|
||||
let input = TxIn {
|
||||
witness: Witness::from_slice(&[b"stuffing"]),
|
||||
..TxIn::default()
|
||||
};
|
||||
let facts = input::analyze(
|
||||
&input,
|
||||
OutputType::P2A,
|
||||
&mut crate::TxFeatureFlags::default(),
|
||||
);
|
||||
assert!(has_nonstandard_witness(OutputType::P2A, &facts,));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforces_witness_stack_item_limits() {
|
||||
let oversized = [0_u8; 81];
|
||||
let input = TxIn {
|
||||
witness: Witness::from_slice(&[oversized.as_slice(), [0x51].as_slice()]),
|
||||
..TxIn::default()
|
||||
};
|
||||
let facts = input::analyze(
|
||||
&input,
|
||||
OutputType::P2WSH,
|
||||
&mut crate::TxFeatureFlags::default(),
|
||||
);
|
||||
assert!(has_nonstandard_p2wsh_witness(&facts.witness));
|
||||
|
||||
let control_block = [0xc0_u8; 33];
|
||||
let input = TxIn {
|
||||
witness: Witness::from_slice(&[
|
||||
oversized.as_slice(),
|
||||
[0x51].as_slice(),
|
||||
control_block.as_slice(),
|
||||
]),
|
||||
..TxIn::default()
|
||||
};
|
||||
let facts = input::analyze(
|
||||
&input,
|
||||
OutputType::P2TR,
|
||||
&mut crate::TxFeatureFlags::default(),
|
||||
);
|
||||
assert!(has_nonstandard_taproot_witness(&facts.witness));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
use bitcoin::{Script, TxIn, TxOut, constants::WITNESS_SCALE_FACTOR};
|
||||
use brk_types::{OutputType, SigOps};
|
||||
|
||||
use crate::processor::txout::ProcessedOutput;
|
||||
|
||||
use super::input;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct Accumulator {
|
||||
legacy: usize,
|
||||
redeem: usize,
|
||||
witness: usize,
|
||||
executed_legacy: usize,
|
||||
track_executed_legacy: bool,
|
||||
}
|
||||
|
||||
impl Accumulator {
|
||||
pub(super) fn new(track_executed_legacy: bool) -> Self {
|
||||
Self {
|
||||
track_executed_legacy,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn scan_input(
|
||||
&mut self,
|
||||
prev_kind: OutputType,
|
||||
prev_legacy_sigops: SigOps,
|
||||
facts: &input::Facts<'_>,
|
||||
) {
|
||||
if !facts.script_sig.push_only {
|
||||
self.legacy = self.legacy.saturating_add(facts.script_sig.legacy_sigops);
|
||||
}
|
||||
|
||||
if self.track_executed_legacy {
|
||||
if !facts.script_sig.push_only {
|
||||
self.executed_legacy = self.executed_legacy.saturating_add(
|
||||
facts
|
||||
.script_sig
|
||||
.accurate_sigops
|
||||
.saturating_mul(WITNESS_SCALE_FACTOR),
|
||||
);
|
||||
}
|
||||
if prev_kind != OutputType::P2SH {
|
||||
self.executed_legacy = self
|
||||
.executed_legacy
|
||||
.saturating_add(u32::from(prev_legacy_sigops) as usize);
|
||||
}
|
||||
}
|
||||
|
||||
match prev_kind {
|
||||
OutputType::P2SH => {
|
||||
let Some(redeem_sigops) = facts.redeem_sigops() else {
|
||||
return;
|
||||
};
|
||||
self.redeem = self.redeem.saturating_add(redeem_sigops);
|
||||
if self.track_executed_legacy {
|
||||
self.executed_legacy = self
|
||||
.executed_legacy
|
||||
.saturating_add(redeem_sigops.saturating_mul(WITNESS_SCALE_FACTOR));
|
||||
}
|
||||
if !facts.script_sig.push_only {
|
||||
return;
|
||||
}
|
||||
if facts.redeem_is_p2wpkh() {
|
||||
self.witness = self.witness.saturating_add(1);
|
||||
} else if facts.redeem_is_p2wsh()
|
||||
&& let Some(last) = facts.witness.last
|
||||
{
|
||||
self.witness = self
|
||||
.witness
|
||||
.saturating_add(Script::from_bytes(last).count_sigops());
|
||||
}
|
||||
}
|
||||
OutputType::P2WPKH => self.witness = self.witness.saturating_add(1),
|
||||
OutputType::P2WSH => {
|
||||
if let Some(last) = facts.witness.last {
|
||||
self.witness = self
|
||||
.witness
|
||||
.saturating_add(Script::from_bytes(last).count_sigops());
|
||||
}
|
||||
}
|
||||
OutputType::P2TR => {}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn scan_coinbase_input(&mut self, input: &TxIn) {
|
||||
self.legacy = self
|
||||
.legacy
|
||||
.saturating_add(input.script_sig.count_sigops_legacy());
|
||||
}
|
||||
|
||||
pub(super) fn scan_output(&mut self, txout: &TxOut, output: &ProcessedOutput) {
|
||||
self.legacy = self
|
||||
.legacy
|
||||
.saturating_add(legacy_sigops_for_output(output, &txout.script_pubkey));
|
||||
}
|
||||
|
||||
pub(super) fn finish(self) -> ComputedSigOps {
|
||||
ComputedSigOps {
|
||||
total: SigOps::from(
|
||||
self.legacy
|
||||
.saturating_mul(WITNESS_SCALE_FACTOR)
|
||||
.saturating_add(self.redeem.saturating_mul(WITNESS_SCALE_FACTOR))
|
||||
.saturating_add(self.witness),
|
||||
),
|
||||
executed_legacy: SigOps::from(self.executed_legacy),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
pub(super) struct ComputedSigOps {
|
||||
pub(super) total: SigOps,
|
||||
pub(super) executed_legacy: SigOps,
|
||||
}
|
||||
|
||||
/// Legacy sigop count of a script_pubkey, dispatched on `OutputType`.
|
||||
/// Every variant except `OpReturn` and `Unknown` has a canonical shape
|
||||
/// recognised by `OutputType::from`'s exact byte-pattern matchers, so
|
||||
/// the legacy sigop count is fixed: P2PKH and P2PK both end in a
|
||||
/// single OP_CHECKSIG (1), P2MS contains one OP_CHECKMULTISIG counted
|
||||
/// as 20 in legacy mode, and P2SH/P2WPKH/P2WSH/P2TR/P2A/Empty contain
|
||||
/// no CHECKSIG-class opcodes outside their pushdata. `OpReturn`
|
||||
/// payloads can include 0xac/0xae bytes outside a push, and `Unknown`
|
||||
/// can be anything, so both fall back to a real script walk.
|
||||
#[inline]
|
||||
fn legacy_sigops_for_output(output: &ProcessedOutput, script_pubkey: &Script) -> usize {
|
||||
match output.output_type {
|
||||
OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65 => 1,
|
||||
OutputType::P2MS => 20,
|
||||
OutputType::P2SH
|
||||
| OutputType::P2WPKH
|
||||
| OutputType::P2WSH
|
||||
| OutputType::P2TR
|
||||
| OutputType::P2A
|
||||
| OutputType::Empty => 0,
|
||||
OutputType::OpReturn => output.op_return_legacy_sigops(),
|
||||
OutputType::Unknown => script_pubkey.count_sigops_legacy(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use bitcoin::{ScriptBuf, TxIn};
|
||||
use brk_types::{OutputType, SigOps};
|
||||
|
||||
use super::{Accumulator, input};
|
||||
use crate::TxFeatureFlags;
|
||||
|
||||
#[test]
|
||||
fn push_only_script_sig_keeps_prevout_executed_sigops() {
|
||||
let input = TxIn {
|
||||
script_sig: ScriptBuf::from_hex("00").unwrap(),
|
||||
..TxIn::default()
|
||||
};
|
||||
let facts = input::analyze(&input, OutputType::P2PKH, &mut TxFeatureFlags::default());
|
||||
let mut accumulator = Accumulator::new(true);
|
||||
|
||||
accumulator.scan_input(OutputType::P2PKH, SigOps::new(4), &facts);
|
||||
|
||||
assert_eq!(u32::from(accumulator.finish().executed_legacy), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counts_coinbase_legacy_script_sigops() {
|
||||
let input = TxIn {
|
||||
script_sig: ScriptBuf::from_hex("ac").unwrap(),
|
||||
..TxIn::default()
|
||||
};
|
||||
let mut accumulator = Accumulator::new(false);
|
||||
|
||||
accumulator.scan_coinbase_input(&input);
|
||||
|
||||
assert_eq!(u32::from(accumulator.finish().total), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counts_legacy_script_sigops_for_known_prevout_types() {
|
||||
let input = TxIn {
|
||||
script_sig: ScriptBuf::from_hex("ac").unwrap(),
|
||||
..TxIn::default()
|
||||
};
|
||||
let facts = input::analyze(&input, OutputType::P2TR, &mut TxFeatureFlags::default());
|
||||
let mut accumulator = Accumulator::new(false);
|
||||
|
||||
accumulator.scan_input(OutputType::P2TR, SigOps::ZERO, &facts);
|
||||
|
||||
assert_eq!(u32::from(accumulator.finish().total), 4);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
use bitcoin::Transaction;
|
||||
use brk_types::{TxIndex, Txid, TxidPrefix, Vout};
|
||||
|
||||
pub(crate) struct ComputedTx<'a> {
|
||||
pub(crate) tx_index: TxIndex,
|
||||
pub(crate) tx: &'a Transaction,
|
||||
pub(crate) txid: Txid,
|
||||
pub(crate) insert_txid_prefix: bool,
|
||||
pub(crate) base_size: u32,
|
||||
pub(crate) total_size: u32,
|
||||
input_offset: usize,
|
||||
output_offset: usize,
|
||||
}
|
||||
|
||||
impl ComputedTx<'_> {
|
||||
pub(super) fn new(
|
||||
tx_index: TxIndex,
|
||||
tx: &Transaction,
|
||||
txid: Txid,
|
||||
insert_txid_prefix: bool,
|
||||
base_size: u32,
|
||||
total_size: u32,
|
||||
) -> ComputedTx<'_> {
|
||||
ComputedTx {
|
||||
tx_index,
|
||||
tx,
|
||||
txid,
|
||||
insert_txid_prefix,
|
||||
base_size,
|
||||
total_size,
|
||||
input_offset: 0,
|
||||
output_offset: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn set_block_offsets(txs: &mut [Self]) {
|
||||
let mut input_offset = 0;
|
||||
let mut output_offset = 0;
|
||||
|
||||
for tx in txs {
|
||||
tx.input_offset = input_offset;
|
||||
tx.output_offset = output_offset;
|
||||
input_offset += tx.tx.input.len();
|
||||
output_offset += tx.tx.output.len();
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(super) fn inputs<'a, T>(&self, inputs: &'a [T]) -> &'a [T] {
|
||||
&inputs[self.input_offset..self.input_offset + self.tx.input.len()]
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(super) fn outputs<'a, T>(&self, outputs: &'a [T]) -> &'a [T] {
|
||||
&outputs[self.output_offset..self.output_offset + self.tx.output.len()]
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn txout_offset(&self, vout: Vout) -> usize {
|
||||
self.output_offset + usize::from(vout)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn is_coinbase(&self, block_first_tx_index: TxIndex) -> bool {
|
||||
self.tx_index == block_first_tx_index
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn txid_prefix(&self) -> TxidPrefix {
|
||||
TxidPrefix::from(&self.txid)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn is_segwit(&self) -> bool {
|
||||
self.base_size != self.total_size
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn weight(&self) -> bitcoin::Weight {
|
||||
brk_types::Weight::from_sizes(self.base_size, self.total_size).into()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
mod analysis;
|
||||
mod computed;
|
||||
|
||||
use brk_error::{Error, Result};
|
||||
use brk_store::Store;
|
||||
use brk_types::{Height, StoredBool, TxIndex, TxVersion, Txid, TxidPrefix};
|
||||
use rayon::prelude::*;
|
||||
use tracing::error;
|
||||
use vecdb::{AnyVec, WritableVec, likely, unlikely};
|
||||
|
||||
use crate::constants::DUPLICATE_TXIDS;
|
||||
use crate::{TransactionCounts, TransactionFeaturesVecs, TxMetadataVecs};
|
||||
|
||||
pub(super) use computed::ComputedTx;
|
||||
|
||||
use self::analysis::TransactionAnalysis;
|
||||
use super::{
|
||||
BlockProcessor,
|
||||
txin::{self, InputSource},
|
||||
txout::{self, BlockAddresses, ProcessedOutput},
|
||||
};
|
||||
|
||||
impl<'a> BlockProcessor<'a> {
|
||||
pub(crate) fn compute_txids(&self) -> Result<Vec<ComputedTx<'a>>> {
|
||||
let will_check_collisions = self.check_collisions;
|
||||
let base_tx_index = self.lengths.tx_index;
|
||||
|
||||
let mut txs = self
|
||||
.block
|
||||
.txdata
|
||||
.par_iter()
|
||||
.enumerate()
|
||||
.map(|(index, tx)| {
|
||||
let (btc_txid, base_size, total_size) = self.block.compute_tx_id_and_sizes(index);
|
||||
let txid = Txid::from(btc_txid);
|
||||
let tx_index = base_tx_index + TxIndex::from(index);
|
||||
|
||||
let insert_txid_prefix = if likely(!will_check_collisions) {
|
||||
true
|
||||
} else {
|
||||
let txid_prefix = TxidPrefix::from(&txid);
|
||||
let prev_tx_index = self
|
||||
.stores
|
||||
.txid_prefix_to_tx_index
|
||||
.get(&txid_prefix)?
|
||||
.map(|value| *value);
|
||||
|
||||
if let Some(prev_tx_index) = prev_tx_index {
|
||||
self.validate_txid_collision(tx_index, prev_tx_index)?;
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
};
|
||||
|
||||
Ok(ComputedTx::new(
|
||||
tx_index,
|
||||
tx,
|
||||
txid,
|
||||
insert_txid_prefix,
|
||||
base_size,
|
||||
total_size,
|
||||
))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
ComputedTx::set_block_offsets(&mut txs);
|
||||
Ok(txs)
|
||||
}
|
||||
|
||||
fn validate_txid_collision(&self, tx_index: TxIndex, prev_tx_index: TxIndex) -> Result<()> {
|
||||
if tx_index == prev_tx_index {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let len = self.vecs.transactions.txid.len();
|
||||
let prev_txid = self
|
||||
.vecs
|
||||
.transactions
|
||||
.txid
|
||||
.get_pushed_or_read(prev_tx_index, &self.readers.txid)
|
||||
.ok_or(Error::Internal("Missing txid for tx_index"))
|
||||
.inspect_err(|_| {
|
||||
error!(?tx_index, len, "Missing txid for tx_index");
|
||||
})?;
|
||||
|
||||
if unlikely(!DUPLICATE_TXIDS.contains(&prev_txid)) {
|
||||
error!(
|
||||
height = ?self.height,
|
||||
?tx_index,
|
||||
?prev_txid,
|
||||
?prev_tx_index,
|
||||
"Unexpected TXID collision"
|
||||
);
|
||||
return Err(Error::Internal("Unexpected TXID collision"));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Analyzes transactions, then finalizes outputs/inputs in parallel with metadata storage.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn analyze_and_finalize_transactions(
|
||||
&mut self,
|
||||
txs: Vec<ComputedTx>,
|
||||
mut txouts: Vec<ProcessedOutput>,
|
||||
txins: Vec<InputSource>,
|
||||
addresses: &mut BlockAddresses,
|
||||
) -> Result<()> {
|
||||
let transaction_analyses = self.analyze_transactions(&txs, &txins, &txouts);
|
||||
let lengths = &mut *self.lengths;
|
||||
let base_tx_index = lengths.tx_index;
|
||||
let base_txin_index = lengths.txin_index;
|
||||
let transactions = &self.block.txdata;
|
||||
|
||||
// Split transactions vecs: finalize needs first_txout_index/first_txin_index, metadata needs the rest
|
||||
let (first_txout_index, first_txin_index, mut tx_metadata) =
|
||||
self.vecs.transactions.split_for_finalize();
|
||||
|
||||
let outputs = &mut self.vecs.outputs;
|
||||
let inputs = &mut self.vecs.inputs;
|
||||
let addrs = &mut self.vecs.addrs;
|
||||
let scripts = &mut self.vecs.scripts;
|
||||
let op_return = &mut self.vecs.op_return;
|
||||
let transaction_features = &mut self.vecs.transaction_features;
|
||||
let height = self.height;
|
||||
|
||||
let addr_hash_stores = &mut self.stores.addr_type_to_addr_hash_to_addr_index;
|
||||
let addr_tx_index_stores = &mut self.stores.addr_type_to_addr_index_and_tx_index;
|
||||
let addr_outpoint_stores = &mut self.stores.addr_type_to_addr_index_and_unspent_outpoint;
|
||||
let txid_prefix_store = &mut self.stores.txid_prefix_to_tx_index;
|
||||
|
||||
let (finalize_result, metadata_result) = rayon::join(
|
||||
|| -> Result<()> {
|
||||
txout::finalize_outputs(
|
||||
transactions,
|
||||
base_tx_index,
|
||||
lengths,
|
||||
first_txout_index,
|
||||
outputs,
|
||||
addrs,
|
||||
scripts,
|
||||
op_return,
|
||||
addr_hash_stores,
|
||||
addr_tx_index_stores,
|
||||
addr_outpoint_stores,
|
||||
&mut txouts,
|
||||
addresses,
|
||||
)?;
|
||||
txin::finalize_inputs(
|
||||
transactions,
|
||||
base_tx_index,
|
||||
base_txin_index,
|
||||
first_txin_index,
|
||||
inputs,
|
||||
addr_tx_index_stores,
|
||||
addr_outpoint_stores,
|
||||
txins,
|
||||
&txouts,
|
||||
)
|
||||
},
|
||||
|| {
|
||||
store_tx_metadata(
|
||||
height,
|
||||
txs,
|
||||
transaction_analyses,
|
||||
txid_prefix_store,
|
||||
&mut tx_metadata,
|
||||
transaction_features,
|
||||
)
|
||||
},
|
||||
);
|
||||
|
||||
finalize_result?;
|
||||
metadata_result?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn store_tx_metadata(
|
||||
height: Height,
|
||||
txs: Vec<ComputedTx>,
|
||||
transaction_analyses: Vec<TransactionAnalysis>,
|
||||
store: &mut Store<TxidPrefix, TxIndex>,
|
||||
md: &mut TxMetadataVecs<'_>,
|
||||
features: &mut TransactionFeaturesVecs,
|
||||
) -> Result<()> {
|
||||
debug_assert_eq!(txs.len(), transaction_analyses.len());
|
||||
let mut counts = TransactionCounts::default();
|
||||
for (ct, analysis) in txs.into_iter().zip(transaction_analyses) {
|
||||
if likely(ct.insert_txid_prefix) {
|
||||
store.insert(ct.txid_prefix(), ct.tx_index);
|
||||
}
|
||||
let tx_version = TxVersion::from(ct.tx.version);
|
||||
md.tx_version.checked_push(ct.tx_index, tx_version)?;
|
||||
md.txid.checked_push(ct.tx_index, ct.txid)?;
|
||||
md.raw_locktime
|
||||
.checked_push(ct.tx_index, ct.tx.lock_time.into())?;
|
||||
md.weight.checked_push(ct.tx_index, ct.weight().into())?;
|
||||
md.total_size
|
||||
.checked_push(ct.tx_index, ct.total_size.into())?;
|
||||
md.total_sigop_cost
|
||||
.checked_push(ct.tx_index, analysis.total_sigop_cost)?;
|
||||
md.is_explicitly_rbf
|
||||
.checked_push(ct.tx_index, StoredBool::from(analysis.explicitly_rbf))?;
|
||||
counts.add_base(
|
||||
ct.tx.input.len(),
|
||||
ct.tx.output.len(),
|
||||
tx_version,
|
||||
analysis.explicitly_rbf,
|
||||
);
|
||||
features.push_and_count(analysis.features, &mut counts);
|
||||
}
|
||||
features.count.checked_push(height, counts)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
use brk_error::{Error, Result};
|
||||
use brk_store::Store;
|
||||
use brk_types::{SigOps, StoredBool, TxIndex, Txid, TxidPrefix};
|
||||
use rayon::prelude::*;
|
||||
use tracing::error;
|
||||
use vecdb::{AnyVec, WritableVec, likely};
|
||||
|
||||
use crate::TxMetadataVecs;
|
||||
use crate::constants::DUPLICATE_TXIDS;
|
||||
|
||||
use super::{BlockProcessor, ComputedTx};
|
||||
|
||||
impl<'a> BlockProcessor<'a> {
|
||||
pub fn compute_txids(&self) -> Result<Vec<ComputedTx<'a>>> {
|
||||
let will_check_collisions = self.check_collisions;
|
||||
let base_tx_index = self.lengths.tx_index;
|
||||
|
||||
self.block
|
||||
.txdata
|
||||
.par_iter()
|
||||
.enumerate()
|
||||
.map(|(index, tx)| {
|
||||
let (btc_txid, base_size, total_size) = self.block.compute_tx_id_and_sizes(index);
|
||||
let txid = Txid::from(btc_txid);
|
||||
let txid_prefix = TxidPrefix::from(&txid);
|
||||
|
||||
let prev_tx_index_opt = if will_check_collisions {
|
||||
self.stores
|
||||
.txid_prefix_to_tx_index
|
||||
.get(&txid_prefix)?
|
||||
.map(|v| *v)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(ComputedTx {
|
||||
tx_index: base_tx_index + TxIndex::from(index),
|
||||
tx,
|
||||
txid,
|
||||
txid_prefix,
|
||||
prev_tx_index_opt,
|
||||
base_size,
|
||||
total_size,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Only for known duplicate TXIDs (BIP-30).
|
||||
pub fn check_txid_collisions(&self, txs: &[ComputedTx]) -> Result<()> {
|
||||
if likely(!self.check_collisions) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for ct in txs.iter() {
|
||||
let Some(prev_tx_index) = ct.prev_tx_index_opt else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if ct.tx_index == prev_tx_index {
|
||||
continue;
|
||||
}
|
||||
|
||||
let len = self.vecs.transactions.txid.len();
|
||||
let prev_txid = self
|
||||
.vecs
|
||||
.transactions
|
||||
.txid
|
||||
.get_pushed_or_read(prev_tx_index, &self.readers.txid)
|
||||
.ok_or(Error::Internal("Missing txid for tx_index"))
|
||||
.inspect_err(|_| {
|
||||
error!(tx_index = ?ct.tx_index, len, "Missing txid for tx_index");
|
||||
})?;
|
||||
|
||||
let is_dup = DUPLICATE_TXIDS.contains(&prev_txid);
|
||||
|
||||
if !is_dup {
|
||||
error!(
|
||||
height = ?self.height, tx_index = ?ct.tx_index,
|
||||
?prev_txid, ?prev_tx_index,
|
||||
"Unexpected TXID collision"
|
||||
);
|
||||
return Err(Error::Internal("Unexpected TXID collision"));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn store_tx_metadata(
|
||||
txs: Vec<ComputedTx>,
|
||||
sigops: Vec<SigOps>,
|
||||
store: &mut Store<TxidPrefix, TxIndex>,
|
||||
md: &mut TxMetadataVecs<'_>,
|
||||
) -> Result<()> {
|
||||
debug_assert_eq!(txs.len(), sigops.len());
|
||||
for (ct, sigops) in txs.into_iter().zip(sigops) {
|
||||
if ct.prev_tx_index_opt.is_none() {
|
||||
store.insert(ct.txid_prefix, ct.tx_index);
|
||||
}
|
||||
md.tx_version
|
||||
.checked_push(ct.tx_index, ct.tx.version.into())?;
|
||||
md.txid.checked_push(ct.tx_index, ct.txid)?;
|
||||
md.raw_locktime
|
||||
.checked_push(ct.tx_index, ct.tx.lock_time.into())?;
|
||||
md.base_size
|
||||
.checked_push(ct.tx_index, ct.base_size.into())?;
|
||||
md.total_size
|
||||
.checked_push(ct.tx_index, ct.total_size.into())?;
|
||||
md.total_sigop_cost.checked_push(ct.tx_index, sigops)?;
|
||||
md.is_explicitly_rbf
|
||||
.checked_push(ct.tx_index, StoredBool::from(ct.tx.is_explicitly_rbf()))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,222 +0,0 @@
|
||||
use brk_cohort::ByAddrType;
|
||||
use brk_error::{Error, Result};
|
||||
use brk_store::Store;
|
||||
use brk_types::{
|
||||
AddrIndexOutPoint, AddrIndexTxIndex, OutPoint, OutputType, TxInIndex, TxIndex, Txid,
|
||||
TxidPrefix, TypeIndex, Unit, Vin, Vout,
|
||||
};
|
||||
use rayon::prelude::*;
|
||||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
use tracing::error;
|
||||
use vecdb::{PcoVec, WritableVec};
|
||||
|
||||
use super::{BlockProcessor, ComputedTx, InputSource, SameBlockOutputInfo};
|
||||
use crate::InputsVecs;
|
||||
|
||||
impl<'a> BlockProcessor<'a> {
|
||||
pub fn process_inputs(
|
||||
&self,
|
||||
txs: &[ComputedTx],
|
||||
txid_prefix_to_tx_index: &mut FxHashMap<TxidPrefix, TxIndex>,
|
||||
) -> Result<Vec<(TxInIndex, InputSource)>> {
|
||||
txid_prefix_to_tx_index.clear();
|
||||
txid_prefix_to_tx_index.extend(txs.iter().map(|ct| (ct.txid_prefix, ct.tx_index)));
|
||||
|
||||
let base_tx_index = self.lengths.tx_index;
|
||||
let base_txin_index = self.lengths.txin_index;
|
||||
|
||||
let total_inputs: usize = self.block.txdata.iter().map(|tx| tx.input.len()).sum();
|
||||
let mut items = Vec::with_capacity(total_inputs);
|
||||
for (index, tx) in self.block.txdata.iter().enumerate() {
|
||||
for (vin, txin) in tx.input.iter().enumerate() {
|
||||
items.push((TxIndex::from(index), Vin::from(vin), txin, tx));
|
||||
}
|
||||
}
|
||||
|
||||
let txid_prefix_to_tx_index = &*txid_prefix_to_tx_index;
|
||||
|
||||
let txins = items
|
||||
.into_par_iter()
|
||||
.enumerate()
|
||||
.map(
|
||||
|(block_txin_index, (block_tx_index, vin, txin, tx))| -> Result<(TxInIndex, InputSource)> {
|
||||
let tx_index = base_tx_index + block_tx_index;
|
||||
let txin_index = base_txin_index + TxInIndex::from(block_txin_index);
|
||||
|
||||
if tx.is_coinbase() {
|
||||
return Ok((
|
||||
txin_index,
|
||||
InputSource::SameBlock {
|
||||
tx_index,
|
||||
vin,
|
||||
outpoint: OutPoint::COINBASE,
|
||||
},
|
||||
));
|
||||
}
|
||||
|
||||
let outpoint = txin.previous_output;
|
||||
let txid = Txid::from(outpoint.txid);
|
||||
let txid_prefix = TxidPrefix::from(&txid);
|
||||
let vout = Vout::from(outpoint.vout);
|
||||
|
||||
if let Some(&same_block_tx_index) = txid_prefix_to_tx_index
|
||||
.get(&txid_prefix) {
|
||||
let outpoint = OutPoint::new(same_block_tx_index, vout);
|
||||
return Ok((
|
||||
txin_index,
|
||||
InputSource::SameBlock {
|
||||
tx_index,
|
||||
vin,
|
||||
outpoint,
|
||||
},
|
||||
));
|
||||
}
|
||||
|
||||
let store_result = self
|
||||
.stores
|
||||
.txid_prefix_to_tx_index
|
||||
.get(&txid_prefix)?
|
||||
.map(|v| *v);
|
||||
|
||||
let prev_tx_index = match store_result {
|
||||
Some(tx_index) if tx_index < self.lengths.tx_index => tx_index,
|
||||
_ => {
|
||||
error!(
|
||||
"UnknownTxid: txid={}, prefix={:?}, store_result={:?}, current_tx_index={:?}",
|
||||
txid, txid_prefix, store_result, self.lengths.tx_index
|
||||
);
|
||||
return Err(Error::UnknownTxid);
|
||||
}
|
||||
};
|
||||
|
||||
let txout_index = self
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txout_index
|
||||
.get_pushed_or_read(prev_tx_index, &self.readers.tx_index_to_first_txout_index)
|
||||
.ok_or(Error::Internal("Missing txout_index"))?
|
||||
+ vout;
|
||||
|
||||
let outpoint = OutPoint::new(prev_tx_index, vout);
|
||||
|
||||
let output_type = self
|
||||
.vecs
|
||||
.outputs
|
||||
.output_type
|
||||
.get_pushed_or_read(txout_index, &self.readers.txout_index_to_output_type)
|
||||
.ok_or(Error::Internal("Missing output_type"))?;
|
||||
|
||||
let type_index = self
|
||||
.vecs
|
||||
.outputs
|
||||
.type_index
|
||||
.get_pushed_or_read(txout_index, &self.readers.txout_index_to_type_index)
|
||||
.ok_or(Error::Internal("Missing type_index"))?;
|
||||
|
||||
Ok((
|
||||
txin_index,
|
||||
InputSource::PreviousBlock {
|
||||
vin,
|
||||
tx_index,
|
||||
outpoint,
|
||||
output_type,
|
||||
type_index,
|
||||
},
|
||||
))
|
||||
},
|
||||
)
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
Ok(txins)
|
||||
}
|
||||
|
||||
pub fn collect_same_block_spent_outpoints(
|
||||
txins: &[(TxInIndex, InputSource)],
|
||||
out: &mut FxHashSet<OutPoint>,
|
||||
) {
|
||||
out.clear();
|
||||
out.extend(
|
||||
txins
|
||||
.iter()
|
||||
.filter_map(|(_, input_source)| match input_source {
|
||||
InputSource::SameBlock { outpoint, .. } if !outpoint.is_coinbase() => {
|
||||
Some(*outpoint)
|
||||
}
|
||||
_ => None,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn finalize_inputs(
|
||||
first_txin_index: &mut PcoVec<TxIndex, TxInIndex>,
|
||||
inputs: &mut InputsVecs,
|
||||
addr_tx_index_stores: &mut ByAddrType<Store<AddrIndexTxIndex, Unit>>,
|
||||
addr_outpoint_stores: &mut ByAddrType<Store<AddrIndexOutPoint, Unit>>,
|
||||
txins: Vec<(TxInIndex, InputSource)>,
|
||||
same_block_output_info: &mut FxHashMap<OutPoint, SameBlockOutputInfo>,
|
||||
) -> Result<()> {
|
||||
for (txin_index, input_source) in txins {
|
||||
let (vin, tx_index, outpoint, output_type, type_index) = match input_source {
|
||||
InputSource::PreviousBlock {
|
||||
vin,
|
||||
tx_index,
|
||||
outpoint,
|
||||
output_type,
|
||||
type_index,
|
||||
} => (vin, tx_index, outpoint, output_type, type_index),
|
||||
InputSource::SameBlock {
|
||||
tx_index,
|
||||
vin,
|
||||
outpoint,
|
||||
} => {
|
||||
if outpoint.is_coinbase() {
|
||||
(
|
||||
vin,
|
||||
tx_index,
|
||||
outpoint,
|
||||
OutputType::Unknown,
|
||||
TypeIndex::COINBASE,
|
||||
)
|
||||
} else {
|
||||
let info = same_block_output_info
|
||||
.remove(&outpoint)
|
||||
.ok_or(Error::Internal("Same-block output not found"))
|
||||
.inspect_err(|_| {
|
||||
error!(
|
||||
?outpoint,
|
||||
remaining = same_block_output_info.len(),
|
||||
"Same-block output not found"
|
||||
);
|
||||
})?;
|
||||
(vin, tx_index, outpoint, info.output_type, info.type_index)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if vin.is_zero() {
|
||||
first_txin_index.checked_push(tx_index, txin_index)?;
|
||||
}
|
||||
|
||||
inputs.tx_index.checked_push(txin_index, tx_index)?;
|
||||
inputs.outpoint.checked_push(txin_index, outpoint)?;
|
||||
inputs.output_type.checked_push(txin_index, output_type)?;
|
||||
inputs.type_index.checked_push(txin_index, type_index)?;
|
||||
|
||||
if !output_type.is_addr() {
|
||||
continue;
|
||||
}
|
||||
let addr_type = output_type;
|
||||
let addr_index = type_index;
|
||||
|
||||
addr_tx_index_stores
|
||||
.get_mut_unwrap(addr_type)
|
||||
.insert(AddrIndexTxIndex::from((addr_index, tx_index)), Unit);
|
||||
|
||||
addr_outpoint_stores
|
||||
.get_mut_unwrap(addr_type)
|
||||
.remove(AddrIndexOutPoint::from((addr_index, outpoint)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
mod resolver;
|
||||
mod source;
|
||||
|
||||
pub(crate) use resolver::InputResolver;
|
||||
pub(super) use source::InputSource;
|
||||
|
||||
use bitcoin::Transaction;
|
||||
use brk_cohort::ByAddrType;
|
||||
use brk_error::Result;
|
||||
use brk_store::Store;
|
||||
use brk_types::{
|
||||
AddrIndexOutPoint, AddrIndexTxIndex, OutPoint, OutputType, TxInIndex, TxIndex, TypeIndex, Unit,
|
||||
Vin,
|
||||
};
|
||||
use vecdb::{PcoVec, WritableVec, unlikely};
|
||||
|
||||
use super::{BlockProcessor, transaction::ComputedTx, txout::ProcessedOutput};
|
||||
use crate::InputsVecs;
|
||||
|
||||
impl<'a> BlockProcessor<'a> {
|
||||
pub(crate) fn process_inputs(
|
||||
&self,
|
||||
txs: &[ComputedTx],
|
||||
resolver: &mut InputResolver,
|
||||
) -> Result<Vec<InputSource>> {
|
||||
resolver.resolve(self, txs)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) fn finalize_inputs(
|
||||
transactions: &[Transaction],
|
||||
base_tx_index: TxIndex,
|
||||
base_txin_index: TxInIndex,
|
||||
first_txin_index: &mut PcoVec<TxIndex, TxInIndex>,
|
||||
inputs: &mut InputsVecs,
|
||||
addr_tx_index_stores: &mut ByAddrType<Store<AddrIndexTxIndex, Unit>>,
|
||||
addr_outpoint_stores: &mut ByAddrType<Store<AddrIndexOutPoint, Unit>>,
|
||||
txins: Vec<InputSource>,
|
||||
txouts: &[ProcessedOutput],
|
||||
) -> Result<()> {
|
||||
let mut input_offset = 0;
|
||||
for (block_tx_index, tx) in transactions.iter().enumerate() {
|
||||
let tx_index = base_tx_index + TxIndex::from(block_tx_index);
|
||||
let next_input_offset = input_offset + tx.input.len();
|
||||
|
||||
for (vin, input_source) in txins[input_offset..next_input_offset].iter().enumerate() {
|
||||
let block_txin_index = input_offset + vin;
|
||||
let txin_index = base_txin_index + TxInIndex::from(block_txin_index);
|
||||
let vin = Vin::from(vin);
|
||||
let (outpoint, output_type, type_index) = match input_source {
|
||||
InputSource::PreviousBlock {
|
||||
outpoint,
|
||||
output_type,
|
||||
legacy_sigops: _,
|
||||
type_index,
|
||||
} => (*outpoint, *output_type, *type_index),
|
||||
InputSource::Coinbase => {
|
||||
(OutPoint::COINBASE, OutputType::Unknown, TypeIndex::COINBASE)
|
||||
}
|
||||
InputSource::SameBlock {
|
||||
outpoint,
|
||||
txout_offset,
|
||||
} => {
|
||||
let output = &txouts[*txout_offset];
|
||||
(*outpoint, output.output_type, output.resolved_type_index())
|
||||
}
|
||||
};
|
||||
|
||||
if vin.is_zero() {
|
||||
first_txin_index.checked_push(tx_index, txin_index)?;
|
||||
}
|
||||
|
||||
inputs.tx_index.checked_push(txin_index, tx_index)?;
|
||||
inputs.outpoint.checked_push(txin_index, outpoint)?;
|
||||
inputs.output_type.checked_push(txin_index, output_type)?;
|
||||
inputs.type_index.checked_push(txin_index, type_index)?;
|
||||
|
||||
if unlikely(!output_type.is_addr()) {
|
||||
continue;
|
||||
}
|
||||
let addr_type = output_type;
|
||||
let addr_index = type_index;
|
||||
|
||||
addr_tx_index_stores
|
||||
.get_mut_unwrap(addr_type)
|
||||
.insert(AddrIndexTxIndex::from((addr_index, tx_index)), Unit);
|
||||
|
||||
addr_outpoint_stores
|
||||
.get_mut_unwrap(addr_type)
|
||||
.remove(AddrIndexOutPoint::from((addr_index, outpoint)));
|
||||
}
|
||||
|
||||
input_offset = next_input_offset;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,329 @@
|
||||
use std::collections::hash_map::Entry;
|
||||
|
||||
use brk_error::{Error, Result};
|
||||
use brk_types::{
|
||||
OutPoint, OutputType, SigOps, TxIndex, TxOutIndex, Txid, TxidPrefix, TypeIndex, Vout,
|
||||
};
|
||||
use rayon::prelude::*;
|
||||
use rustc_hash::FxHashMap;
|
||||
use tracing::error;
|
||||
use vecdb::unlikely;
|
||||
|
||||
use super::InputSource;
|
||||
use crate::processor::{BlockProcessor, transaction::ComputedTx};
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct InputResolver {
|
||||
same_block_transactions: FxHashMap<TxidPrefix, TxIndex>,
|
||||
previous_parent_indexes: FxHashMap<TxidPrefix, usize>,
|
||||
previous_parents: Vec<PreviousParent>,
|
||||
inputs: Vec<UnresolvedInput>,
|
||||
reads: ReadBatch,
|
||||
}
|
||||
|
||||
impl InputResolver {
|
||||
pub(crate) fn resolve(
|
||||
&mut self,
|
||||
processor: &BlockProcessor<'_>,
|
||||
txs: &[ComputedTx<'_>],
|
||||
) -> Result<Vec<InputSource>> {
|
||||
self.prepare(txs, processor.lengths.tx_index);
|
||||
self.reads.resolve(
|
||||
processor,
|
||||
&self.previous_parents,
|
||||
&self.inputs,
|
||||
processor.lengths.tx_index,
|
||||
)?;
|
||||
|
||||
let tracks_executed_legacy_sigops = processor.tracks_executed_legacy_sigops();
|
||||
let reads = &self.reads;
|
||||
|
||||
self.inputs
|
||||
.par_iter()
|
||||
.enumerate()
|
||||
.map(|(input_index, input)| match *input {
|
||||
UnresolvedInput::Coinbase => Ok(InputSource::Coinbase),
|
||||
UnresolvedInput::SameBlock {
|
||||
outpoint,
|
||||
txout_offset,
|
||||
} => Ok(InputSource::SameBlock {
|
||||
outpoint,
|
||||
txout_offset,
|
||||
}),
|
||||
UnresolvedInput::PreviousBlock { parent_index, vout } => {
|
||||
let parent = reads.parent(parent_index);
|
||||
let outpoint = OutPoint::new(parent.tx_index, vout);
|
||||
let (output_type, type_index) = reads.output(input_index);
|
||||
|
||||
let legacy_sigops = if tracks_executed_legacy_sigops {
|
||||
processor
|
||||
.vecs
|
||||
.scripts
|
||||
.legacy_sigops(output_type, type_index, &processor.readers.scripts)
|
||||
.ok_or(Error::Internal("Missing legacy_sigops"))?
|
||||
} else {
|
||||
SigOps::ZERO
|
||||
};
|
||||
|
||||
Ok(InputSource::PreviousBlock {
|
||||
outpoint,
|
||||
output_type,
|
||||
legacy_sigops,
|
||||
type_index,
|
||||
})
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn prepare(&mut self, txs: &[ComputedTx<'_>], block_first_tx_index: TxIndex) {
|
||||
self.same_block_transactions.clear();
|
||||
self.previous_parent_indexes.clear();
|
||||
self.previous_parents.clear();
|
||||
self.inputs.clear();
|
||||
|
||||
self.same_block_transactions.reserve(txs.len());
|
||||
self.same_block_transactions
|
||||
.extend(txs.iter().map(|tx| (tx.txid_prefix(), tx.tx_index)));
|
||||
|
||||
let total_inputs = txs.iter().map(|tx| tx.tx.input.len()).sum();
|
||||
self.inputs.reserve(total_inputs);
|
||||
|
||||
for tx in txs {
|
||||
for txin in &tx.tx.input {
|
||||
let previous_output = &txin.previous_output;
|
||||
if unlikely(previous_output.is_null()) {
|
||||
self.inputs.push(UnresolvedInput::Coinbase);
|
||||
continue;
|
||||
}
|
||||
|
||||
let txid = *<&Txid>::from(&previous_output.txid);
|
||||
let txid_prefix = TxidPrefix::from(&txid);
|
||||
let vout = Vout::from(previous_output.vout);
|
||||
|
||||
if let Some(&tx_index) = self.same_block_transactions.get(&txid_prefix) {
|
||||
let block_tx_index = usize::from(tx_index) - usize::from(block_first_tx_index);
|
||||
self.inputs.push(UnresolvedInput::SameBlock {
|
||||
outpoint: OutPoint::new(tx_index, vout),
|
||||
txout_offset: txs[block_tx_index].txout_offset(vout),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
let parent_index = match self.previous_parent_indexes.entry(txid_prefix) {
|
||||
Entry::Occupied(entry) => *entry.get(),
|
||||
Entry::Vacant(entry) => {
|
||||
let parent_index = self.previous_parents.len();
|
||||
entry.insert(parent_index);
|
||||
self.previous_parents
|
||||
.push(PreviousParent { txid, txid_prefix });
|
||||
parent_index
|
||||
}
|
||||
};
|
||||
|
||||
self.inputs
|
||||
.push(UnresolvedInput::PreviousBlock { parent_index, vout });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct PreviousParent {
|
||||
txid: Txid,
|
||||
txid_prefix: TxidPrefix,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ParentRead {
|
||||
original_index: usize,
|
||||
tx_index: TxIndex,
|
||||
first_txout_index: TxOutIndex,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct OutputRead {
|
||||
input_index: usize,
|
||||
txout_index: TxOutIndex,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ReadBatch {
|
||||
parents: Vec<ParentRead>,
|
||||
parent_positions: Vec<usize>,
|
||||
outputs: Vec<OutputRead>,
|
||||
output_positions: Vec<usize>,
|
||||
output_types: Vec<OutputType>,
|
||||
type_indices: Vec<TypeIndex>,
|
||||
}
|
||||
|
||||
impl ReadBatch {
|
||||
fn resolve(
|
||||
&mut self,
|
||||
processor: &BlockProcessor<'_>,
|
||||
previous_parents: &[PreviousParent],
|
||||
inputs: &[UnresolvedInput],
|
||||
current_tx_index: TxIndex,
|
||||
) -> Result<()> {
|
||||
self.resolve_parents(processor, previous_parents, current_tx_index)?;
|
||||
self.prepare_outputs(inputs);
|
||||
self.read_outputs(processor)
|
||||
}
|
||||
|
||||
fn resolve_parents(
|
||||
&mut self,
|
||||
processor: &BlockProcessor<'_>,
|
||||
previous_parents: &[PreviousParent],
|
||||
current_tx_index: TxIndex,
|
||||
) -> Result<()> {
|
||||
self.parents.clear();
|
||||
self.parents.extend(
|
||||
(0..previous_parents.len()).map(|original_index| ParentRead {
|
||||
original_index,
|
||||
tx_index: TxIndex::default(),
|
||||
first_txout_index: TxOutIndex::default(),
|
||||
}),
|
||||
);
|
||||
|
||||
self.parents
|
||||
.par_iter_mut()
|
||||
.try_for_each(|read| {
|
||||
let parent = &previous_parents[read.original_index];
|
||||
let store_result = processor
|
||||
.stores
|
||||
.txid_prefix_to_tx_index
|
||||
.get(&parent.txid_prefix)?
|
||||
.map(|value| *value);
|
||||
|
||||
let tx_index = match store_result {
|
||||
Some(tx_index) if tx_index < current_tx_index => tx_index,
|
||||
_ => {
|
||||
error!(
|
||||
"UnknownTxid: txid={}, prefix={:?}, store_result={:?}, current_tx_index={:?}",
|
||||
parent.txid,
|
||||
parent.txid_prefix,
|
||||
store_result,
|
||||
current_tx_index
|
||||
);
|
||||
return Err(Error::UnknownTxid);
|
||||
}
|
||||
};
|
||||
|
||||
read.tx_index = tx_index;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
self.parents.sort_unstable_by_key(|read| read.tx_index);
|
||||
self.parent_positions.clear();
|
||||
self.parent_positions.resize(self.parents.len(), 0);
|
||||
|
||||
for (position, read) in self.parents.iter_mut().enumerate() {
|
||||
self.parent_positions[read.original_index] = position;
|
||||
read.first_txout_index = processor
|
||||
.vecs
|
||||
.transactions
|
||||
.first_txout_index
|
||||
.get_pushed_or_read(
|
||||
read.tx_index,
|
||||
&processor.readers.tx_index_to_first_txout_index,
|
||||
)
|
||||
.ok_or(Error::Internal("Missing txout_index"))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn prepare_outputs(&mut self, inputs: &[UnresolvedInput]) {
|
||||
self.outputs.clear();
|
||||
self.outputs.reserve(inputs.len());
|
||||
|
||||
for (input_index, input) in inputs.iter().enumerate() {
|
||||
if let UnresolvedInput::PreviousBlock { parent_index, vout } = *input {
|
||||
let parent = self.parent(parent_index);
|
||||
self.outputs.push(OutputRead {
|
||||
input_index,
|
||||
txout_index: parent.first_txout_index + vout,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
self.outputs.sort_unstable_by_key(|read| read.txout_index);
|
||||
self.output_positions.clear();
|
||||
self.output_positions.resize(inputs.len(), 0);
|
||||
|
||||
for (position, read) in self.outputs.iter().enumerate() {
|
||||
self.output_positions[read.input_index] = position;
|
||||
}
|
||||
}
|
||||
|
||||
fn read_outputs(&mut self, processor: &BlockProcessor<'_>) -> Result<()> {
|
||||
self.output_types.clear();
|
||||
self.output_types.reserve(self.outputs.len());
|
||||
self.type_indices.clear();
|
||||
self.type_indices.reserve(self.outputs.len());
|
||||
|
||||
let outputs = &self.outputs;
|
||||
let output_types = &mut self.output_types;
|
||||
let type_indices = &mut self.type_indices;
|
||||
|
||||
let (output_types_result, type_indices_result) = rayon::join(
|
||||
|| -> Result<()> {
|
||||
for read in outputs {
|
||||
output_types.push(
|
||||
processor
|
||||
.vecs
|
||||
.outputs
|
||||
.output_type
|
||||
.get_pushed_or_read(
|
||||
read.txout_index,
|
||||
&processor.readers.txout_index_to_output_type,
|
||||
)
|
||||
.ok_or(Error::Internal("Missing output_type"))?,
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
|| -> Result<()> {
|
||||
for read in outputs {
|
||||
type_indices.push(
|
||||
processor
|
||||
.vecs
|
||||
.outputs
|
||||
.type_index
|
||||
.get_pushed_or_read(
|
||||
read.txout_index,
|
||||
&processor.readers.txout_index_to_type_index,
|
||||
)
|
||||
.ok_or(Error::Internal("Missing type_index"))?,
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
);
|
||||
|
||||
output_types_result?;
|
||||
type_indices_result
|
||||
}
|
||||
|
||||
fn parent(&self, original_index: usize) -> ParentRead {
|
||||
self.parents[self.parent_positions[original_index]]
|
||||
}
|
||||
|
||||
fn output(&self, input_index: usize) -> (OutputType, TypeIndex) {
|
||||
let position = self.output_positions[input_index];
|
||||
(self.output_types[position], self.type_indices[position])
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum UnresolvedInput {
|
||||
Coinbase,
|
||||
PreviousBlock {
|
||||
parent_index: usize,
|
||||
vout: Vout,
|
||||
},
|
||||
SameBlock {
|
||||
outpoint: OutPoint,
|
||||
txout_offset: usize,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
use brk_types::{OutPoint, OutputType, SigOps, TypeIndex};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum InputSource {
|
||||
Coinbase,
|
||||
PreviousBlock {
|
||||
outpoint: OutPoint,
|
||||
output_type: OutputType,
|
||||
legacy_sigops: SigOps,
|
||||
type_index: TypeIndex,
|
||||
},
|
||||
SameBlock {
|
||||
outpoint: OutPoint,
|
||||
txout_offset: usize,
|
||||
},
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
use brk_cohort::ByAddrType;
|
||||
use brk_error::{Error, Result};
|
||||
use brk_store::Store;
|
||||
use brk_types::{
|
||||
AddrBytes, AddrHash, AddrIndexOutPoint, AddrIndexTxIndex, OutPoint, OutputType, Sats, TxIndex,
|
||||
TxOutIndex, TypeIndex, Unit, Vout,
|
||||
};
|
||||
use rayon::prelude::*;
|
||||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
use tracing::error;
|
||||
use vecdb::{BytesVec, WritableVec};
|
||||
|
||||
use super::{BlockProcessor, ProcessedOutput, SameBlockOutputInfo};
|
||||
use crate::{AddrsVecs, Lengths, OutputsVecs, ScriptsVecs};
|
||||
|
||||
impl<'a> BlockProcessor<'a> {
|
||||
pub fn process_outputs(&self) -> Result<Vec<ProcessedOutput<'a>>> {
|
||||
let height = self.height;
|
||||
let check_collisions = self.check_collisions;
|
||||
|
||||
let base_tx_index = self.lengths.tx_index;
|
||||
let base_txout_index = self.lengths.txout_index;
|
||||
|
||||
let total_outputs: usize = self.block.txdata.iter().map(|tx| tx.output.len()).sum();
|
||||
let mut items = Vec::with_capacity(total_outputs);
|
||||
for (index, tx) in self.block.txdata.iter().enumerate() {
|
||||
for (vout, txout) in tx.output.iter().enumerate() {
|
||||
items.push((TxIndex::from(index), Vout::from(vout), txout));
|
||||
}
|
||||
}
|
||||
|
||||
items
|
||||
.into_par_iter()
|
||||
.enumerate()
|
||||
.map(
|
||||
|(block_txout_index, (block_tx_index, vout, txout))| -> Result<ProcessedOutput> {
|
||||
let tx_index = base_tx_index + block_tx_index;
|
||||
let txout_index = base_txout_index + TxOutIndex::from(block_txout_index);
|
||||
|
||||
let script = &txout.script_pubkey;
|
||||
let output_type = OutputType::from(script);
|
||||
|
||||
if output_type.is_not_addr() {
|
||||
return Ok(ProcessedOutput {
|
||||
txout_index,
|
||||
txout,
|
||||
tx_index,
|
||||
vout,
|
||||
output_type,
|
||||
addr_info: None,
|
||||
existing_type_index: None,
|
||||
});
|
||||
}
|
||||
|
||||
let addr_type = output_type;
|
||||
let addr_bytes = AddrBytes::try_from((script, addr_type)).unwrap();
|
||||
let addr_hash = AddrHash::from(&addr_bytes);
|
||||
|
||||
let existing_type_index = self
|
||||
.stores
|
||||
.addr_type_to_addr_hash_to_addr_index
|
||||
.get_unwrap(addr_type)
|
||||
.get(&addr_hash)?
|
||||
.map(|v| *v)
|
||||
.and_then(|type_index_local| {
|
||||
(type_index_local < self.lengths.to_type_index(addr_type))
|
||||
.then_some(type_index_local)
|
||||
});
|
||||
|
||||
if check_collisions && let Some(type_index) = existing_type_index {
|
||||
let prev_addrbytes = self
|
||||
.vecs
|
||||
.addrs
|
||||
.get_bytes_by_type(addr_type, type_index, &self.readers.addrbytes)
|
||||
.ok_or(Error::Internal("Missing addrbytes"))?;
|
||||
|
||||
if prev_addrbytes != addr_bytes {
|
||||
error!(
|
||||
?height,
|
||||
?vout,
|
||||
?block_tx_index,
|
||||
?addr_type,
|
||||
?prev_addrbytes,
|
||||
?addr_bytes,
|
||||
?type_index,
|
||||
"Address hash collision"
|
||||
);
|
||||
return Err(Error::Internal("Address hash collision"));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ProcessedOutput {
|
||||
txout_index,
|
||||
txout,
|
||||
tx_index,
|
||||
vout,
|
||||
output_type,
|
||||
addr_info: Some((addr_bytes, addr_hash)),
|
||||
existing_type_index,
|
||||
})
|
||||
},
|
||||
)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) fn finalize_outputs(
|
||||
lengths: &mut Lengths,
|
||||
first_txout_index: &mut BytesVec<TxIndex, TxOutIndex>,
|
||||
outputs: &mut OutputsVecs,
|
||||
addrs: &mut AddrsVecs,
|
||||
scripts: &mut ScriptsVecs,
|
||||
addr_hash_stores: &mut ByAddrType<Store<AddrHash, TypeIndex>>,
|
||||
addr_tx_index_stores: &mut ByAddrType<Store<AddrIndexTxIndex, Unit>>,
|
||||
addr_outpoint_stores: &mut ByAddrType<Store<AddrIndexOutPoint, Unit>>,
|
||||
txouts: Vec<ProcessedOutput>,
|
||||
same_block_spent_outpoints: &FxHashSet<OutPoint>,
|
||||
already_added_addr_hash: &mut ByAddrType<FxHashMap<AddrHash, TypeIndex>>,
|
||||
same_block_output_info: &mut FxHashMap<OutPoint, SameBlockOutputInfo>,
|
||||
) -> Result<()> {
|
||||
already_added_addr_hash.values_mut().for_each(|m| m.clear());
|
||||
same_block_output_info.clear();
|
||||
|
||||
for ProcessedOutput {
|
||||
txout_index,
|
||||
txout,
|
||||
tx_index,
|
||||
vout,
|
||||
output_type,
|
||||
addr_info,
|
||||
existing_type_index,
|
||||
} in txouts
|
||||
{
|
||||
let sats = Sats::from(txout.value);
|
||||
|
||||
if vout.is_zero() {
|
||||
first_txout_index.checked_push(tx_index, txout_index)?;
|
||||
}
|
||||
|
||||
outputs.tx_index.checked_push(txout_index, tx_index)?;
|
||||
|
||||
let type_index = if let Some(ti) = existing_type_index {
|
||||
ti
|
||||
} else if let Some((addr_bytes, addr_hash)) = addr_info {
|
||||
let addr_type = output_type;
|
||||
if let Some(&ti) = already_added_addr_hash
|
||||
.get_unwrap(addr_type)
|
||||
.get(&addr_hash)
|
||||
{
|
||||
ti
|
||||
} else {
|
||||
let ti = lengths.increment_addr_index(addr_type);
|
||||
|
||||
already_added_addr_hash
|
||||
.get_mut_unwrap(addr_type)
|
||||
.insert(addr_hash, ti);
|
||||
addr_hash_stores
|
||||
.get_mut_unwrap(addr_type)
|
||||
.insert(addr_hash, ti);
|
||||
addrs.push_bytes_if_needed(ti, addr_bytes)?;
|
||||
|
||||
ti
|
||||
}
|
||||
} else {
|
||||
match output_type {
|
||||
OutputType::P2MS => {
|
||||
scripts
|
||||
.p2ms
|
||||
.to_tx_index
|
||||
.checked_push(lengths.p2ms_output_index, tx_index)?;
|
||||
lengths.p2ms_output_index.copy_then_increment()
|
||||
}
|
||||
OutputType::OpReturn => {
|
||||
scripts
|
||||
.op_return
|
||||
.to_tx_index
|
||||
.checked_push(lengths.op_return_index, tx_index)?;
|
||||
lengths.op_return_index.copy_then_increment()
|
||||
}
|
||||
OutputType::Empty => {
|
||||
scripts
|
||||
.empty
|
||||
.to_tx_index
|
||||
.checked_push(lengths.empty_output_index, tx_index)?;
|
||||
lengths.empty_output_index.copy_then_increment()
|
||||
}
|
||||
OutputType::Unknown => {
|
||||
scripts
|
||||
.unknown
|
||||
.to_tx_index
|
||||
.checked_push(lengths.unknown_output_index, tx_index)?;
|
||||
lengths.unknown_output_index.copy_then_increment()
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
};
|
||||
|
||||
outputs.value.checked_push(txout_index, sats)?;
|
||||
outputs.output_type.checked_push(txout_index, output_type)?;
|
||||
outputs.type_index.checked_push(txout_index, type_index)?;
|
||||
|
||||
if output_type.is_unspendable() {
|
||||
continue;
|
||||
} else if output_type.is_addr() {
|
||||
let addr_type = output_type;
|
||||
let addr_index = type_index;
|
||||
|
||||
addr_tx_index_stores
|
||||
.get_mut_unwrap(addr_type)
|
||||
.insert(AddrIndexTxIndex::from((addr_index, tx_index)), Unit);
|
||||
}
|
||||
|
||||
let outpoint = OutPoint::new(tx_index, vout);
|
||||
|
||||
if same_block_spent_outpoints.contains(&outpoint) {
|
||||
same_block_output_info.insert(
|
||||
outpoint,
|
||||
SameBlockOutputInfo {
|
||||
output_type,
|
||||
type_index,
|
||||
},
|
||||
);
|
||||
} else if output_type.is_addr() {
|
||||
let addr_type = output_type;
|
||||
let addr_index = type_index;
|
||||
|
||||
addr_outpoint_stores
|
||||
.get_mut_unwrap(addr_type)
|
||||
.insert(AddrIndexOutPoint::from((addr_index, outpoint)), Unit);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
mod cache;
|
||||
|
||||
use std::collections::hash_map::Entry;
|
||||
|
||||
use brk_cohort::ByAddrType;
|
||||
use brk_error::{Error, Result};
|
||||
use brk_types::{AddrBytes, AddrHash, OutputType, TxIndex, TypeIndex, Vout};
|
||||
use rayon::prelude::*;
|
||||
use rustc_hash::FxHashMap;
|
||||
use tracing::error;
|
||||
use vecdb::likely;
|
||||
|
||||
use self::cache::AddressCache;
|
||||
use super::{ProcessedOutput, processed::ProcessedOutputData};
|
||||
use crate::processor::BlockProcessor;
|
||||
|
||||
struct Lookup {
|
||||
index: usize,
|
||||
output_type: OutputType,
|
||||
hash: AddrHash,
|
||||
type_index: Option<TypeIndex>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct BlockAddresses {
|
||||
cache: AddressCache,
|
||||
indexes: ByAddrType<FxHashMap<AddrHash, usize>>,
|
||||
lookups: Vec<Lookup>,
|
||||
unique: Vec<(OutputType, AddrHash)>,
|
||||
resolved: Vec<Option<TypeIndex>>,
|
||||
}
|
||||
|
||||
impl BlockAddresses {
|
||||
pub(super) fn resolve(
|
||||
&mut self,
|
||||
processor: &BlockProcessor,
|
||||
outputs: &[ProcessedOutput],
|
||||
) -> Result<()> {
|
||||
self.clear_block();
|
||||
|
||||
for output in outputs {
|
||||
let ProcessedOutputData::Address(addr_hash) = &output.data else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if let Entry::Vacant(entry) = self
|
||||
.indexes
|
||||
.get_mut_unwrap(output.output_type)
|
||||
.entry(*addr_hash)
|
||||
{
|
||||
entry.insert(self.unique.len());
|
||||
self.unique.push((output.output_type, *addr_hash));
|
||||
}
|
||||
}
|
||||
|
||||
self.resolved.resize(self.unique.len(), None);
|
||||
|
||||
for (index, &(output_type, hash)) in self.unique.iter().enumerate() {
|
||||
if let Some(type_index) = self.cache.get(output_type, hash) {
|
||||
self.resolved[index] = Some(type_index);
|
||||
} else {
|
||||
self.lookups.push(Lookup {
|
||||
index,
|
||||
output_type,
|
||||
hash,
|
||||
type_index: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
self.lookups
|
||||
.sort_unstable_by_key(|lookup| (lookup.output_type, lookup.hash));
|
||||
|
||||
let stores = &processor.stores.addr_type_to_addr_hash_to_addr_index;
|
||||
let lengths = &*processor.lengths;
|
||||
|
||||
self.lookups
|
||||
.par_iter_mut()
|
||||
.try_for_each(|lookup| -> Result<()> {
|
||||
lookup.type_index = stores
|
||||
.get_unwrap(lookup.output_type)
|
||||
.get(&lookup.hash)?
|
||||
.map(|type_index| *type_index)
|
||||
.filter(|type_index| *type_index < lengths.to_type_index(lookup.output_type));
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
for lookup in &self.lookups {
|
||||
self.resolved[lookup.index] = lookup.type_index;
|
||||
if let Some(type_index) = lookup.type_index {
|
||||
self.cache
|
||||
.insert(lookup.output_type, lookup.hash, type_index);
|
||||
}
|
||||
}
|
||||
|
||||
if likely(!processor.check_collisions) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut output_offset = 0;
|
||||
for (block_tx_index, tx) in processor.block.txdata.iter().enumerate() {
|
||||
let tx_index = processor.lengths.tx_index + TxIndex::from(block_tx_index);
|
||||
let next_output_offset = output_offset + tx.output.len();
|
||||
|
||||
for (vout, (txout, output)) in tx
|
||||
.output
|
||||
.iter()
|
||||
.zip(&outputs[output_offset..next_output_offset])
|
||||
.enumerate()
|
||||
{
|
||||
let ProcessedOutputData::Address(addr_hash) = &output.data else {
|
||||
continue;
|
||||
};
|
||||
let Some(type_index) = self.index(output.output_type, addr_hash) else {
|
||||
continue;
|
||||
};
|
||||
let addr_bytes =
|
||||
AddrBytes::try_from((&txout.script_pubkey, output.output_type)).unwrap();
|
||||
|
||||
let prev_addrbytes = processor
|
||||
.vecs
|
||||
.addrs
|
||||
.get_bytes_by_type(output.output_type, type_index, &processor.readers.addrbytes)
|
||||
.ok_or(Error::Internal("Missing addrbytes"))?;
|
||||
|
||||
if prev_addrbytes != addr_bytes {
|
||||
error!(
|
||||
height = ?processor.height,
|
||||
vout = ?Vout::from(vout),
|
||||
?tx_index,
|
||||
addr_type = ?output.output_type,
|
||||
?prev_addrbytes,
|
||||
?addr_bytes,
|
||||
?type_index,
|
||||
"Address hash collision"
|
||||
);
|
||||
return Err(Error::Internal("Address hash collision"));
|
||||
}
|
||||
}
|
||||
|
||||
output_offset = next_output_offset;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn index_mut(
|
||||
&mut self,
|
||||
addr_type: OutputType,
|
||||
addr_hash: &AddrHash,
|
||||
) -> &mut Option<TypeIndex> {
|
||||
let index = *self
|
||||
.indexes
|
||||
.get_mut_unwrap(addr_type)
|
||||
.get_mut(addr_hash)
|
||||
.unwrap();
|
||||
&mut self.resolved[index]
|
||||
}
|
||||
|
||||
fn index(&self, addr_type: OutputType, addr_hash: &AddrHash) -> Option<TypeIndex> {
|
||||
let index = *self.indexes.get_unwrap(addr_type).get(addr_hash).unwrap();
|
||||
self.resolved[index]
|
||||
}
|
||||
|
||||
pub(crate) fn clear_cache(&mut self) {
|
||||
self.cache.clear();
|
||||
}
|
||||
|
||||
fn clear_block(&mut self) {
|
||||
self.indexes.values_mut().for_each(FxHashMap::clear);
|
||||
self.lookups.clear();
|
||||
self.unique.clear();
|
||||
self.resolved.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
use std::mem;
|
||||
|
||||
use brk_types::{AddrHash, OutputType, TypeIndex};
|
||||
|
||||
const ASSOCIATIVITY: usize = 4;
|
||||
const SET_COUNT: usize = 1 << 19;
|
||||
const SET_MASK: usize = SET_COUNT - 1;
|
||||
const TYPE_MIX: u64 = 0x9e37_79b9_7f4a_7c15;
|
||||
const REFERENCE_SHIFT: usize = 2;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
#[repr(C)]
|
||||
struct Entry {
|
||||
hash: AddrHash,
|
||||
type_index: TypeIndex,
|
||||
output_type: OutputType,
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
const EMPTY: Self = Self {
|
||||
hash: AddrHash::new(0),
|
||||
type_index: TypeIndex::COINBASE,
|
||||
output_type: OutputType::Unknown,
|
||||
};
|
||||
|
||||
#[inline]
|
||||
fn matches(&self, output_type: OutputType, hash: AddrHash) -> bool {
|
||||
self.output_type == output_type && self.hash == hash
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_empty(&self) -> bool {
|
||||
self.output_type == OutputType::Unknown
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
#[repr(C, align(64))]
|
||||
struct Set([Entry; ASSOCIATIVITY]);
|
||||
|
||||
impl Set {
|
||||
const EMPTY: Self = Self([Entry::EMPTY; ASSOCIATIVITY]);
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct SetState(u8);
|
||||
|
||||
impl SetState {
|
||||
#[inline]
|
||||
fn mark_referenced(&mut self, entry_index: usize) {
|
||||
self.0 |= 1 << (REFERENCE_SHIFT + entry_index);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn victim(&mut self) -> usize {
|
||||
loop {
|
||||
let entry_index = usize::from(self.0 & (ASSOCIATIVITY as u8 - 1));
|
||||
self.0 = (self.0 & !(ASSOCIATIVITY as u8 - 1))
|
||||
| ((entry_index + 1) & (ASSOCIATIVITY - 1)) as u8;
|
||||
|
||||
let reference = 1 << (REFERENCE_SHIFT + entry_index);
|
||||
if self.0 & reference == 0 {
|
||||
return entry_index;
|
||||
}
|
||||
self.0 &= !reference;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const _: () = assert!(mem::size_of::<Entry>() == 16);
|
||||
const _: () = assert!(mem::size_of::<Set>() == 64);
|
||||
const _: () = assert!(mem::size_of::<SetState>() == 1);
|
||||
|
||||
struct Storage {
|
||||
sets: Box<[Set]>,
|
||||
states: Box<[SetState]>,
|
||||
}
|
||||
|
||||
impl Storage {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
sets: vec![Set::EMPTY; SET_COUNT].into_boxed_slice(),
|
||||
states: vec![SetState::default(); SET_COUNT].into_boxed_slice(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct AddressCache {
|
||||
storage: Option<Storage>,
|
||||
}
|
||||
|
||||
impl AddressCache {
|
||||
#[inline]
|
||||
pub(super) fn get(&mut self, output_type: OutputType, hash: AddrHash) -> Option<TypeIndex> {
|
||||
let storage = self.storage.as_mut()?;
|
||||
let set_index = Self::set_index(output_type, hash);
|
||||
let entry_index = storage.sets[set_index]
|
||||
.0
|
||||
.iter()
|
||||
.position(|entry| entry.matches(output_type, hash))?;
|
||||
storage.states[set_index].mark_referenced(entry_index);
|
||||
Some(storage.sets[set_index].0[entry_index].type_index)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(super) fn insert(
|
||||
&mut self,
|
||||
output_type: OutputType,
|
||||
hash: AddrHash,
|
||||
type_index: TypeIndex,
|
||||
) {
|
||||
let set_index = Self::set_index(output_type, hash);
|
||||
let storage = self.storage.get_or_insert_with(Storage::new);
|
||||
let set = &mut storage.sets[set_index].0;
|
||||
let state = &mut storage.states[set_index];
|
||||
|
||||
let entry_index = set
|
||||
.iter()
|
||||
.position(Entry::is_empty)
|
||||
.unwrap_or_else(|| state.victim());
|
||||
|
||||
set[entry_index] = Entry {
|
||||
hash,
|
||||
type_index,
|
||||
output_type,
|
||||
};
|
||||
state.mark_referenced(entry_index);
|
||||
}
|
||||
|
||||
pub(super) fn clear(&mut self) {
|
||||
self.storage = None;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn set_index(output_type: OutputType, hash: AddrHash) -> usize {
|
||||
let hash = *hash ^ (output_type as u64).wrapping_mul(TYPE_MIX);
|
||||
hash as usize & SET_MASK
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
mod address;
|
||||
mod op_return;
|
||||
mod processed;
|
||||
|
||||
pub(crate) use address::BlockAddresses;
|
||||
pub(super) use processed::ProcessedOutput;
|
||||
use processed::ProcessedOutputData;
|
||||
|
||||
use bitcoin::{Script, Transaction, constants::WITNESS_SCALE_FACTOR};
|
||||
use brk_cohort::ByAddrType;
|
||||
use brk_error::Result;
|
||||
use brk_store::Store;
|
||||
use brk_types::{
|
||||
AddrBytes, AddrHash, AddrIndexOutPoint, AddrIndexTxIndex, OutPoint, OutputType, Sats, SigOps,
|
||||
TxIndex, TxOutIndex, TypeIndex, Unit, Vout,
|
||||
};
|
||||
use rayon::prelude::*;
|
||||
use vecdb::{BytesVec, WritableVec, likely};
|
||||
|
||||
use super::BlockProcessor;
|
||||
use crate::{AddrsVecs, Lengths, OpReturnVecs, OutputsVecs, ScriptsVecs};
|
||||
|
||||
impl<'a> BlockProcessor<'a> {
|
||||
pub(crate) fn process_outputs(
|
||||
&self,
|
||||
addresses: &mut BlockAddresses,
|
||||
) -> Result<Vec<ProcessedOutput>> {
|
||||
let total_outputs: usize = self.block.txdata.iter().map(|tx| tx.output.len()).sum();
|
||||
let mut items = Vec::with_capacity(total_outputs);
|
||||
for tx in &self.block.txdata {
|
||||
items.extend(&tx.output);
|
||||
}
|
||||
|
||||
let outputs = items
|
||||
.into_par_iter()
|
||||
.map(|txout| {
|
||||
let script = &txout.script_pubkey;
|
||||
let output_type = OutputType::from(script);
|
||||
let legacy_sigops = executed_legacy_sigops_for_output(output_type, script);
|
||||
let data = if output_type.is_addr() {
|
||||
ProcessedOutputData::Address(
|
||||
AddrHash::from_script(script, output_type).unwrap(),
|
||||
)
|
||||
} else if likely(output_type == OutputType::OpReturn) {
|
||||
ProcessedOutputData::OpReturn(op_return::analyze(script))
|
||||
} else {
|
||||
ProcessedOutputData::None
|
||||
};
|
||||
|
||||
ProcessedOutput {
|
||||
output_type,
|
||||
legacy_sigops,
|
||||
data,
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
addresses.resolve(self, &outputs)?;
|
||||
|
||||
Ok(outputs)
|
||||
}
|
||||
}
|
||||
|
||||
fn executed_legacy_sigops_for_output(output_type: OutputType, script_pubkey: &Script) -> SigOps {
|
||||
SigOps::from(
|
||||
match output_type {
|
||||
OutputType::P2PKH | OutputType::P2PK33 | OutputType::P2PK65 => 1,
|
||||
OutputType::P2MS | OutputType::Unknown => script_pubkey.count_sigops(),
|
||||
_ => 0,
|
||||
}
|
||||
.saturating_mul(WITNESS_SCALE_FACTOR),
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) fn finalize_outputs(
|
||||
transactions: &[Transaction],
|
||||
base_tx_index: TxIndex,
|
||||
lengths: &mut Lengths,
|
||||
first_txout_index: &mut BytesVec<TxIndex, TxOutIndex>,
|
||||
outputs: &mut OutputsVecs,
|
||||
addrs: &mut AddrsVecs,
|
||||
scripts: &mut ScriptsVecs,
|
||||
op_return_vecs: &mut OpReturnVecs,
|
||||
addr_hash_stores: &mut ByAddrType<Store<AddrHash, TypeIndex>>,
|
||||
addr_tx_index_stores: &mut ByAddrType<Store<AddrIndexTxIndex, Unit>>,
|
||||
addr_outpoint_stores: &mut ByAddrType<Store<AddrIndexOutPoint, Unit>>,
|
||||
txouts: &mut [ProcessedOutput],
|
||||
addresses: &mut BlockAddresses,
|
||||
) -> Result<()> {
|
||||
let base_txout_index = lengths.txout_index;
|
||||
let mut output_offset = 0;
|
||||
for (block_tx_index, tx) in transactions.iter().enumerate() {
|
||||
let tx_index = base_tx_index + TxIndex::from(block_tx_index);
|
||||
let next_output_offset = output_offset + tx.output.len();
|
||||
|
||||
for (vout, (txout, processed)) in tx
|
||||
.output
|
||||
.iter()
|
||||
.zip(&mut txouts[output_offset..next_output_offset])
|
||||
.enumerate()
|
||||
{
|
||||
let block_txout_index = output_offset + vout;
|
||||
let txout_index = base_txout_index + TxOutIndex::from(block_txout_index);
|
||||
let vout = Vout::from(vout);
|
||||
let output_type = processed.output_type;
|
||||
let legacy_sigops = processed.legacy_sigops;
|
||||
let data = processed.data;
|
||||
let sats = Sats::from(txout.value);
|
||||
|
||||
if vout.is_zero() {
|
||||
first_txout_index.checked_push(tx_index, txout_index)?;
|
||||
}
|
||||
|
||||
outputs.tx_index.checked_push(txout_index, tx_index)?;
|
||||
|
||||
let type_index = match data {
|
||||
ProcessedOutputData::Address(addr_hash) => {
|
||||
let addr_type = output_type;
|
||||
let type_index = addresses.index_mut(addr_type, &addr_hash);
|
||||
|
||||
if let Some(ti) = *type_index {
|
||||
ti
|
||||
} else {
|
||||
let ti = lengths.increment_addr_index(addr_type);
|
||||
|
||||
*type_index = Some(ti);
|
||||
addr_hash_stores
|
||||
.get_mut_unwrap(addr_type)
|
||||
.insert(addr_hash, ti);
|
||||
let addr_bytes =
|
||||
AddrBytes::try_from((&txout.script_pubkey, addr_type)).unwrap();
|
||||
addrs.push_bytes_if_needed(ti, addr_bytes)?;
|
||||
|
||||
ti
|
||||
}
|
||||
}
|
||||
ProcessedOutputData::OpReturn(op_return) => {
|
||||
let op_return_index = lengths.op_return_index;
|
||||
|
||||
op_return_vecs
|
||||
.to_tx_index
|
||||
.checked_push(lengths.op_return_index, tx_index)?;
|
||||
op_return_vecs
|
||||
.kind
|
||||
.checked_push(op_return_index, op_return.kind)?;
|
||||
op_return_vecs
|
||||
.post_op_return_bytes
|
||||
.checked_push(op_return_index, op_return.post_op_return_bytes)?;
|
||||
lengths.op_return_index.copy_then_increment()
|
||||
}
|
||||
ProcessedOutputData::None => match output_type {
|
||||
OutputType::P2MS => {
|
||||
let index = lengths.p2ms_output_index;
|
||||
scripts.p2ms.to_tx_index.checked_push(index, tx_index)?;
|
||||
scripts
|
||||
.p2ms
|
||||
.legacy_sigops
|
||||
.checked_push(index, legacy_sigops)?;
|
||||
lengths.p2ms_output_index.copy_then_increment()
|
||||
}
|
||||
OutputType::Empty => {
|
||||
scripts
|
||||
.empty
|
||||
.to_tx_index
|
||||
.checked_push(lengths.empty_output_index, tx_index)?;
|
||||
lengths.empty_output_index.copy_then_increment()
|
||||
}
|
||||
OutputType::Unknown => {
|
||||
let index = lengths.unknown_output_index;
|
||||
scripts.unknown.to_tx_index.checked_push(index, tx_index)?;
|
||||
scripts
|
||||
.unknown
|
||||
.legacy_sigops
|
||||
.checked_push(index, legacy_sigops)?;
|
||||
lengths.unknown_output_index.copy_then_increment()
|
||||
}
|
||||
_ => unreachable!(),
|
||||
},
|
||||
ProcessedOutputData::Resolved(_) => unreachable!(),
|
||||
};
|
||||
|
||||
outputs.value.checked_push(txout_index, sats)?;
|
||||
outputs.output_type.checked_push(txout_index, output_type)?;
|
||||
outputs.type_index.checked_push(txout_index, type_index)?;
|
||||
processed.data = ProcessedOutputData::Resolved(type_index);
|
||||
|
||||
if likely(output_type.is_addr()) {
|
||||
let addr_type = output_type;
|
||||
let addr_index = type_index;
|
||||
|
||||
addr_tx_index_stores
|
||||
.get_mut_unwrap(addr_type)
|
||||
.insert(AddrIndexTxIndex::from((addr_index, tx_index)), Unit);
|
||||
|
||||
addr_outpoint_stores.get_mut_unwrap(addr_type).insert(
|
||||
AddrIndexOutPoint::from((addr_index, OutPoint::new(tx_index, vout))),
|
||||
Unit,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
output_offset = next_output_offset;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user