Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-12-25 22:21:12 +01:00
parent eadf93b804
commit bbb74b76c8
68 changed files with 6497 additions and 2659 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
Cargo.lock generated
View File

@@ -742,11 +742,12 @@ dependencies = [
name = "brk_mcp"
version = "0.1.0-alpha.1"
dependencies = [
"aide",
"brk_query",
"axum",
"brk_rmcp",
"brk_types",
"log",
"minreq",
"schemars",
"serde",
"serde_json",
]
@@ -2780,9 +2781,9 @@ checksum = "7ee5b5339afb4c41626dde77b7a611bd4f2c202b897852b4bcf5d03eddc61010"
[[package]]
name = "jiff"
version = "0.2.16"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35"
checksum = "a87d9b8105c23642f50cbbae03d1f75d8422c5cb98ce7ee9271f7ff7505be6b8"
dependencies = [
"jiff-static",
"jiff-tzdb-platform",
@@ -2795,9 +2796,9 @@ dependencies = [
[[package]]
name = "jiff-static"
version = "0.2.16"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69"
checksum = "b787bebb543f8969132630c51fd0afab173a86c6abae56ff3b9e5e3e3f9f6e58"
dependencies = [
"proc-macro2",
"quote",
@@ -3635,14 +3636,15 @@ dependencies = [
[[package]]
name = "oxc_resolver"
version = "11.16.0"
version = "11.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82835b74b32841714c1342b1636992d19622d4ec19666b55edb4c654fb6eb719"
checksum = "5467a6fd6e1b2a0cc25f4f89a5ece8594213427e430ba8f0a8f900808553cb1e"
dependencies = [
"cfg-if",
"fast-glob",
"indexmap",
"json-strip-comments",
"nodejs-built-in-modules",
"once_cell",
"papaya",
"parking_lot",
@@ -4235,8 +4237,6 @@ dependencies = [
[[package]]
name = "rawdb"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c73aead6409391fb8d52ca74985f75983c61a81247de5d78312b7134dd1818a"
dependencies = [
"libc",
"log",
@@ -5427,8 +5427,6 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]]
name = "vecdb"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8386c4148b31b9ba931394b0e97f5fc8d5f1644c5b55cdae52611e18227aee5"
dependencies = [
"ctrlc",
"log",
@@ -5436,6 +5434,7 @@ dependencies = [
"parking_lot",
"pco",
"rawdb",
"schemars",
"serde",
"serde_json",
"thiserror 2.0.17",
@@ -5447,8 +5446,6 @@ dependencies = [
[[package]]
name = "vecdb_derive"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54a57c4efc56bf5aa76ccf39e52bc4d3154848c13996a1c0779dec4fd21eaf4a"
dependencies = [
"quote",
"syn 2.0.111",
@@ -6042,9 +6039,9 @@ checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3"
[[package]]
name = "zmij"
version = "0.1.7"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e404bcd8afdaf006e529269d3e85a743f9480c3cef60034d77860d02964f3ba"
checksum = "d0095ecd462946aa3927d9297b63ef82fb9a5316d7a37d134eeb36e58228615a"
[[package]]
name = "zopfli"

6
Cargo.toml
View File

@@ -66,7 +66,7 @@ byteview = "0.9.1"
color-eyre = "0.6.5"
derive_deref = "1.1.1"
fjall = "3.0.0-rc.6"
jiff = "0.2.16"
jiff = "0.2.17"
log = "0.4.29"
mimalloc = { version = "0.1.48", features = ["v3"] }
minreq = { version = "2.14.1", features = ["https", "serde_json"] }
@@ -80,8 +80,8 @@ serde_derive = "1.0.228"
serde_json = { version = "1.0.147", features = ["float_roundtrip"] }
smallvec = "1.15.1"
tokio = { version = "1.48.0", features = ["rt-multi-thread"] }
vecdb = { version = "0.4.6", features = ["derive", "serde_json", "pco"] }
# vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] }
# vecdb = { version = "0.4.6", features = ["derive", "serde_json", "pco"] }
vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] }
# vecdb = { git = "https://github.com/anydb-rs/anydb", features = ["derive", "serde_json", "pco"] }
[workspace.metadata.release]

55
crates/brk_binder/src/javascript.rs
View File

@@ -13,12 +13,14 @@ use crate::{
get_node_fields, get_pattern_instance_base, to_camel_case, to_pascal_case,
};
/// Generate JavaScript + JSDoc client from metadata and OpenAPI endpoints
/// Generate JavaScript + JSDoc client from metadata and OpenAPI endpoints.
///
/// `output_path` is the full path to the output file (e.g., "modules/brk-client/index.js").
pub fn generate_javascript_client(
metadata: &ClientMetadata,
endpoints: &[Endpoint],
schemas: &TypeSchemas,
output_dir: &Path,
output_path: &Path,
) -> io::Result<()> {
let mut output = String::new();
@@ -44,7 +46,7 @@ pub fn generate_javascript_client(
// Generate the main client class with tree and API methods
generate_main_client(&mut output, &metadata.catalog, metadata, endpoints);
fs::write(output_dir.join("client.js"), output)?;
fs::write(output_path, output)?;
Ok(())
}
@@ -446,13 +448,21 @@ fn generate_structural_patterns(
// Generate factory function
writeln!(output, "/**").unwrap();
writeln!(output, " * Create a {} pattern node", pattern.name).unwrap();
if pattern.is_generic {
writeln!(output, " * @template T").unwrap();
}
writeln!(output, " * @param {{BrkClientBase}} client").unwrap();
if is_parameterizable {
writeln!(output, " * @param {{string}} acc - Accumulated metric name").unwrap();
} else {
writeln!(output, " * @param {{string}} basePath").unwrap();
}
writeln!(output, " * @returns {{{}}}", pattern.name).unwrap();
let return_type = if pattern.is_generic {
format!("{}<T>", pattern.name)
} else {
pattern.name.clone()
};
writeln!(output, " * @returns {{{}}}", return_type).unwrap();
writeln!(output, " */").unwrap();
let param_name = if is_parameterizable {
@@ -613,11 +623,17 @@ fn field_to_js_type_with_generic_value(
};
if metadata.is_pattern_type(&field.rust_type) {
// Check if this pattern is generic and we have a value type
if metadata.is_pattern_generic(&field.rust_type)
&& let Some(vt) = generic_value_type
{
return format!("{}<{}>", field.rust_type, vt);
// Check if this pattern is generic
if metadata.is_pattern_generic(&field.rust_type) {
if let Some(vt) = generic_value_type {
return format!("{}<{}>", field.rust_type, vt);
} else if is_generic {
// Propagate T when inside a generic pattern
return format!("{}<T>", field.rust_type);
} else {
// Generic pattern without known type - use unknown
return format!("{}<unknown>", field.rust_type);
}
}
field.rust_type.clone()
} else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) {
@@ -629,7 +645,6 @@ fn field_to_js_type_with_generic_value(
}
}
/// Generate tree typedefs
fn generate_tree_typedefs(output: &mut String, catalog: &TreeNode, metadata: &ClientMetadata) {
writeln!(output, "// Catalog tree typedefs\n").unwrap();
@@ -666,7 +681,8 @@ fn generate_tree_typedef(
.collect();
// Skip if this matches a pattern (already generated)
if pattern_lookup.contains_key(&fields) && pattern_lookup.get(&fields) != Some(&name.to_string())
if pattern_lookup.contains_key(&fields)
&& pattern_lookup.get(&fields) != Some(&name.to_string())
{
return;
}
@@ -680,15 +696,13 @@ fn generate_tree_typedef(
writeln!(output, " * @typedef {{Object}} {}", name).unwrap();
for (field, child_fields) in &fields_with_child_info {
// Look up type parameter for generic patterns
let generic_value_type = child_fields
.as_ref()
.and_then(|cf| metadata.get_generic_value_type(&field.rust_type, cf));
let js_type = field_to_js_type_with_generic_value(
field,
metadata,
false,
generic_value_type.as_deref(),
);
.and_then(|cf| metadata.get_type_param(cf))
.map(String::as_str);
let js_type =
field_to_js_type_with_generic_value(field, metadata, false, generic_value_type);
writeln!(
output,
" * @property {{{}}} {}",
@@ -899,6 +913,11 @@ fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
if let Some(summary) = &endpoint.summary {
writeln!(output, " * {}", summary).unwrap();
}
if let Some(desc) = &endpoint.description
&& endpoint.summary.as_ref() != Some(desc)
{
writeln!(output, " * @description {}", desc).unwrap();
}
for param in &endpoint.path_params {
let desc = param.description.as_deref().unwrap_or("");

95
crates/brk_binder/src/lib.rs
View File

@@ -27,14 +27,57 @@
//! 2. **Schema collection** - Merges OpenAPI schemas with schemars-generated type schemas
//!
//! 3. **Code generation** - Produces language-specific clients:
//! - Rust: Uses `brk_types` directly, generates structs with lifetimes
//! - Rust: Uses `brk_types` directly, generates structs with Arc-based sharing
//! - JavaScript: Generates JSDoc-typed ES modules with factory functions
//! - Python: Generates typed classes with TypedDict and Generic support
use std::{collections::btree_map::Entry, fs::create_dir_all, io, path::Path};
use std::{collections::btree_map::Entry, fs::create_dir_all, io, path::PathBuf};
use brk_query::Vecs;
/// Output path configuration for each language client.
///
/// Each path should be the full path to the output file, not just a directory.
/// Parent directories will be created automatically if they don't exist.
///
/// # Example
/// ```ignore
/// let paths = ClientOutputPaths::new()
/// .rust("crates/brk_client/src/lib.rs")
/// .javascript("modules/brk-client/index.js")
/// .python("packages/brk_client/__init__.py");
/// ```
#[derive(Debug, Clone, Default)]
pub struct ClientOutputPaths {
/// Full path to Rust client file (e.g., "crates/brk_client/src/lib.rs")
pub rust: Option<PathBuf>,
/// Full path to JavaScript client file (e.g., "modules/brk-client/index.js")
pub javascript: Option<PathBuf>,
/// Full path to Python client file (e.g., "packages/brk_client/__init__.py")
pub python: Option<PathBuf>,
}
impl ClientOutputPaths {
pub fn new() -> Self {
Self::default()
}
pub fn rust(mut self, path: impl Into<PathBuf>) -> Self {
self.rust = Some(path.into());
self
}
pub fn javascript(mut self, path: impl Into<PathBuf>) -> Self {
self.javascript = Some(path.into());
self
}
pub fn python(mut self, path: impl Into<PathBuf>) -> Self {
self.python = Some(path.into());
self
}
}
mod javascript;
mod js;
mod openapi;
@@ -51,8 +94,25 @@ pub use types::*;
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
/// Generate all client libraries from the query vecs and OpenAPI JSON
pub fn generate_clients(vecs: &Vecs, openapi_json: &str, output_dir: &Path) -> io::Result<()> {
/// Generate all client libraries from the query vecs and OpenAPI JSON.
///
/// Uses `ClientOutputPaths` to specify the output file path for each language.
/// Only languages with a configured path will be generated.
///
/// # Example
/// ```ignore
/// let paths = ClientOutputPaths::new()
/// .rust("crates/brk_client/src/lib.rs")
/// .javascript("modules/brk-client/index.js")
/// .python("packages/brk_client/__init__.py");
///
/// generate_clients(&vecs, &openapi_json, &paths)?;
/// ```
pub fn generate_clients(
vecs: &Vecs,
openapi_json: &str,
output_paths: &ClientOutputPaths,
) -> io::Result<()> {
let metadata = ClientMetadata::from_vecs(vecs);
// Parse OpenAPI spec
@@ -71,19 +131,28 @@ pub fn generate_clients(vecs: &Vecs, openapi_json: &str, output_dir: &Path) -> i
}
// Generate Rust client (uses real brk_types, no schema conversion needed)
let rust_path = output_dir.join("rust");
create_dir_all(&rust_path)?;
generate_rust_client(&metadata, &endpoints, &rust_path)?;
if let Some(rust_path) = &output_paths.rust {
if let Some(parent) = rust_path.parent() {
create_dir_all(parent)?;
}
generate_rust_client(&metadata, &endpoints, rust_path)?;
}
// Generate JavaScript client (needs schemas for type definitions)
let js_path = output_dir.join("javascript");
create_dir_all(&js_path)?;
generate_javascript_client(&metadata, &endpoints, &schemas, &js_path)?;
if let Some(js_path) = &output_paths.javascript {
if let Some(parent) = js_path.parent() {
create_dir_all(parent)?;
}
generate_javascript_client(&metadata, &endpoints, &schemas, js_path)?;
}
// Generate Python client (needs schemas for type definitions)
let python_path = output_dir.join("python");
create_dir_all(&python_path)?;
generate_python_client(&metadata, &endpoints, &schemas, &python_path)?;
if let Some(python_path) = &output_paths.python {
if let Some(parent) = python_path.parent() {
create_dir_all(parent)?;
}
generate_python_client(&metadata, &endpoints, &schemas, python_path)?;
}
Ok(())
}

10
crates/brk_binder/src/openapi.rs
View File

@@ -17,8 +17,10 @@ pub struct Endpoint {
pub path: String,
/// Operation ID (e.g., "getBlockByHash")
pub operation_id: Option<String>,
/// Summary/description
/// Short summary
pub summary: Option<String>,
/// Detailed description
pub description: Option<String>,
/// Tags for grouping
pub tags: Vec<String>,
/// Path parameters
@@ -185,10 +187,8 @@ fn extract_endpoint(path: &str, method: &str, operation: &Operation) -> Option<E
method: method.to_string(),
path: path.to_string(),
operation_id: operation.operation_id.clone(),
summary: operation
.summary
.clone()
.or_else(|| operation.description.clone()),
summary: operation.summary.clone(),
description: operation.description.clone(),
tags: operation.tags.clone(),
path_params,
query_params,

29
crates/brk_binder/src/python.rs
View File

@@ -13,12 +13,14 @@ use crate::{
get_pattern_instance_base, is_enum_schema, to_pascal_case, to_snake_case,
};
/// Generate Python client from metadata and OpenAPI endpoints
/// Generate Python client from metadata and OpenAPI endpoints.
///
/// `output_path` is the full path to the output file (e.g., "packages/brk_client/__init__.py").
pub fn generate_python_client(
metadata: &ClientMetadata,
endpoints: &[Endpoint],
schemas: &TypeSchemas,
output_dir: &Path,
output_path: &Path,
) -> io::Result<()> {
let mut output = String::new();
@@ -57,7 +59,7 @@ pub fn generate_python_client(
// Generate main client with tree and API methods
generate_main_client(&mut output, endpoints);
fs::write(output_dir.join("client.py"), output)?;
fs::write(output_path, output)?;
Ok(())
}
@@ -720,14 +722,16 @@ fn generate_tree_class(
for ((field, child_fields_opt), (child_name, child_node)) in
fields_with_child_info.iter().zip(children.iter())
{
// Look up type parameter for generic patterns
let generic_value_type = child_fields_opt
.as_ref()
.and_then(|cf| metadata.get_generic_value_type(&field.rust_type, cf));
.and_then(|cf| metadata.get_type_param(cf))
.map(String::as_str);
let py_type = field_to_python_type_with_generic_value(
field,
metadata,
false,
generic_value_type.as_deref(),
generic_value_type,
);
let field_name_py = to_snake_case(&field.name);
@@ -864,8 +868,19 @@ fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
.unwrap();
// Docstring
if let Some(summary) = &endpoint.summary {
writeln!(output, " \"\"\"{}\"\"\"", summary).unwrap();
match (&endpoint.summary, &endpoint.description) {
(Some(summary), Some(desc)) if summary != desc => {
writeln!(output, " \"\"\"{}.", summary.trim_end_matches('.')).unwrap();
writeln!(output).unwrap();
writeln!(output, " {}\"\"\"", desc).unwrap();
}
(Some(summary), _) => {
writeln!(output, " \"\"\"{}\"\"\"", summary).unwrap();
}
(None, Some(desc)) => {
writeln!(output, " \"\"\"{}\"\"\"", desc).unwrap();
}
(None, None) => {}
}
// Build path

179
crates/brk_binder/src/rust.rs
View File

@@ -12,11 +12,13 @@ use crate::{
to_pascal_case, to_snake_case,
};
/// Generate Rust client from metadata and OpenAPI endpoints
/// Generate Rust client from metadata and OpenAPI endpoints.
///
/// `output_path` is the full path to the output file (e.g., "crates/brk_client/src/lib.rs").
pub fn generate_rust_client(
metadata: &ClientMetadata,
endpoints: &[Endpoint],
output_dir: &Path,
output_path: &Path,
) -> io::Result<()> {
let mut output = String::new();
@@ -47,7 +49,7 @@ pub fn generate_rust_client(
// Generate main client with API methods
generate_main_client(&mut output, endpoints);
fs::write(output_dir.join("client.rs"), output)?;
fs::write(output_path, output)?;
Ok(())
}
@@ -55,7 +57,7 @@ pub fn generate_rust_client(
fn generate_imports(output: &mut String) {
writeln!(
output,
r#"use std::marker::PhantomData;
r#"use std::sync::Arc;
use serde::de::DeserializeOwned;
use brk_types::*;
@@ -88,14 +90,14 @@ pub type Result<T> = std::result::Result<T, BrkError>;
#[derive(Debug, Clone)]
pub struct BrkClientOptions {{
pub base_url: String,
pub timeout_ms: u64,
pub timeout_secs: u64,
}}
impl Default for BrkClientOptions {{
fn default() -> Self {{
Self {{
base_url: "http://localhost:3000".to_string(),
timeout_ms: 30000,
timeout_secs: 30,
}}
}}
}}
@@ -104,36 +106,41 @@ impl Default for BrkClientOptions {{
#[derive(Debug, Clone)]
pub struct BrkClientBase {{
base_url: String,
client: reqwest::blocking::Client,
timeout_secs: u64,
}}
impl BrkClientBase {{
/// Create a new client with the given base URL.
pub fn new(base_url: impl Into<String>) -> Result<Self> {{
let base_url = base_url.into();
let client = reqwest::blocking::Client::new();
Ok(Self {{ base_url, client }})
pub fn new(base_url: impl Into<String>) -> Self {{
Self {{
base_url: base_url.into(),
timeout_secs: 30,
}}
}}
/// Create a new client with options.
pub fn with_options(options: BrkClientOptions) -> Result<Self> {{
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_millis(options.timeout_ms))
.build()
.map_err(|e| BrkError {{ message: e.to_string() }})?;
Ok(Self {{
pub fn with_options(options: BrkClientOptions) -> Self {{
Self {{
base_url: options.base_url,
client,
}})
timeout_secs: options.timeout_secs,
}}
}}
/// Make a GET request.
pub fn get<T: DeserializeOwned>(&self, path: &str) -> Result<T> {{
let url = format!("{{}}{{}}", self.base_url, path);
self.client
.get(&url)
let response = minreq::get(&url)
.with_timeout(self.timeout_secs)
.send()
.map_err(|e| BrkError {{ message: e.to_string() }})?
.map_err(|e| BrkError {{ message: e.to_string() }})?;
if response.status_code >= 400 {{
return Err(BrkError {{
message: format!("HTTP {{}}", response.status_code),
}});
}}
response
.json()
.map_err(|e| BrkError {{ message: e.to_string() }})
}}
@@ -148,18 +155,18 @@ fn generate_metric_node(output: &mut String) {
writeln!(
output,
r#"/// A metric node that can fetch data for different indexes.
pub struct MetricNode<'a, T> {{
client: &'a BrkClientBase,
pub struct MetricNode<T> {{
client: Arc<BrkClientBase>,
path: String,
_marker: PhantomData<T>,
_marker: std::marker::PhantomData<T>,
}}
impl<'a, T: DeserializeOwned> MetricNode<'a, T> {{
pub fn new(client: &'a BrkClientBase, path: String) -> Self {{
impl<T: DeserializeOwned> MetricNode<T> {{
pub fn new(client: Arc<BrkClientBase>, path: String) -> Self {{
Self {{
client,
path,
_marker: PhantomData,
_marker: std::marker::PhantomData,
}}
}}
@@ -168,7 +175,7 @@ impl<'a, T: DeserializeOwned> MetricNode<'a, T> {{
self.client.get(&self.path)
}}
/// Fetch data points within a date range.
/// Fetch data points within a range.
pub fn get_range(&self, from: &str, to: &str) -> Result<Vec<T>> {{
let path = format!("{{}}?from={{}}&to={{}}", self.path, from, to);
self.client.get(&path)
@@ -195,26 +202,20 @@ fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) {
pattern.indexes.len()
)
.unwrap();
writeln!(output, "pub struct {}<'a, T> {{", pattern.name).unwrap();
writeln!(output, "pub struct {}<T> {{", pattern.name).unwrap();
for index in &pattern.indexes {
let field_name = index_to_field_name(index);
writeln!(output, " pub {}: MetricNode<'a, T>,", field_name).unwrap();
writeln!(output, " pub {}: MetricNode<T>,", field_name).unwrap();
}
writeln!(output, " _marker: PhantomData<T>,").unwrap();
writeln!(output, "}}\n").unwrap();
// Generate impl block with constructor
writeln!(output, "impl<T: DeserializeOwned> {}<T> {{", pattern.name).unwrap();
writeln!(
output,
"impl<'a, T: DeserializeOwned> {}<'a, T> {{",
pattern.name
)
.unwrap();
writeln!(
output,
" pub fn new(client: &'a BrkClientBase, base_path: &str) -> Self {{"
" pub fn new(client: Arc<BrkClientBase>, base_path: &str) -> Self {{"
)
.unwrap();
writeln!(output, " Self {{").unwrap();
@@ -224,13 +225,12 @@ fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) {
let path_segment = index.serialize_long();
writeln!(
output,
" {}: MetricNode::new(client, format!(\"{{base_path}}/{}\")),",
" {}: MetricNode::new(client.clone(), format!(\"{{base_path}}/{}\")),",
field_name, path_segment
)
.unwrap();
}
writeln!(output, " _marker: PhantomData,").unwrap();
writeln!(output, " }}").unwrap();
writeln!(output, " }}").unwrap();
writeln!(output, "}}\n").unwrap();
@@ -256,11 +256,7 @@ fn generate_pattern_structs(
for pattern in patterns {
let is_parameterizable = pattern.is_parameterizable();
let generic_params = if pattern.is_generic {
"<'a, T>"
} else {
"<'a>"
};
let generic_params = if pattern.is_generic { "<T>" } else { "" };
writeln!(output, "/// Pattern struct for repeated tree structure.").unwrap();
writeln!(output, "pub struct {}{} {{", pattern.name, generic_params).unwrap();
@@ -275,10 +271,15 @@ fn generate_pattern_structs(
writeln!(output, "}}\n").unwrap();
// Generate impl block with constructor
let impl_generic = if pattern.is_generic {
"<T: DeserializeOwned>"
} else {
""
};
writeln!(
output,
"impl{} {}{} {{",
generic_params, pattern.name, generic_params
impl_generic, pattern.name, generic_params
)
.unwrap();
@@ -290,13 +291,13 @@ fn generate_pattern_structs(
.unwrap();
writeln!(
output,
" pub fn new(client: &'a BrkClientBase, acc: &str) -> Self {{"
" pub fn new(client: Arc<BrkClientBase>, acc: &str) -> Self {{"
)
.unwrap();
} else {
writeln!(
output,
" pub fn new(client: &'a BrkClientBase, base_path: &str) -> Self {{"
" pub fn new(client: Arc<BrkClientBase>, base_path: &str) -> Self {{"
)
.unwrap();
}
@@ -340,7 +341,7 @@ fn generate_parameterized_rust_field(
writeln!(
output,
" {}: {}::new(client, {}),",
" {}: {}::new(client.clone(), {}),",
field_name, field.rust_type, child_acc
)
.unwrap();
@@ -363,14 +364,14 @@ fn generate_parameterized_rust_field(
let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap();
writeln!(
output,
" {}: {}::new(client, &{}),",
" {}: {}::new(client.clone(), &{}),",
field_name, accessor.name, metric_expr
)
.unwrap();
} else {
writeln!(
output,
" {}: MetricNode::new(client, {}),",
" {}: MetricNode::new(client.clone(), {}),",
field_name, metric_expr
)
.unwrap();
@@ -388,7 +389,7 @@ fn generate_tree_path_rust_field(
if metadata.is_pattern_type(&field.rust_type) {
writeln!(
output,
" {}: {}::new(client, &format!(\"{{base_path}}/{}\")),",
" {}: {}::new(client.clone(), &format!(\"{{base_path}}/{}\")),",
field_name, field.rust_type, field.name
)
.unwrap();
@@ -396,14 +397,14 @@ fn generate_tree_path_rust_field(
let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap();
writeln!(
output,
" {}: {}::new(client, &format!(\"{{base_path}}/{}\")),",
" {}: {}::new(client.clone(), &format!(\"{{base_path}}/{}\")),",
field_name, accessor.name, field.name
)
.unwrap();
} else {
writeln!(
output,
" {}: MetricNode::new(client, format!(\"{{base_path}}/{}\")),",
" {}: MetricNode::new(client.clone(), format!(\"{{base_path}}/{}\")),",
field_name, field.name
)
.unwrap();
@@ -441,15 +442,16 @@ fn field_to_type_annotation_with_generic(
if metadata.is_pattern_generic(&field.rust_type)
&& let Some(vt) = generic_value_type
{
return format!("{}<'a, {}>", field.rust_type, vt);
return format!("{}<{}>", field.rust_type, vt);
}
format!("{}<'a>", field.rust_type)
// Non-generic pattern has no type params
field.rust_type.clone()
} else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) {
// Leaf with a reusable accessor pattern
format!("{}<'a, {}>", accessor.name, value_type)
format!("{}<{}>", accessor.name, value_type)
} else {
// Leaf with unique index set - use MetricNode directly
format!("MetricNode<'a, {}>", value_type)
format!("MetricNode<{}>", value_type)
}
}
@@ -501,29 +503,27 @@ fn generate_tree_node(
generated.insert(name.to_string());
writeln!(output, "/// Catalog tree node.").unwrap();
writeln!(output, "pub struct {}<'a> {{", name).unwrap();
writeln!(output, "pub struct {} {{", name).unwrap();
for (field, child_fields) in &fields_with_child_info {
let field_name = to_snake_case(&field.name);
// Look up type parameter for generic patterns
let generic_value_type = child_fields
.as_ref()
.and_then(|cf| metadata.get_generic_value_type(&field.rust_type, cf));
let type_annotation = field_to_type_annotation_with_generic(
field,
metadata,
false,
generic_value_type.as_deref(),
);
.and_then(|cf| metadata.get_type_param(cf))
.map(String::as_str);
let type_annotation =
field_to_type_annotation_with_generic(field, metadata, false, generic_value_type);
writeln!(output, " pub {}: {},", field_name, type_annotation).unwrap();
}
writeln!(output, "}}\n").unwrap();
// Generate impl block
writeln!(output, "impl<'a> {}<'a> {{", name).unwrap();
writeln!(output, "impl {} {{", name).unwrap();
writeln!(
output,
" pub fn new(client: &'a BrkClientBase, base_path: &str) -> Self {{"
" pub fn new(client: Arc<BrkClientBase>, base_path: &str) -> Self {{"
)
.unwrap();
writeln!(output, " Self {{").unwrap();
@@ -538,14 +538,14 @@ fn generate_tree_node(
let metric_base = get_pattern_instance_base(child_node, child_name);
writeln!(
output,
" {}: {}::new(client, \"{}\"),",
" {}: {}::new(client.clone(), \"{}\"),",
field_name, field.rust_type, metric_base
)
.unwrap();
} else {
writeln!(
output,
" {}: {}::new(client, &format!(\"{{base_path}}/{}\")),",
" {}: {}::new(client.clone(), &format!(\"{{base_path}}/{}\")),",
field_name, field.rust_type, field.name
)
.unwrap();
@@ -560,14 +560,14 @@ fn generate_tree_node(
if metric_path.contains("{base_path}") {
writeln!(
output,
" {}: {}::new(client, &format!(\"{}\")),",
" {}: {}::new(client.clone(), &format!(\"{}\")),",
field_name, accessor.name, metric_path
)
.unwrap();
} else {
writeln!(
output,
" {}: {}::new(client, \"{}\"),",
" {}: {}::new(client.clone(), \"{}\"),",
field_name, accessor.name, metric_path
)
.unwrap();
@@ -581,14 +581,14 @@ fn generate_tree_node(
if metric_path.contains("{base_path}") {
writeln!(
output,
" {}: MetricNode::new(client, format!(\"{}\")),",
" {}: MetricNode::new(client.clone(), format!(\"{}\")),",
field_name, metric_path
)
.unwrap();
} else {
writeln!(
output,
" {}: MetricNode::new(client, \"{}\".to_string()),",
" {}: MetricNode::new(client.clone(), \"{}\".to_string()),",
field_name, metric_path
)
.unwrap();
@@ -625,27 +625,28 @@ fn generate_main_client(output: &mut String, endpoints: &[Endpoint]) {
output,
r#"/// Main BRK client with catalog tree and API methods.
pub struct BrkClient {{
base: BrkClientBase,
base: Arc<BrkClientBase>,
tree: CatalogTree,
}}
impl BrkClient {{
/// Create a new client with the given base URL.
pub fn new(base_url: impl Into<String>) -> Result<Self> {{
Ok(Self {{
base: BrkClientBase::new(base_url)?,
}})
pub fn new(base_url: impl Into<String>) -> Self {{
let base = Arc::new(BrkClientBase::new(base_url));
let tree = CatalogTree::new(base.clone(), "");
Self {{ base, tree }}
}}
/// Create a new client with options.
pub fn with_options(options: BrkClientOptions) -> Result<Self> {{
Ok(Self {{
base: BrkClientBase::with_options(options)?,
}})
pub fn with_options(options: BrkClientOptions) -> Self {{
let base = Arc::new(BrkClientBase::with_options(options));
let tree = CatalogTree::new(base.clone(), "");
Self {{ base, tree }}
}}
/// Get the catalog tree for navigating metrics.
pub fn tree(&self) -> CatalogTree<'_> {{
CatalogTree::new(&self.base, "")
pub fn tree(&self) -> &CatalogTree {{
&self.tree
}}
"#
)
@@ -678,6 +679,12 @@ fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
endpoint.summary.as_deref().unwrap_or(&method_name)
)
.unwrap();
if let Some(desc) = &endpoint.description
&& endpoint.summary.as_ref() != Some(desc)
{
writeln!(output, " ///").unwrap();
writeln!(output, " /// {}", desc).unwrap();
}
// Build method signature
let params = build_method_params(endpoint);

23
crates/brk_binder/src/types/mod.rs
View File

@@ -39,14 +39,16 @@ pub struct ClientMetadata {
/// Index set patterns - sets of indexes that appear together on metrics
pub index_set_patterns: Vec<IndexSetPattern>,
/// Maps concrete field signatures to pattern names
pub concrete_to_pattern: HashMap<Vec<PatternField>, String>,
concrete_to_pattern: HashMap<Vec<PatternField>, String>,
/// Maps concrete field signatures to their type parameter (for generic patterns)
concrete_to_type_param: HashMap<Vec<PatternField>, String>,
}
impl ClientMetadata {
/// Extract metadata from brk_query::Vecs.
pub fn from_vecs(vecs: &Vecs) -> Self {
let catalog = vecs.catalog().clone();
let (structural_patterns, concrete_to_pattern) =
let (structural_patterns, concrete_to_pattern, concrete_to_type_param) =
patterns::detect_structural_patterns(&catalog);
let (used_indexes, index_set_patterns) = tree::detect_index_patterns(&catalog);
@@ -56,6 +58,7 @@ impl ClientMetadata {
used_indexes,
index_set_patterns,
concrete_to_pattern,
concrete_to_type_param,
}
}
@@ -81,19 +84,9 @@ impl ClientMetadata {
self.find_pattern(name).is_some_and(|p| p.is_generic)
}
/// Extract the value type from concrete fields for a generic pattern.
pub fn get_generic_value_type(
&self,
pattern_name: &str,
fields: &[PatternField],
) -> Option<String> {
if !self.is_pattern_generic(pattern_name) {
return None;
}
fields
.iter()
.find(|f| f.is_leaf())
.map(|f| extract_inner_type(&f.rust_type))
/// Get the type parameter for a generic pattern given its concrete fields.
pub fn get_type_param(&self, fields: &[PatternField]) -> Option<&String> {
self.concrete_to_type_param.get(fields)
}
/// Build a lookup map from field signatures to pattern names.

100
crates/brk_binder/src/types/patterns.rs
View File

@@ -1,19 +1,24 @@
//! Pattern detection for structural patterns in the metric tree.
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::collections::{BTreeSet, HashMap};
use brk_types::TreeNode;
use super::{
case::to_pascal_case, schema::schema_to_json_type, FieldNamePosition, PatternField,
StructuralPattern,
FieldNamePosition, PatternField, StructuralPattern, case::to_pascal_case,
schema::schema_to_json_type,
tree::{get_first_leaf_name, get_node_fields},
};
/// Detect structural patterns in the tree using a bottom-up approach.
/// Returns (patterns, concrete_to_pattern_mapping).
/// Returns (patterns, concrete_to_pattern, concrete_to_type_param).
pub fn detect_structural_patterns(
tree: &TreeNode,
) -> (Vec<StructuralPattern>, HashMap<Vec<PatternField>, String>) {
) -> (
Vec<StructuralPattern>,
HashMap<Vec<PatternField>, String>,
HashMap<Vec<PatternField>, String>,
) {
let mut signature_to_pattern: HashMap<Vec<PatternField>, String> = HashMap::new();
let mut signature_counts: HashMap<Vec<PatternField>, usize> = HashMap::new();
let mut normalized_to_name: HashMap<Vec<PatternField>, String> = HashMap::new();
@@ -29,8 +34,9 @@ pub fn detect_structural_patterns(
&mut name_counts,
);
// Identify generic patterns
let (generic_patterns, generic_mappings) = detect_generic_patterns(&signature_to_pattern);
// Identify generic patterns (also extracts type params)
let (generic_patterns, generic_mappings, type_mappings) =
detect_generic_patterns(&signature_to_pattern);
// Build non-generic patterns: signatures appearing 2+ times that weren't merged into generics
let mut patterns: Vec<StructuralPattern> = signature_to_pattern
@@ -64,33 +70,43 @@ pub fn detect_structural_patterns(
analyze_pattern_field_positions(tree, &mut patterns, &pattern_lookup);
patterns.sort_by(|a, b| b.fields.len().cmp(&a.fields.len()));
(patterns, concrete_to_pattern)
(patterns, concrete_to_pattern, type_mappings)
}
/// Detect generic patterns by grouping signatures by their normalized form.
/// Returns (patterns, concrete_to_pattern, concrete_to_type_param).
fn detect_generic_patterns(
signature_to_pattern: &HashMap<Vec<PatternField>, String>,
) -> (Vec<StructuralPattern>, HashMap<Vec<PatternField>, String>) {
let mut normalized_groups: HashMap<Vec<PatternField>, Vec<(Vec<PatternField>, String)>> =
HashMap::new();
) -> (
Vec<StructuralPattern>,
HashMap<Vec<PatternField>, String>,
HashMap<Vec<PatternField>, String>,
) {
// Group by normalized form, tracking the extracted type for each concrete signature
let mut normalized_groups: HashMap<
Vec<PatternField>,
Vec<(Vec<PatternField>, String, String)>,
> = HashMap::new();
for (fields, name) in signature_to_pattern {
if let Some(normalized) = normalize_fields_for_generic(fields) {
if let Some((normalized, extracted_type)) = normalize_fields_for_generic(fields) {
normalized_groups
.entry(normalized)
.or_default()
.push((fields.clone(), name.clone()));
.push((fields.clone(), name.clone(), extracted_type));
}
}
let mut patterns = Vec::new();
let mut mappings: HashMap<Vec<PatternField>, String> = HashMap::new();
let mut pattern_mappings: HashMap<Vec<PatternField>, String> = HashMap::new();
let mut type_mappings: HashMap<Vec<PatternField>, String> = HashMap::new();
for (normalized_fields, group) in normalized_groups {
if group.len() >= 2 {
let generic_name = group[0].1.clone();
for (concrete_fields, _) in &group {
mappings.insert(concrete_fields.clone(), generic_name.clone());
for (concrete_fields, _, extracted_type) in &group {
pattern_mappings.insert(concrete_fields.clone(), generic_name.clone());
type_mappings.insert(concrete_fields.clone(), extracted_type.clone());
}
patterns.push(StructuralPattern {
name: generic_name,
@@ -101,11 +117,12 @@ fn detect_generic_patterns(
}
}
(patterns, mappings)
(patterns, pattern_mappings, type_mappings)
}
/// Normalize fields by replacing concrete value types with "T".
fn normalize_fields_for_generic(fields: &[PatternField]) -> Option<Vec<PatternField>> {
/// Returns (normalized_fields, extracted_type) where extracted_type is the concrete type replaced.
fn normalize_fields_for_generic(fields: &[PatternField]) -> Option<(Vec<PatternField>, String)> {
let leaf_types: Vec<&str> = fields
.iter()
.filter(|f| f.is_leaf())
@@ -137,7 +154,7 @@ fn normalize_fields_for_generic(fields: &[PatternField]) -> Option<Vec<PatternFi
})
.collect();
Some(normalized)
Some((normalized, super::extract_inner_type(first_type)))
}
/// Recursively resolve branch patterns bottom-up.
@@ -266,7 +283,7 @@ fn collect_pattern_instances(
return;
};
let fields = get_node_fields_for_analysis(children, pattern_lookup);
let fields = get_node_fields(children, pattern_lookup);
if let Some(pattern_name) = pattern_lookup.get(&fields) {
for (field_name, child_node) in children {
if let TreeNode::Leaf(leaf) = child_node {
@@ -283,7 +300,7 @@ fn collect_pattern_instances(
let child_accumulated = match child_node {
TreeNode::Leaf(leaf) => leaf.name().to_string(),
TreeNode::Branch(_) => {
if let Some(desc_leaf_name) = get_descendant_leaf_name(child_node) {
if let Some(desc_leaf_name) = get_first_leaf_name(child_node) {
infer_accumulated_name(accumulated_name, field_name, &desc_leaf_name)
} else if accumulated_name.is_empty() {
field_name.clone()
@@ -296,13 +313,6 @@ fn collect_pattern_instances(
}
}
fn get_descendant_leaf_name(node: &TreeNode) -> Option<String> {
match node {
TreeNode::Leaf(leaf) => Some(leaf.name().to_string()),
TreeNode::Branch(children) => children.values().find_map(get_descendant_leaf_name),
}
}
fn infer_accumulated_name(parent_acc: &str, field_name: &str, descendant_leaf: &str) -> String {
if let Some(pos) = descendant_leaf.find(field_name) {
if pos == 0 {
@@ -324,40 +334,6 @@ fn infer_accumulated_name(parent_acc: &str, field_name: &str, descendant_leaf: &
}
}
fn get_node_fields_for_analysis(
children: &BTreeMap<String, TreeNode>,
pattern_lookup: &HashMap<Vec<PatternField>, String>,
) -> Vec<PatternField> {
let mut fields: Vec<PatternField> = children
.iter()
.map(|(name, node)| {
let (rust_type, json_type, indexes) = match node {
TreeNode::Leaf(leaf) => (
leaf.value_type().to_string(),
schema_to_json_type(&leaf.schema),
leaf.indexes().clone(),
),
TreeNode::Branch(grandchildren) => {
let child_fields = get_node_fields_for_analysis(grandchildren, pattern_lookup);
let pattern_name = pattern_lookup
.get(&child_fields)
.cloned()
.unwrap_or_else(|| "Unknown".to_string());
(pattern_name.clone(), pattern_name, BTreeSet::new())
}
};
PatternField {
name: name.clone(),
rust_type,
json_type,
indexes,
}
})
.collect();
fields.sort_by(|a, b| a.name.cmp(&b.name));
fields
}
fn analyze_field_positions_from_instances(
instances: &[(String, String, String)],
) -> HashMap<String, FieldNamePosition> {

36
crates/brk_binder/src/types/tree.rs
View File

@@ -59,42 +59,6 @@ pub fn get_node_fields(
fields
}
/// Like get_node_fields but takes a parent name for generating child pattern names.
pub fn get_node_fields_with_parent(
children: &BTreeMap<String, TreeNode>,
parent_name: &str,
pattern_lookup: &HashMap<Vec<PatternField>, String>,
) -> Vec<PatternField> {
let mut fields: Vec<PatternField> = children
.iter()
.map(|(name, node)| {
let (rust_type, json_type, indexes) = match node {
TreeNode::Leaf(leaf) => (
leaf.value_type().to_string(),
schema_to_json_type(&leaf.schema),
leaf.indexes().clone(),
),
TreeNode::Branch(grandchildren) => {
let child_fields = get_node_fields(grandchildren, pattern_lookup);
let pattern_name = pattern_lookup
.get(&child_fields)
.cloned()
.unwrap_or_else(|| format!("{}_{}", parent_name, to_pascal_case(name)));
(pattern_name.clone(), pattern_name, BTreeSet::new())
}
};
PatternField {
name: name.clone(),
rust_type,
json_type,
indexes,
}
})
.collect();
fields.sort_by(|a, b| a.name.cmp(&b.name));
fields
}
/// Get fields with child field information for generic pattern lookup.
/// Returns (field, child_fields) pairs where child_fields is Some for branches.
pub fn get_fields_with_child_info(

2
crates/brk_computer/examples/computer_read.rs
View File

@@ -35,7 +35,7 @@ fn run() -> Result<()> {
let computer = Computer::forced_import(&outputs_dir, &indexer, Some(fetcher))?;
let _a = dbg!(computer.chain.txinindex_to_value.region().meta());
let _b = dbg!(indexer.vecs.txout.txoutindex_to_value.region().meta());
let _b = dbg!(indexer.vecs.txout.txoutindex_to_txoutdata.region().meta());
Ok(())
}

40
crates/brk_computer/src/chain/compute.rs
View File

@@ -4,7 +4,7 @@ use brk_types::{
CheckedSub, FeeRate, HalvingEpoch, Height, ONE_DAY_IN_SEC_F64, Sats, StoredF32, StoredF64,
StoredU32, StoredU64, Timestamp, TxOutIndex, TxVersion,
};
use vecdb::{Exit, GenericStoredVec, IterableVec, TypedVecIterator, VecIndex, unlikely};
use vecdb::{Exit, IterableVec, TypedVecIterator, VecIndex, unlikely};
use crate::{grouped::ComputedVecsFromHeight, indexes, price, utils::OptionExt, Indexes};
@@ -275,39 +275,11 @@ impl Vecs {
// TxInIndex
// ---
let txindex_to_first_txoutindex = &indexer.vecs.tx.txindex_to_first_txoutindex;
let txindex_to_first_txoutindex_reader = txindex_to_first_txoutindex.create_reader();
let txoutindex_to_value = &indexer.vecs.txout.txoutindex_to_value;
let txoutindex_to_value_reader = indexer.vecs.txout.txoutindex_to_value.create_reader();
self.txinindex_to_value.compute_transform(
starting_indexes.txinindex,
&indexer.vecs.txin.txinindex_to_outpoint,
|(txinindex, outpoint, ..)| {
if unlikely(outpoint.is_coinbase()) {
return (txinindex, Sats::MAX);
}
let txoutindex = txindex_to_first_txoutindex
.read_unwrap(outpoint.txindex(), &txindex_to_first_txoutindex_reader)
+ outpoint.vout();
let value = if unlikely(txoutindex == TxOutIndex::COINBASE) {
unreachable!()
} else {
txoutindex_to_value
.unchecked_read(txoutindex, &txoutindex_to_value_reader)
.unwrap()
};
(txinindex, value)
},
exit,
)?;
self.txindex_to_input_value.compute_sum_from_indexes(
starting_indexes.txindex,
&indexer.vecs.tx.txindex_to_first_txinindex,
&indexes.txindex_to_input_count,
&self.txinindex_to_value,
&indexer.vecs.txin.txinindex_to_value,
exit,
)?;
@@ -393,7 +365,8 @@ impl Vecs {
let mut txindex_to_first_txoutindex_iter =
indexer.vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut txindex_to_output_count_iter = indexes.txindex_to_output_count.iter();
let mut txoutindex_to_value_iter = indexer.vecs.txout.txoutindex_to_value.iter()?;
let mut txoutindex_to_txoutdata_iter =
indexer.vecs.txout.txoutindex_to_txoutdata.iter()?;
vec.compute_transform(
starting_indexes.height,
&indexer.vecs.tx.height_to_first_txindex,
@@ -405,8 +378,9 @@ impl Vecs {
let mut sats = Sats::ZERO;
(first_txoutindex..first_txoutindex + usize::from(output_count)).for_each(
|txoutindex| {
sats += txoutindex_to_value_iter
.get_unwrap(TxOutIndex::from(txoutindex));
sats += txoutindex_to_txoutdata_iter
.get_unwrap(TxOutIndex::from(txoutindex))
.value;
},
);
(height, sats)

10
crates/brk_computer/src/pools/mod.rs
View File

@@ -126,9 +126,8 @@ impl Vecs {
let mut txindex_to_first_txoutindex_iter =
indexer.vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut txindex_to_output_count_iter = indexes.txindex_to_output_count.iter();
let mut txoutindex_to_outputtype_iter =
indexer.vecs.txout.txoutindex_to_outputtype.iter()?;
let mut txoutindex_to_typeindex_iter = indexer.vecs.txout.txoutindex_to_typeindex.iter()?;
let mut txoutindex_to_txoutdata_iter =
indexer.vecs.txout.txoutindex_to_txoutdata.iter()?;
let mut p2pk65addressindex_to_p2pk65bytes_iter = indexer
.vecs
.address
@@ -181,8 +180,9 @@ impl Vecs {
let pool = (*txoutindex..(*txoutindex + *outputcount))
.map(TxOutIndex::from)
.find_map(|txoutindex| {
let outputtype = txoutindex_to_outputtype_iter.get_unwrap(txoutindex);
let typeindex = txoutindex_to_typeindex_iter.get_unwrap(txoutindex);
let txoutdata = txoutindex_to_txoutdata_iter.get_unwrap(txoutindex);
let outputtype = txoutdata.outputtype;
let typeindex = txoutdata.typeindex;
match outputtype {
OutputType::P2PK65 => Some(AddressBytes::from(

53
crates/brk_computer/src/stateful/compute/block_loop.rs
View File

@@ -24,8 +24,8 @@ use crate::{
address::AddressTypeToAddressCount,
compute::write::{process_address_updates, write},
process::{
AddressCache, InputsResult, build_txoutindex_to_height_map, process_inputs,
process_outputs, process_received, process_sent,
AddressCache, InputsResult, process_inputs, process_outputs, process_received,
process_sent,
},
states::{BlockState, Transacted},
},
@@ -38,8 +38,8 @@ use super::{
vecs::Vecs,
},
BIP30_DUPLICATE_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_2, BIP30_ORIGINAL_HEIGHT_1,
BIP30_ORIGINAL_HEIGHT_2, ComputeContext, FLUSH_INTERVAL, IndexerReaders, TxInIterators,
TxOutIterators, VecsReaders, build_txinindex_to_txindex, build_txoutindex_to_txindex,
BIP30_ORIGINAL_HEIGHT_2, ComputeContext, FLUSH_INTERVAL, TxInIterators, TxOutIterators,
VecsReaders, build_txinindex_to_txindex, build_txoutindex_to_txindex,
};
/// Process all blocks from starting_height to last_height.
@@ -124,15 +124,8 @@ pub fn process_blocks(
let mut height_to_price_iter = height_to_price.map(|v| v.into_iter());
let mut dateindex_to_price_iter = dateindex_to_price.map(|v| v.into_iter());
info!("Building txoutindex_to_height map...");
// Build txoutindex -> height map for input processing
let txoutindex_to_height = build_txoutindex_to_height_map(height_to_first_txoutindex);
info!("Creating readers...");
// Create readers for parallel data access
let ir = IndexerReaders::new(indexer);
let mut vr = VecsReaders::new(&vecs.any_address_indexes, &vecs.addresses_data);
// Create reusable iterators for sequential txout/txin reads (16KB buffered)
@@ -273,14 +266,14 @@ pub fn process_blocks(
// Collect output/input data using reusable iterators (16KB buffered reads)
// Must be done before thread::scope since iterators aren't Send
let (output_values, output_types, output_typeindexes) =
txout_iters.collect_block_outputs(first_txoutindex, output_count);
let txoutdata_vec = txout_iters.collect_block_outputs(first_txoutindex, output_count);
let input_outpoints = if input_count > 1 {
txin_iters.collect_block_outpoints(first_txinindex + 1, input_count - 1)
} else {
Vec::new()
};
let (input_values, input_prev_heights, input_outputtypes, input_typeindexes) =
if input_count > 1 {
txin_iters.collect_block_inputs(first_txinindex + 1, input_count - 1)
} else {
(Vec::new(), Vec::new(), Vec::new(), Vec::new())
};
// Process outputs and inputs in parallel with tick-tock
let (outputs_result, inputs_result) = thread::scope(|scope| {
@@ -293,11 +286,8 @@ pub fn process_blocks(
let outputs_handle = scope.spawn(|| {
// Process outputs (receive)
process_outputs(
output_count,
&txoutindex_to_txindex,
&output_values,
&output_types,
&output_typeindexes,
&txoutdata_vec,
&first_addressindexes,
&cache,
&vr,
@@ -309,16 +299,12 @@ pub fn process_blocks(
// Process inputs (send) - skip coinbase input
let inputs_result = if input_count > 1 {
process_inputs(
first_txinindex + 1, // Skip coinbase
input_count - 1,
&txinindex_to_txindex[1..], // Skip coinbase
&input_outpoints,
&indexer.vecs.tx.txindex_to_first_txoutindex,
&indexer.vecs.txout.txoutindex_to_value,
&indexer.vecs.txout.txoutindex_to_outputtype,
&indexer.vecs.txout.txoutindex_to_typeindex,
&txoutindex_to_height,
&ir,
&input_values,
&input_outputtypes,
&input_typeindexes,
&input_prev_heights,
&first_addressindexes,
&cache,
&vr,
@@ -331,7 +317,6 @@ pub fn process_blocks(
sent_data: Default::default(),
address_data: Default::default(),
txindex_vecs: Default::default(),
txoutindex_to_txinindex_updates: Default::default(),
}
};
@@ -426,12 +411,6 @@ pub fn process_blocks(
vecs.utxo_cohorts.send(height_to_sent, chain_state);
});
// Update txoutindex_to_txinindex
vecs.update_txoutindex_to_txinindex(
output_count,
inputs_result.txoutindex_to_txinindex_updates,
)?;
// Push to height-indexed vectors
vecs.height_to_unspendable_supply
.truncate_push(height, unspendable_supply)?;

2
crates/brk_computer/src/stateful/compute/mod.rs
View File

@@ -17,7 +17,7 @@ mod write;
pub use block_loop::process_blocks;
pub use context::ComputeContext;
pub use readers::{
IndexerReaders, TxInIterators, TxOutIterators, VecsReaders, build_txinindex_to_txindex,
TxInIterators, TxOutIterators, VecsReaders, build_txinindex_to_txindex,
build_txoutindex_to_txindex,
};
pub use recover::{StartMode, determine_start_mode, recover_state, reset_state};

86
crates/brk_computer/src/stateful/compute/readers.rs
View File

@@ -4,7 +4,9 @@
use brk_grouper::{ByAddressType, ByAnyAddress};
use brk_indexer::Indexer;
use brk_types::{OutPoint, OutputType, Sats, StoredU64, TxInIndex, TxIndex, TxOutIndex, TypeIndex};
use brk_types::{
Height, OutputType, Sats, StoredU64, TxInIndex, TxIndex, TxOutData, TxOutIndex, TypeIndex,
};
use vecdb::{
BoxedVecIterator, BytesVecIterator, GenericStoredVec, PcodecVecIterator, Reader, VecIndex,
VecIterator,
@@ -12,45 +14,18 @@ use vecdb::{
use crate::stateful::address::{AddressesDataVecs, AnyAddressIndexesVecs};
/// Cached readers for indexer vectors.
pub struct IndexerReaders {
pub txindex_to_first_txoutindex: Reader,
pub txoutindex_to_value: Reader,
pub txoutindex_to_outputtype: Reader,
pub txoutindex_to_typeindex: Reader,
}
impl IndexerReaders {
pub fn new(indexer: &Indexer) -> Self {
Self {
txindex_to_first_txoutindex: indexer
.vecs
.tx
.txindex_to_first_txoutindex
.create_reader(),
txoutindex_to_value: indexer.vecs.txout.txoutindex_to_value.create_reader(),
txoutindex_to_outputtype: indexer.vecs.txout.txoutindex_to_outputtype.create_reader(),
txoutindex_to_typeindex: indexer.vecs.txout.txoutindex_to_typeindex.create_reader(),
}
}
}
/// Reusable iterators for txout vectors (16KB buffered reads).
///
/// Iterators are created once and re-positioned each block to avoid
/// creating new file handles repeatedly.
pub struct TxOutIterators<'a> {
value_iter: BytesVecIterator<'a, TxOutIndex, Sats>,
outputtype_iter: BytesVecIterator<'a, TxOutIndex, OutputType>,
typeindex_iter: BytesVecIterator<'a, TxOutIndex, TypeIndex>,
txoutdata_iter: BytesVecIterator<'a, TxOutIndex, TxOutData>,
}
impl<'a> TxOutIterators<'a> {
pub fn new(indexer: &'a Indexer) -> Self {
Self {
value_iter: indexer.vecs.txout.txoutindex_to_value.into_iter(),
outputtype_iter: indexer.vecs.txout.txoutindex_to_outputtype.into_iter(),
typeindex_iter: indexer.vecs.txout.txoutindex_to_typeindex.into_iter(),
txoutdata_iter: indexer.vecs.txout.txoutindex_to_txoutdata.into_iter(),
}
}
@@ -59,43 +34,50 @@ impl<'a> TxOutIterators<'a> {
&mut self,
first_txoutindex: usize,
output_count: usize,
) -> (Vec<Sats>, Vec<OutputType>, Vec<TypeIndex>) {
let mut values = Vec::with_capacity(output_count);
let mut output_types = Vec::with_capacity(output_count);
let mut type_indexes = Vec::with_capacity(output_count);
for i in first_txoutindex..first_txoutindex + output_count {
values.push(self.value_iter.get_at_unwrap(i));
output_types.push(self.outputtype_iter.get_at_unwrap(i));
type_indexes.push(self.typeindex_iter.get_at_unwrap(i));
}
(values, output_types, type_indexes)
) -> Vec<TxOutData> {
(first_txoutindex..first_txoutindex + output_count)
.map(|i| self.txoutdata_iter.get_at_unwrap(i))
.collect()
}
}
/// Reusable iterator for txin outpoints (PcoVec - avoids repeated page decompression).
/// Reusable iterators for txin vectors (PcoVec - avoids repeated page decompression).
pub struct TxInIterators<'a> {
outpoint_iter: PcodecVecIterator<'a, TxInIndex, OutPoint>,
value_iter: PcodecVecIterator<'a, TxInIndex, Sats>,
prev_height_iter: PcodecVecIterator<'a, TxInIndex, Height>,
outputtype_iter: PcodecVecIterator<'a, TxInIndex, OutputType>,
typeindex_iter: PcodecVecIterator<'a, TxInIndex, TypeIndex>,
}
impl<'a> TxInIterators<'a> {
pub fn new(indexer: &'a Indexer) -> Self {
Self {
outpoint_iter: indexer.vecs.txin.txinindex_to_outpoint.into_iter(),
value_iter: indexer.vecs.txin.txinindex_to_value.into_iter(),
prev_height_iter: indexer.vecs.txin.txinindex_to_prev_height.into_iter(),
outputtype_iter: indexer.vecs.txin.txinindex_to_outputtype.into_iter(),
typeindex_iter: indexer.vecs.txin.txinindex_to_typeindex.into_iter(),
}
}
/// Collect outpoints for a block range using buffered iteration.
/// This avoids repeated PcoVec page decompression (~1000x speedup).
pub fn collect_block_outpoints(
/// Collect input data for a block range using buffered iteration.
pub fn collect_block_inputs(
&mut self,
first_txinindex: usize,
input_count: usize,
) -> Vec<OutPoint> {
(first_txinindex..first_txinindex + input_count)
.map(|i| self.outpoint_iter.get_at_unwrap(i))
.collect()
) -> (Vec<Sats>, Vec<Height>, Vec<OutputType>, Vec<TypeIndex>) {
let mut values = Vec::with_capacity(input_count);
let mut prev_heights = Vec::with_capacity(input_count);
let mut outputtypes = Vec::with_capacity(input_count);
let mut typeindexes = Vec::with_capacity(input_count);
for i in first_txinindex..first_txinindex + input_count {
values.push(self.value_iter.get_at_unwrap(i));
prev_heights.push(self.prev_height_iter.get_at_unwrap(i));
outputtypes.push(self.outputtype_iter.get_at_unwrap(i));
typeindexes.push(self.typeindex_iter.get_at_unwrap(i));
}
(values, prev_heights, outputtypes, typeindexes)
}
}

8
crates/brk_computer/src/stateful/compute/recover.rs
View File

@@ -27,7 +27,6 @@ pub struct RecoveredState {
pub fn recover_state(
height: Height,
chain_state_rollback: vecdb::Result<Stamp>,
txoutindex_rollback: vecdb::Result<Stamp>,
any_address_indexes: &mut AnyAddressIndexesVecs,
addresses_data: &mut AddressesDataVecs,
utxo_cohorts: &mut UTXOCohorts,
@@ -42,7 +41,6 @@ pub fn recover_state(
// Verify rollback consistency - all must agree on the same height
let consistent_height = rollback_states(
chain_state_rollback,
txoutindex_rollback,
address_indexes_rollback,
address_data_rollback,
);
@@ -127,7 +125,6 @@ pub enum StartMode {
/// otherwise returns Height::ZERO (need fresh start).
fn rollback_states(
chain_state_rollback: vecdb::Result<Stamp>,
txoutindex_rollback: vecdb::Result<Stamp>,
address_indexes_rollbacks: Result<Vec<Stamp>>,
address_data_rollbacks: Result<[Stamp; 2]>,
) -> Height {
@@ -139,11 +136,6 @@ fn rollback_states(
};
heights.insert(Height::from(s).incremented());
let Ok(s) = txoutindex_rollback else {
return Height::ZERO;
};
heights.insert(Height::from(s).incremented());
let Ok(stamps) = address_indexes_rollbacks else {
return Height::ZERO;
};

3
crates/brk_computer/src/stateful/compute/write.rs
View File

@@ -89,9 +89,6 @@ pub fn write(
vecs.addresstype_to_height_to_empty_addr_count
.par_iter_mut(),
)
.chain(rayon::iter::once(
&mut vecs.txoutindex_to_txinindex as &mut dyn AnyStoredVec,
))
.chain(rayon::iter::once(
&mut vecs.chain_state as &mut dyn AnyStoredVec,
))

5
crates/brk_computer/src/stateful/mod.rs
View File

@@ -39,8 +39,3 @@ pub use address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexe
// Cohort re-exports
pub use cohorts::{AddressCohorts, CohortVecs, DynCohortVecs, UTXOCohorts};
// Compute re-exports
pub use compute::IndexerReaders;
// Metrics re-exports

71
crates/brk_computer/src/stateful/process/inputs.rs
View File

@@ -1,24 +1,20 @@
//! Parallel input processing.
//!
//! Processes a block's inputs (spent UTXOs) in parallel, building:
//! - height_to_sent: map from creation height -> Transacted for sends
//! - Address data for address cohort tracking (optional)
use brk_grouper::ByAddressType;
use brk_types::{Height, OutPoint, OutputType, Sats, TxInIndex, TxIndex, TxOutIndex, TypeIndex};
use brk_types::{Height, OutputType, Sats, TxIndex, TypeIndex};
use rayon::prelude::*;
use rustc_hash::FxHashMap;
use vecdb::{BytesVec, GenericStoredVec};
use crate::stateful::address::{
AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs,
use crate::stateful::{
address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs},
compute::VecsReaders,
states::Transacted,
};
use crate::stateful::compute::VecsReaders;
use crate::stateful::states::Transacted;
use crate::stateful::{IndexerReaders, process::RangeMap};
use super::super::address::HeightToAddressTypeToVec;
use super::{load_uncached_address_data, AddressCache, LoadedAddressDataWithSource, TxIndexVec};
use super::{
super::address::HeightToAddressTypeToVec, AddressCache, LoadedAddressDataWithSource,
TxIndexVec, load_uncached_address_data,
};
/// Result of processing inputs for a block.
pub struct InputsResult {
@@ -30,8 +26,6 @@ pub struct InputsResult {
pub address_data: AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>,
/// Transaction indexes per address for tx_count tracking.
pub txindex_vecs: AddressTypeToTypeIndexMap<TxIndexVec>,
/// Updates to txoutindex_to_txinindex: (spent txoutindex, spending txinindex).
pub txoutindex_to_txinindex_updates: Vec<(TxOutIndex, TxInIndex)>,
}
/// Process inputs (spent UTXOs) for a block.
@@ -49,52 +43,32 @@ pub struct InputsResult {
/// expensive merge overhead from rayon's fold/reduce pattern.
#[allow(clippy::too_many_arguments)]
pub fn process_inputs(
first_txinindex: usize,
input_count: usize,
txinindex_to_txindex: &[TxIndex],
// Pre-collected outpoints (from reusable iterator with page caching)
outpoints: &[OutPoint],
txindex_to_first_txoutindex: &BytesVec<TxIndex, TxOutIndex>,
txoutindex_to_value: &BytesVec<TxOutIndex, Sats>,
txoutindex_to_outputtype: &BytesVec<TxOutIndex, OutputType>,
txoutindex_to_typeindex: &BytesVec<TxOutIndex, TypeIndex>,
txoutindex_to_height: &RangeMap<TxOutIndex, Height>,
ir: &IndexerReaders,
// Address lookup parameters
txinindex_to_value: &[Sats],
txinindex_to_outputtype: &[OutputType],
txinindex_to_typeindex: &[TypeIndex],
txinindex_to_prev_height: &[Height],
first_addressindexes: &ByAddressType<TypeIndex>,
cache: &AddressCache,
vr: &VecsReaders,
any_address_indexes: &AnyAddressIndexesVecs,
addresses_data: &AddressesDataVecs,
) -> InputsResult {
// Parallel reads - collect all input data (outpoints already in memory)
let items: Vec<_> = (0..input_count)
.into_par_iter()
.map(|local_idx| {
let txinindex = TxInIndex::from(first_txinindex + local_idx);
let txindex = txinindex_to_txindex[local_idx];
// Get outpoint from pre-collected vec and resolve to txoutindex
let outpoint = outpoints[local_idx];
let first_txoutindex = txindex_to_first_txoutindex
.read_unwrap(outpoint.txindex(), &ir.txindex_to_first_txoutindex);
let txoutindex = first_txoutindex + outpoint.vout();
let prev_height = *txinindex_to_prev_height.get(local_idx).unwrap();
let value = *txinindex_to_value.get(local_idx).unwrap();
let input_type = *txinindex_to_outputtype.get(local_idx).unwrap();
// Get creation height
let prev_height = *txoutindex_to_height.get(txoutindex).unwrap();
// Get value and type from the output being spent
let value = txoutindex_to_value.read_unwrap(txoutindex, &ir.txoutindex_to_value);
let input_type =
txoutindex_to_outputtype.read_unwrap(txoutindex, &ir.txoutindex_to_outputtype);
// Non-address inputs don't need typeindex or address lookup
if input_type.is_not_address() {
return (txinindex, txoutindex, prev_height, value, input_type, None);
return (prev_height, value, input_type, None);
}
let typeindex =
txoutindex_to_typeindex.read_unwrap(txoutindex, &ir.txoutindex_to_typeindex);
let typeindex = *txinindex_to_typeindex.get(local_idx).unwrap();
// Look up address data
let addr_data_opt = load_uncached_address_data(
@@ -108,8 +82,6 @@ pub fn process_inputs(
);
(
txinindex,
txoutindex,
prev_height,
value,
input_type,
@@ -131,16 +103,13 @@ pub fn process_inputs(
AddressTypeToTypeIndexMap::<LoadedAddressDataWithSource>::with_capacity(estimated_per_type);
let mut txindex_vecs =
AddressTypeToTypeIndexMap::<TxIndexVec>::with_capacity(estimated_per_type);
let mut txoutindex_to_txinindex_updates = Vec::with_capacity(input_count);
for (txinindex, txoutindex, prev_height, value, output_type, addr_info) in items {
for (prev_height, value, output_type, addr_info) in items {
height_to_sent
.entry(prev_height)
.or_default()
.iterate(value, output_type);
txoutindex_to_txinindex_updates.push((txoutindex, txinindex));
if let Some((typeindex, txindex, value, addr_data_opt)) = addr_info {
sent_data
.entry(prev_height)
@@ -167,7 +136,5 @@ pub fn process_inputs(
sent_data,
address_data,
txindex_vecs,
txoutindex_to_txinindex_updates,
}
}

2
crates/brk_computer/src/stateful/process/mod.rs
View File

@@ -3,7 +3,6 @@ mod cache;
mod inputs;
mod lookup;
mod outputs;
mod range_map;
mod received;
mod sent;
mod tx_counts;
@@ -14,7 +13,6 @@ pub use cache::*;
pub use inputs::*;
pub use lookup::*;
pub use outputs::*;
pub use range_map::*;
pub use received::*;
pub use sent::*;
pub use tx_counts::*;

19
crates/brk_computer/src/stateful/process/outputs.rs
View File

@@ -5,7 +5,7 @@
//! - Address data for address cohort tracking (optional)
use brk_grouper::ByAddressType;
use brk_types::{OutputType, Sats, TxIndex, TypeIndex};
use brk_types::{Sats, TxIndex, TxOutData, TypeIndex};
use crate::stateful::address::{
AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs,
@@ -37,19 +37,16 @@ pub struct OutputsResult {
/// 4. Track address-specific data for address cohort processing
#[allow(clippy::too_many_arguments)]
pub fn process_outputs(
output_count: usize,
txoutindex_to_txindex: &[TxIndex],
// Pre-collected output data (from reusable iterators with 16KB buffered reads)
values: &[Sats],
output_types: &[OutputType],
typeindexes: &[TypeIndex],
// Address lookup parameters
txoutdata_vec: &[TxOutData],
first_addressindexes: &ByAddressType<TypeIndex>,
cache: &AddressCache,
vr: &VecsReaders,
any_address_indexes: &AnyAddressIndexesVecs,
addresses_data: &AddressesDataVecs,
) -> OutputsResult {
let output_count = txoutdata_vec.len();
// Pre-allocate result structures
let estimated_per_type = (output_count / 8).max(8);
let mut transacted = Transacted::default();
@@ -60,10 +57,10 @@ pub fn process_outputs(
AddressTypeToTypeIndexMap::<TxIndexVec>::with_capacity(estimated_per_type);
// Single pass: read from pre-collected vecs and accumulate
for local_idx in 0..output_count {
for (local_idx, txoutdata) in txoutdata_vec.iter().enumerate() {
let txindex = txoutindex_to_txindex[local_idx];
let value = values[local_idx];
let output_type = output_types[local_idx];
let value = txoutdata.value;
let output_type = txoutdata.outputtype;
transacted.iterate(value, output_type);
@@ -71,7 +68,7 @@ pub fn process_outputs(
continue;
}
let typeindex = typeindexes[local_idx];
let typeindex = txoutdata.typeindex;
received_data
.get_mut(output_type)

65
crates/brk_computer/src/stateful/process/range_map.rs
View File

@@ -1,65 +0,0 @@
//! Range-based lookup map.
//!
//! Maps ranges of indices to values for efficient reverse lookups.
use std::collections::BTreeMap;
use brk_types::{Height, TxOutIndex};
use vecdb::{BytesVec, BytesVecValue, PcoVec, PcoVecValue, VecIndex};
/// Maps ranges of indices to their corresponding height.
/// Used to efficiently look up which block a txoutindex belongs to.
#[derive(Debug)]
pub struct RangeMap<I, T>(BTreeMap<I, T>);
impl<I, T> RangeMap<I, T>
where
I: VecIndex,
T: VecIndex,
{
/// Look up value for a key using range search.
/// Returns the value associated with the largest key <= given key.
#[inline]
pub fn get(&self, key: I) -> Option<&T> {
self.0.range(..=key).next_back().map(|(_, value)| value)
}
}
impl<I, T> From<&BytesVec<I, T>> for RangeMap<T, I>
where
I: VecIndex,
T: VecIndex + BytesVecValue,
{
#[inline]
fn from(vec: &BytesVec<I, T>) -> Self {
Self(
vec.into_iter()
.enumerate()
.map(|(i, v)| (v, I::from(i)))
.collect(),
)
}
}
impl<I, T> From<&PcoVec<I, T>> for RangeMap<T, I>
where
I: VecIndex,
T: VecIndex + PcoVecValue,
{
#[inline]
fn from(vec: &PcoVec<I, T>) -> Self {
Self(
vec.into_iter()
.enumerate()
.map(|(i, v)| (v, I::from(i)))
.collect(),
)
}
}
/// Creates a RangeMap from height_to_first_txoutindex for fast txoutindex -> height lookups.
pub fn build_txoutindex_to_height_map(
height_to_first_txoutindex: &PcoVec<Height, TxOutIndex>,
) -> RangeMap<TxOutIndex, Height> {
RangeMap::from(height_to_first_txoutindex)
}

37
crates/brk_computer/src/stateful/vecs.rs
View File

@@ -7,11 +7,11 @@ use brk_indexer::Indexer;
use brk_traversable::Traversable;
use brk_types::{
Dollars, EmptyAddressData, EmptyAddressIndex, Height, LoadedAddressData, LoadedAddressIndex,
Sats, StoredU64, TxInIndex, TxOutIndex, Version,
Sats, StoredU64, Version,
};
use log::info;
use vecdb::{
AnyStoredVec, AnyVec, BytesVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec,
AnyVec, BytesVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec,
IterableCloneableVec, LazyVecFrom1, PAGE_SIZE, PcoVec, Stamp, TypedVecIterator, VecIndex,
};
@@ -47,7 +47,6 @@ pub struct Vecs {
// States
// ---
pub chain_state: BytesVec<Height, SupplyState>,
pub txoutindex_to_txinindex: BytesVec<TxOutIndex, TxInIndex>,
pub any_address_indexes: AnyAddressIndexesVecs,
pub addresses_data: AddressesDataVecs,
pub utxo_cohorts: UTXOCohorts,
@@ -126,10 +125,6 @@ impl Vecs {
vecdb::ImportOptions::new(&db, "chain", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
txoutindex_to_txinindex: BytesVec::forced_import_with(
vecdb::ImportOptions::new(&db, "txinindex", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
@@ -265,12 +260,13 @@ impl Vecs {
let stateful_min = utxo_min
.min(address_min)
.min(Height::from(self.chain_state.len()))
.min(Height::from(self.txoutindex_to_txinindex.stamp()).incremented())
.min(self.any_address_indexes.min_stamped_height())
.min(self.addresses_data.min_stamped_height())
.min(Height::from(self.height_to_unspendable_supply.len()))
.min(Height::from(self.height_to_opreturn_supply.len()))
.min(Height::from(self.addresstype_to_height_to_addr_count.min_len()))
.min(Height::from(
self.addresstype_to_height_to_addr_count.min_len(),
))
.min(Height::from(
self.addresstype_to_height_to_empty_addr_count.min_len(),
));
@@ -285,13 +281,11 @@ impl Vecs {
// Rollback BytesVec state and capture results for validation
let chain_state_rollback = self.chain_state.rollback_before(stamp);
let txoutindex_rollback = self.txoutindex_to_txinindex.rollback_before(stamp);
// Validate all rollbacks and imports are consistent
let recovered = recover_state(
height,
chain_state_rollback,
txoutindex_rollback,
&mut self.any_address_indexes,
&mut self.addresses_data,
&mut self.utxo_cohorts,
@@ -309,7 +303,6 @@ impl Vecs {
// Fresh start: reset all state
let (starting_height, mut chain_state) = if recovered_height.is_zero() {
self.chain_state.reset()?;
self.txoutindex_to_txinindex.reset()?;
self.height_to_unspendable_supply.reset()?;
self.height_to_opreturn_supply.reset()?;
self.addresstype_to_height_to_addr_count.reset()?;
@@ -505,24 +498,4 @@ impl Vecs {
self.db.compact()?;
Ok(())
}
/// Update txoutindex_to_txinindex for a block.
///
/// 1. Push UNSPENT for all new outputs in the block
/// 2. Update spent outputs with their spending txinindex
pub fn update_txoutindex_to_txinindex(
&mut self,
output_count: usize,
updates: Vec<(TxOutIndex, TxInIndex)>,
) -> Result<()> {
// Push UNSPENT for all new outputs in this block
for _ in 0..output_count {
self.txoutindex_to_txinindex.push(TxInIndex::UNSPENT);
}
// Update spent outputs with their spending txinindex
for (txoutindex, txinindex) in updates {
self.txoutindex_to_txinindex.update(txoutindex, txinindex)?;
}
Ok(())
}
}

9
crates/brk_error/src/lib.rs
View File

@@ -1,6 +1,6 @@
#![doc = include_str!("../README.md")]
use std::{io, result, time};
use std::{io, path::PathBuf, result, time};
use thiserror::Error;
@@ -123,6 +123,13 @@ pub enum Error {
#[error("Fetch failed after retries: {0}")]
FetchFailed(String),
#[error("Version mismatch at {path:?}: expected {expected}, found {found}")]
VersionMismatch {
path: PathBuf,
expected: usize,
found: usize,
},
}

2
crates/brk_grouper/src/by_type.rs
View File

@@ -25,7 +25,6 @@ impl<T> GroupedByType<T> {
OutputType::Empty => &self.spendable.empty,
OutputType::Unknown => &self.spendable.unknown,
OutputType::OpReturn => &self.unspendable.opreturn,
_ => unreachable!(),
}
}
@@ -43,7 +42,6 @@ impl<T> GroupedByType<T> {
OutputType::Unknown => &mut self.spendable.unknown,
OutputType::Empty => &mut self.spendable.empty,
OutputType::OpReturn => &mut self.unspendable.opreturn,
_ => unreachable!(),
}
}
}

2
crates/brk_indexer/examples/indexer_read.rs
View File

@@ -29,7 +29,7 @@ fn main() -> Result<()> {
indexer
.vecs
.txout
.txoutindex_to_value
.txoutindex_to_txoutdata
.iter()?
.enumerate()
.take(200)

5
crates/brk_indexer/examples/indexer_read_speed.rs
View File

@@ -13,9 +13,8 @@ fn run_benchmark(indexer: &Indexer) -> (Sats, std::time::Duration, usize) {
let mut sum = Sats::ZERO;
let mut count = 0;
for value in indexer.vecs.txout.txoutindex_to_value.clean_iter().unwrap() {
// for value in indexer.vecs.txoutindex_to_value.values() {
sum += value;
for txoutdata in indexer.vecs.txout.txoutindex_to_txoutdata.clean_iter().unwrap() {
sum += txoutdata.value;
count += 1;
}

2
crates/brk_indexer/src/constants.rs
View File

@@ -4,7 +4,7 @@ use brk_types::{Height, TxIndex, Txid, TxidPrefix, Version};
// One version for all data sources
// Increment on **change _OR_ addition**
pub const VERSION: Version = Version::new(23);
pub const VERSION: Version = Version::new(24);
pub const SNAPSHOT_BLOCK_RANGE: usize = 1_000;
pub const COLLISIONS_CHECKED_UP_TO: Height = Height::new(0);

3
crates/brk_indexer/src/indexes.rs
View File

@@ -45,7 +45,6 @@ impl Indexes {
OutputType::P2WPKH => *self.p2wpkhaddressindex,
OutputType::P2WSH => *self.p2wshaddressindex,
OutputType::Unknown => *self.unknownoutputindex,
_ => unreachable!(),
}
}
@@ -225,7 +224,7 @@ impl From<(Height, &mut Vecs, &Stores)> for Indexes {
let txoutindex = starting_index(
&vecs.txout.height_to_first_txoutindex,
&vecs.txout.txoutindex_to_value,
&vecs.txout.txoutindex_to_txoutdata,
height,
)
.unwrap();

30
crates/brk_indexer/src/lib.rs
View File

@@ -1,8 +1,8 @@
#![doc = include_str!("../README.md")]
use std::{path::Path, thread, time::Instant};
use std::{fs, path::Path, thread, time::Instant};
use brk_error::Result;
use brk_error::{Error, Result};
use brk_iterator::Blocks;
use brk_rpc::Client;
use brk_types::Height;
@@ -11,6 +11,7 @@ use vecdb::Exit;
mod constants;
mod indexes;
mod processor;
mod range_map;
mod readers;
mod stores;
mod vecs;
@@ -18,6 +19,7 @@ mod vecs;
use constants::*;
pub use indexes::*;
pub use processor::*;
pub use range_map::*;
pub use readers::*;
pub use stores::*;
pub use vecs::*;
@@ -30,6 +32,19 @@ pub struct Indexer {
impl Indexer {
pub fn forced_import(outputs_dir: &Path) -> Result<Self> {
match Self::forced_import_inner(outputs_dir) {
Ok(result) => Ok(result),
Err(Error::VersionMismatch { path, .. }) => {
let indexed_path = outputs_dir.join("indexed");
info!("Version mismatch at {path:?}, deleting {indexed_path:?} and retrying");
fs::remove_dir_all(&indexed_path)?;
Self::forced_import(outputs_dir)
}
Err(e) => Err(e),
}
}
fn forced_import_inner(outputs_dir: &Path) -> Result<Self> {
info!("Increasing number of open files limit...");
let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?;
rlimit::setrlimit(
@@ -129,6 +144,13 @@ impl Indexer {
let mut readers = Readers::new(&self.vecs);
// Build txindex -> height map from existing data for efficient lookups
let mut txindex_to_height = RangeMap::new();
for (height, first_txindex) in self.vecs.tx.height_to_first_txindex.into_iter().enumerate()
{
txindex_to_height.insert(first_txindex, Height::from(height));
}
let vecs = &mut self.vecs;
let stores = &mut self.stores;
@@ -139,6 +161,9 @@ impl Indexer {
indexes.height = height;
// Insert current block's first_txindex -> height before processing inputs
txindex_to_height.insert(indexes.txindex, height);
// Used to check rapidhash collisions
let block_check_collisions = check_collisions && height > COLLISIONS_CHECKED_UP_TO;
@@ -150,6 +175,7 @@ impl Indexer {
vecs,
stores,
readers: &readers,
txindex_to_height: &txindex_to_height,
};
// Phase 1: Process block metadata

717
crates/brk_indexer/src/processor.rs
View File

@@ -1,717 +0,0 @@
use bitcoin::{Transaction, TxIn, TxOut};
use brk_error::{Error, Result};
use brk_grouper::ByAddressType;
use brk_types::{
AddressBytes, AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, Block, BlockHashPrefix,
Height, OutPoint, OutputType, Sats, StoredBool, Timestamp, TxInIndex, TxIndex, TxOutIndex,
Txid, TxidPrefix, TypeIndex, Unit, Vin, Vout,
};
use log::error;
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use vecdb::{AnyVec, GenericStoredVec, TypedVecIterator, likely};
use crate::{Indexes, Readers, Stores, Vecs, constants::*};
/// Input source for tracking where an input came from.
#[derive(Debug)]
pub enum InputSource<'a> {
PreviousBlock {
vin: Vin,
txindex: TxIndex,
outpoint: OutPoint,
address_info: Option<(OutputType, TypeIndex)>,
},
SameBlock {
txindex: TxIndex,
txin: &'a TxIn,
vin: Vin,
outpoint: OutPoint,
},
}
/// Processed output data from parallel output processing.
pub struct ProcessedOutput<'a> {
pub txoutindex: TxOutIndex,
pub txout: &'a TxOut,
pub txindex: TxIndex,
pub vout: Vout,
pub outputtype: OutputType,
pub address_info: Option<(AddressBytes, AddressHash)>,
pub existing_typeindex: Option<TypeIndex>,
}
/// Computed transaction data from parallel TXID computation.
pub struct ComputedTx<'a> {
pub txindex: TxIndex,
pub tx: &'a Transaction,
pub txid: Txid,
pub txid_prefix: TxidPrefix,
pub prev_txindex_opt: Option<TxIndex>,
}
/// Processes a single block, extracting and storing all indexed data.
pub struct BlockProcessor<'a> {
pub block: &'a Block,
pub height: Height,
pub check_collisions: bool,
pub indexes: &'a mut Indexes,
pub vecs: &'a mut Vecs,
pub stores: &'a mut Stores,
pub readers: &'a Readers,
}
impl<'a> BlockProcessor<'a> {
/// Process block metadata (blockhash, difficulty, timestamp, etc.)
pub fn process_block_metadata(&mut self) -> Result<()> {
let height = self.height;
let blockhash = self.block.hash();
let blockhash_prefix = BlockHashPrefix::from(blockhash);
// Check for blockhash prefix collision
if self
.stores
.blockhashprefix_to_height
.get(&blockhash_prefix)?
.is_some_and(|prev_height| *prev_height != height)
{
error!("BlockHash: {blockhash}");
return Err(Error::Internal("BlockHash prefix collision"));
}
self.indexes.checked_push(self.vecs)?;
self.stores
.blockhashprefix_to_height
.insert_if_needed(blockhash_prefix, height, height);
self.stores.height_to_coinbase_tag.insert_if_needed(
height,
self.block.coinbase_tag().into(),
height,
);
self.vecs
.block
.height_to_blockhash
.checked_push(height, blockhash.clone())?;
self.vecs
.block
.height_to_difficulty
.checked_push(height, self.block.header.difficulty_float().into())?;
self.vecs
.block
.height_to_timestamp
.checked_push(height, Timestamp::from(self.block.header.time))?;
self.vecs
.block
.height_to_total_size
.checked_push(height, self.block.total_size().into())?;
self.vecs
.block
.height_to_weight
.checked_push(height, self.block.weight().into())?;
Ok(())
}
/// Compute TXIDs in parallel (CPU-intensive operation).
pub fn compute_txids(&self) -> Result<Vec<ComputedTx<'a>>> {
let will_check_collisions =
self.check_collisions && self.stores.txidprefix_to_txindex.needs(self.height);
let base_txindex = self.indexes.txindex;
self.block
.txdata
.par_iter()
.enumerate()
.map(|(index, tx)| {
let txid = Txid::from(tx.compute_txid());
let txid_prefix = TxidPrefix::from(&txid);
let prev_txindex_opt = if will_check_collisions {
self.stores
.txidprefix_to_txindex
.get(&txid_prefix)?
.map(|v| *v)
} else {
None
};
Ok(ComputedTx {
txindex: base_txindex + TxIndex::from(index),
tx,
txid,
txid_prefix,
prev_txindex_opt,
})
})
.collect()
}
/// Process inputs in parallel.
///
/// Uses collect().into_par_iter() pattern because:
/// 1. The inner work (store lookups, vector reads) is expensive
/// 2. We want to parallelize across ALL inputs, not just per-transaction
/// 3. The intermediate allocation (~8KB per block) is negligible compared to parallelism gains
pub fn process_inputs<'c>(
&self,
txs: &[ComputedTx<'c>],
) -> Result<Vec<(TxInIndex, InputSource<'a>)>> {
let txid_prefix_to_txindex: FxHashMap<_, _> =
txs.iter().map(|ct| (ct.txid_prefix, &ct.txindex)).collect();
let base_txindex = self.indexes.txindex;
let base_txinindex = self.indexes.txinindex;
let txins = self
.block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.input
.iter()
.enumerate()
.map(move |(vin, txin)| (TxIndex::from(index), Vin::from(vin), txin, tx))
})
.collect::<Vec<_>>()
.into_par_iter()
.enumerate()
.map(
|(block_txinindex, (block_txindex, vin, txin, tx))| -> Result<(TxInIndex, InputSource)> {
let txindex = base_txindex + block_txindex;
let txinindex = base_txinindex + TxInIndex::from(block_txinindex);
if tx.is_coinbase() {
return Ok((
txinindex,
InputSource::SameBlock {
txindex,
txin,
vin,
outpoint: OutPoint::COINBASE,
},
));
}
let outpoint = txin.previous_output;
let txid = Txid::from(outpoint.txid);
let txid_prefix = TxidPrefix::from(&txid);
let vout = Vout::from(outpoint.vout);
if let Some(&&same_block_txindex) = txid_prefix_to_txindex
.get(&txid_prefix) {
let outpoint = OutPoint::new(same_block_txindex, vout);
return Ok((
txinindex,
InputSource::SameBlock {
txindex,
txin,
vin,
outpoint,
},
));
}
let prev_txindex = if let Some(txindex) = self
.stores
.txidprefix_to_txindex
.get(&txid_prefix)?
.map(|v| *v)
.and_then(|txindex| {
(txindex < self.indexes.txindex).then_some(txindex)
})
{
txindex
} else {
return Err(Error::UnknownTxid);
};
let txoutindex = self
.vecs
.tx
.txindex_to_first_txoutindex
.get_pushed_or_read(prev_txindex, &self.readers.txindex_to_first_txoutindex)?
.ok_or(Error::Internal("Missing txoutindex"))?
+ vout;
let outpoint = OutPoint::new(prev_txindex, vout);
let outputtype = self
.vecs
.txout
.txoutindex_to_outputtype
.get_pushed_or_read(txoutindex, &self.readers.txoutindex_to_outputtype)?
.ok_or(Error::Internal("Missing outputtype"))?;
let address_info = if outputtype.is_address() {
let typeindex = self
.vecs
.txout
.txoutindex_to_typeindex
.get_pushed_or_read(txoutindex, &self.readers.txoutindex_to_typeindex)?
.ok_or(Error::Internal("Missing typeindex"))?;
Some((outputtype, typeindex))
} else {
None
};
Ok((
txinindex,
InputSource::PreviousBlock {
vin,
txindex,
outpoint,
address_info,
},
))
},
)
.collect::<Result<Vec<_>>>()?;
Ok(txins)
}
/// Collect same-block spent outpoints.
pub fn collect_same_block_spent_outpoints(
txins: &[(TxInIndex, InputSource)],
) -> FxHashSet<OutPoint> {
txins
.iter()
.filter_map(|(_, input_source)| {
let InputSource::SameBlock { outpoint, .. } = input_source else {
return None;
};
if !outpoint.is_coinbase() {
Some(*outpoint)
} else {
None
}
})
.collect()
}
/// Process outputs in parallel.
pub fn process_outputs(&self) -> Result<Vec<ProcessedOutput<'a>>> {
let height = self.height;
let check_collisions = self.check_collisions;
let base_txindex = self.indexes.txindex;
let base_txoutindex = self.indexes.txoutindex;
// Same pattern as inputs: collect then parallelize for maximum parallelism
self.block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.output
.iter()
.enumerate()
.map(move |(vout, txout)| (TxIndex::from(index), Vout::from(vout), txout, tx))
})
.collect::<Vec<_>>()
.into_par_iter()
.enumerate()
.map(
|(block_txoutindex, (block_txindex, vout, txout, tx))| -> Result<ProcessedOutput> {
let txindex = base_txindex + block_txindex;
let txoutindex = base_txoutindex + TxOutIndex::from(block_txoutindex);
let script = &txout.script_pubkey;
let outputtype = OutputType::from(script);
if outputtype.is_not_address() {
return Ok(ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info: None,
existing_typeindex: None,
});
}
let addresstype = outputtype;
let address_bytes = AddressBytes::try_from((script, addresstype)).unwrap();
let address_hash = AddressHash::from(&address_bytes);
let existing_typeindex = self
.stores
.addresstype_to_addresshash_to_addressindex
.get_unwrap(addresstype)
.get(&address_hash)
.unwrap()
.map(|v| *v)
.and_then(|typeindex_local| {
(typeindex_local < self.indexes.to_typeindex(addresstype))
.then_some(typeindex_local)
});
if check_collisions && let Some(typeindex) = existing_typeindex {
let prev_addressbytes_opt = self.vecs.get_addressbytes_by_type(
addresstype,
typeindex,
self.readers.addressbytes.get_unwrap(addresstype),
)?;
let prev_addressbytes = prev_addressbytes_opt
.as_ref()
.ok_or(Error::Internal("Missing addressbytes"))?;
if self
.stores
.addresstype_to_addresshash_to_addressindex
.get_unwrap(addresstype)
.needs(height)
&& prev_addressbytes != &address_bytes
{
let txid = tx.compute_txid();
dbg!(
height,
txid,
vout,
block_txindex,
addresstype,
prev_addressbytes,
&address_bytes,
&self.indexes,
typeindex,
txout,
AddressHash::from(&address_bytes),
);
panic!()
}
}
Ok(ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info: Some((address_bytes, address_hash)),
existing_typeindex,
})
},
)
.collect()
}
/// Finalize outputs sequentially (stores addresses, tracks UTXOs).
pub fn finalize_outputs(
&mut self,
txouts: Vec<ProcessedOutput>,
same_block_spent_outpoints: &FxHashSet<OutPoint>,
) -> Result<FxHashMap<OutPoint, (OutputType, TypeIndex)>> {
let height = self.height;
let mut already_added_addresshash: ByAddressType<FxHashMap<AddressHash, TypeIndex>> =
ByAddressType::default();
// Pre-size based on the number of same-block spent outpoints
let mut same_block_output_info: FxHashMap<OutPoint, (OutputType, TypeIndex)> =
FxHashMap::with_capacity_and_hasher(
same_block_spent_outpoints.len(),
Default::default(),
);
for ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info,
existing_typeindex,
} in txouts
{
let sats = Sats::from(txout.value);
if vout.is_zero() {
self.vecs
.tx
.txindex_to_first_txoutindex
.checked_push(txindex, txoutindex)?;
}
self.vecs
.txout
.txoutindex_to_value
.checked_push(txoutindex, sats)?;
self.vecs
.txout
.txoutindex_to_txindex
.checked_push(txoutindex, txindex)?;
self.vecs
.txout
.txoutindex_to_outputtype
.checked_push(txoutindex, outputtype)?;
let typeindex = if let Some(ti) = existing_typeindex {
ti
} else if let Some((address_bytes, address_hash)) = address_info {
let addresstype = outputtype;
if let Some(&ti) = already_added_addresshash
.get_unwrap(addresstype)
.get(&address_hash)
{
ti
} else {
let ti = self.indexes.increment_address_index(addresstype);
already_added_addresshash
.get_mut_unwrap(addresstype)
.insert(address_hash, ti);
self.stores
.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(addresstype)
.insert_if_needed(address_hash, ti, height);
self.vecs.push_bytes_if_needed(ti, address_bytes)?;
ti
}
} else {
match outputtype {
OutputType::P2MS => {
self.vecs
.output
.p2msoutputindex_to_txindex
.checked_push(self.indexes.p2msoutputindex, txindex)?;
self.indexes.p2msoutputindex.copy_then_increment()
}
OutputType::OpReturn => {
self.vecs
.output
.opreturnindex_to_txindex
.checked_push(self.indexes.opreturnindex, txindex)?;
self.indexes.opreturnindex.copy_then_increment()
}
OutputType::Empty => {
self.vecs
.output
.emptyoutputindex_to_txindex
.checked_push(self.indexes.emptyoutputindex, txindex)?;
self.indexes.emptyoutputindex.copy_then_increment()
}
OutputType::Unknown => {
self.vecs
.output
.unknownoutputindex_to_txindex
.checked_push(self.indexes.unknownoutputindex, txindex)?;
self.indexes.unknownoutputindex.copy_then_increment()
}
_ => unreachable!(),
}
};
self.vecs
.txout
.txoutindex_to_typeindex
.checked_push(txoutindex, typeindex)?;
if outputtype.is_unspendable() {
continue;
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.insert_if_needed(
AddressIndexTxIndex::from((addressindex, txindex)),
Unit,
height,
);
}
let outpoint = OutPoint::new(txindex, vout);
if same_block_spent_outpoints.contains(&outpoint) {
same_block_output_info.insert(outpoint, (outputtype, typeindex));
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.insert_if_needed(
AddressIndexOutPoint::from((addressindex, outpoint)),
Unit,
height,
);
}
}
Ok(same_block_output_info)
}
/// Finalize inputs sequentially (stores outpoints, updates address UTXOs).
pub fn finalize_inputs(
&mut self,
txins: Vec<(TxInIndex, InputSource)>,
same_block_output_info: &mut FxHashMap<OutPoint, (OutputType, TypeIndex)>,
) -> Result<()> {
let height = self.height;
for (txinindex, input_source) in txins {
let (vin, txindex, outpoint, address_info) = match input_source {
InputSource::PreviousBlock {
vin,
txindex,
outpoint,
address_info,
} => (vin, txindex, outpoint, address_info),
InputSource::SameBlock {
txindex,
txin,
vin,
outpoint,
} => {
if outpoint.is_coinbase() {
(vin, txindex, outpoint, None)
} else {
let outputtype_typeindex = same_block_output_info
.remove(&outpoint)
.ok_or(Error::Internal("Same-block addressindex not found"))
.inspect_err(|_| {
dbg!(&same_block_output_info, txin);
})?;
let address_info = if outputtype_typeindex.0.is_address() {
Some(outputtype_typeindex)
} else {
None
};
(vin, txindex, outpoint, address_info)
}
}
};
if vin.is_zero() {
self.vecs
.tx
.txindex_to_first_txinindex
.checked_push(txindex, txinindex)?;
}
self.vecs
.txin
.txinindex_to_txindex
.checked_push(txinindex, txindex)?;
self.vecs
.txin
.txinindex_to_outpoint
.checked_push(txinindex, outpoint)?;
let Some((addresstype, addressindex)) = address_info else {
continue;
};
self.stores
.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.insert_if_needed(
AddressIndexTxIndex::from((addressindex, txindex)),
Unit,
height,
);
self.stores
.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.remove_if_needed(AddressIndexOutPoint::from((addressindex, outpoint)), height);
}
Ok(())
}
/// Check for TXID collisions (only for known duplicate TXIDs).
pub fn check_txid_collisions(&self, txs: &[ComputedTx]) -> Result<()> {
if likely(!self.check_collisions) {
return Ok(());
}
let mut txindex_to_txid_iter = self.vecs.tx.txindex_to_txid.into_iter();
for ct in txs.iter() {
let Some(prev_txindex) = ct.prev_txindex_opt else {
continue;
};
// In case if we start at an already parsed height
if ct.txindex == prev_txindex {
continue;
}
let len = self.vecs.tx.txindex_to_txid.len();
let prev_txid = txindex_to_txid_iter
.get(prev_txindex)
.ok_or(Error::Internal("Missing txid for txindex"))
.inspect_err(|_| {
dbg!(ct.txindex, len);
})?;
let is_dup = DUPLICATE_TXIDS.contains(&prev_txid);
if !is_dup {
dbg!(self.height, ct.txindex, prev_txid, prev_txindex);
return Err(Error::Internal("Unexpected TXID collision"));
}
}
Ok(())
}
/// Store transaction metadata.
pub fn store_transaction_metadata(&mut self, txs: Vec<ComputedTx>) -> Result<()> {
let height = self.height;
for ct in txs {
if ct.prev_txindex_opt.is_none() {
self.stores.txidprefix_to_txindex.insert_if_needed(
ct.txid_prefix,
ct.txindex,
height,
);
}
self.vecs
.tx
.txindex_to_height
.checked_push(ct.txindex, height)?;
self.vecs
.tx
.txindex_to_txversion
.checked_push(ct.txindex, ct.tx.version.into())?;
self.vecs
.tx
.txindex_to_txid
.checked_push(ct.txindex, ct.txid)?;
self.vecs
.tx
.txindex_to_rawlocktime
.checked_push(ct.txindex, ct.tx.lock_time.into())?;
self.vecs
.tx
.txindex_to_base_size
.checked_push(ct.txindex, ct.tx.base_size().into())?;
self.vecs
.tx
.txindex_to_total_size
.checked_push(ct.txindex, ct.tx.total_size().into())?;
self.vecs
.tx
.txindex_to_is_explicitly_rbf
.checked_push(ct.txindex, StoredBool::from(ct.tx.is_explicitly_rbf()))?;
}
Ok(())
}
/// Update global indexes after processing a block.
pub fn update_indexes(&mut self, tx_count: usize, input_count: usize, output_count: usize) {
self.indexes.txindex += TxIndex::from(tx_count);
self.indexes.txinindex += TxInIndex::from(input_count);
self.indexes.txoutindex += TxOutIndex::from(output_count);
}
}

63
crates/brk_indexer/src/processor/metadata.rs Normal file
View File

@@ -0,0 +1,63 @@
//! Block metadata processing.
use brk_error::{Error, Result};
use brk_types::{BlockHashPrefix, Timestamp};
use log::error;
use vecdb::GenericStoredVec;
use super::BlockProcessor;
impl BlockProcessor<'_> {
/// Process block metadata (blockhash, difficulty, timestamp, etc.)
pub fn process_block_metadata(&mut self) -> Result<()> {
let height = self.height;
let blockhash = self.block.hash();
let blockhash_prefix = BlockHashPrefix::from(blockhash);
// Check for blockhash prefix collision
if self
.stores
.blockhashprefix_to_height
.get(&blockhash_prefix)?
.is_some_and(|prev_height| *prev_height != height)
{
error!("BlockHash: {blockhash}");
return Err(Error::Internal("BlockHash prefix collision"));
}
self.indexes.checked_push(self.vecs)?;
self.stores
.blockhashprefix_to_height
.insert_if_needed(blockhash_prefix, height, height);
self.stores.height_to_coinbase_tag.insert_if_needed(
height,
self.block.coinbase_tag().into(),
height,
);
self.vecs
.block
.height_to_blockhash
.checked_push(height, blockhash.clone())?;
self.vecs
.block
.height_to_difficulty
.checked_push(height, self.block.header.difficulty_float().into())?;
self.vecs
.block
.height_to_timestamp
.checked_push(height, Timestamp::from(self.block.header.time))?;
self.vecs
.block
.height_to_total_size
.checked_push(height, self.block.total_size().into())?;
self.vecs
.block
.height_to_weight
.checked_push(height, self.block.weight().into())?;
Ok(())
}
}

37
crates/brk_indexer/src/processor/mod.rs Normal file
View File

@@ -0,0 +1,37 @@
//! Block processing for indexing.
//!
//! This module handles the extraction and storage of all indexed data from blocks.
//! Processing is split into phases that can be parallelized where possible.
mod metadata;
mod tx;
mod txin;
mod txout;
mod types;
pub use types::*;
use brk_types::{Block, Height, TxInIndex, TxIndex, TxOutIndex};
use crate::{Indexes, RangeMap, Readers, Stores, Vecs};
/// Processes a single block, extracting and storing all indexed data.
pub struct BlockProcessor<'a> {
pub block: &'a Block,
pub height: Height,
pub check_collisions: bool,
pub indexes: &'a mut Indexes,
pub vecs: &'a mut Vecs,
pub stores: &'a mut Stores,
pub readers: &'a Readers,
pub txindex_to_height: &'a RangeMap<TxIndex, Height>,
}
impl BlockProcessor<'_> {
/// Update global indexes after processing a block.
pub fn update_indexes(&mut self, tx_count: usize, input_count: usize, output_count: usize) {
self.indexes.txindex += TxIndex::from(tx_count);
self.indexes.txinindex += TxInIndex::from(input_count);
self.indexes.txoutindex += TxOutIndex::from(output_count);
}
}

128
crates/brk_indexer/src/processor/tx.rs Normal file
View File

@@ -0,0 +1,128 @@
//! TXID computation and collision checking.
use brk_error::{Error, Result};
use brk_types::{StoredBool, TxIndex, Txid, TxidPrefix};
use rayon::prelude::*;
use vecdb::{AnyVec, GenericStoredVec, TypedVecIterator, likely};
use crate::constants::DUPLICATE_TXIDS;
use super::{BlockProcessor, ComputedTx};
impl<'a> BlockProcessor<'a> {
/// Compute TXIDs in parallel (CPU-intensive operation).
pub fn compute_txids(&self) -> Result<Vec<ComputedTx<'a>>> {
let will_check_collisions =
self.check_collisions && self.stores.txidprefix_to_txindex.needs(self.height);
let base_txindex = self.indexes.txindex;
self.block
.txdata
.par_iter()
.enumerate()
.map(|(index, tx)| {
let txid = Txid::from(tx.compute_txid());
let txid_prefix = TxidPrefix::from(&txid);
let prev_txindex_opt = if will_check_collisions {
self.stores
.txidprefix_to_txindex
.get(&txid_prefix)?
.map(|v| *v)
} else {
None
};
Ok(ComputedTx {
txindex: base_txindex + TxIndex::from(index),
tx,
txid,
txid_prefix,
prev_txindex_opt,
})
})
.collect()
}
/// Check for TXID collisions (only for known duplicate TXIDs).
pub fn check_txid_collisions(&self, txs: &[ComputedTx]) -> Result<()> {
if likely(!self.check_collisions) {
return Ok(());
}
let mut txindex_to_txid_iter = self.vecs.tx.txindex_to_txid.into_iter();
for ct in txs.iter() {
let Some(prev_txindex) = ct.prev_txindex_opt else {
continue;
};
// In case if we start at an already parsed height
if ct.txindex == prev_txindex {
continue;
}
let len = self.vecs.tx.txindex_to_txid.len();
let prev_txid = txindex_to_txid_iter
.get(prev_txindex)
.ok_or(Error::Internal("Missing txid for txindex"))
.inspect_err(|_| {
dbg!(ct.txindex, len);
})?;
let is_dup = DUPLICATE_TXIDS.contains(&prev_txid);
if !is_dup {
dbg!(self.height, ct.txindex, prev_txid, prev_txindex);
return Err(Error::Internal("Unexpected TXID collision"));
}
}
Ok(())
}
/// Store transaction metadata.
pub fn store_transaction_metadata(&mut self, txs: Vec<ComputedTx>) -> Result<()> {
let height = self.height;
for ct in txs {
if ct.prev_txindex_opt.is_none() {
self.stores.txidprefix_to_txindex.insert_if_needed(
ct.txid_prefix,
ct.txindex,
height,
);
}
self.vecs
.tx
.txindex_to_height
.checked_push(ct.txindex, height)?;
self.vecs
.tx
.txindex_to_txversion
.checked_push(ct.txindex, ct.tx.version.into())?;
self.vecs
.tx
.txindex_to_txid
.checked_push(ct.txindex, ct.txid)?;
self.vecs
.tx
.txindex_to_rawlocktime
.checked_push(ct.txindex, ct.tx.lock_time.into())?;
self.vecs
.tx
.txindex_to_base_size
.checked_push(ct.txindex, ct.tx.base_size().into())?;
self.vecs
.tx
.txindex_to_total_size
.checked_push(ct.txindex, ct.tx.total_size().into())?;
self.vecs
.tx
.txindex_to_is_explicitly_rbf
.checked_push(ct.txindex, StoredBool::from(ct.tx.is_explicitly_rbf()))?;
}
Ok(())
}
}

284
crates/brk_indexer/src/processor/txin.rs Normal file
View File

@@ -0,0 +1,284 @@
//! Input processing for block indexing.
use brk_error::{Error, Result};
use brk_types::{
AddressIndexOutPoint, AddressIndexTxIndex, OutPoint, OutputType, Sats, TxInIndex, TxIndex,
TxOutIndex, Txid, TxidPrefix, TypeIndex, Unit, Vin, Vout,
};
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use vecdb::GenericStoredVec;
use super::{BlockProcessor, ComputedTx, InputSource, SameBlockOutputInfo};
impl<'a> BlockProcessor<'a> {
/// Process inputs in parallel.
///
/// Uses collect().into_par_iter() pattern because:
/// 1. The inner work (store lookups, vector reads) is expensive
/// 2. We want to parallelize across ALL inputs, not just per-transaction
/// 3. The intermediate allocation (~8KB per block) is negligible compared to parallelism gains
pub fn process_inputs<'c>(
&self,
txs: &[ComputedTx<'c>],
) -> Result<Vec<(TxInIndex, InputSource<'a>)>> {
let txid_prefix_to_txindex: FxHashMap<_, _> =
txs.iter().map(|ct| (ct.txid_prefix, &ct.txindex)).collect();
let base_txindex = self.indexes.txindex;
let base_txinindex = self.indexes.txinindex;
let txins = self
.block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.input
.iter()
.enumerate()
.map(move |(vin, txin)| (TxIndex::from(index), Vin::from(vin), txin, tx))
})
.collect::<Vec<_>>()
.into_par_iter()
.enumerate()
.map(
|(block_txinindex, (block_txindex, vin, txin, tx))| -> Result<(TxInIndex, InputSource)> {
let txindex = base_txindex + block_txindex;
let txinindex = base_txinindex + TxInIndex::from(block_txinindex);
if tx.is_coinbase() {
return Ok((
txinindex,
InputSource::SameBlock {
txindex,
txin,
vin,
outpoint: OutPoint::COINBASE,
},
));
}
let outpoint = txin.previous_output;
let txid = Txid::from(outpoint.txid);
let txid_prefix = TxidPrefix::from(&txid);
let vout = Vout::from(outpoint.vout);
if let Some(&&same_block_txindex) = txid_prefix_to_txindex
.get(&txid_prefix) {
let outpoint = OutPoint::new(same_block_txindex, vout);
return Ok((
txinindex,
InputSource::SameBlock {
txindex,
txin,
vin,
outpoint,
},
));
}
let prev_txindex = if let Some(txindex) = self
.stores
.txidprefix_to_txindex
.get(&txid_prefix)?
.map(|v| *v)
.and_then(|txindex| {
(txindex < self.indexes.txindex).then_some(txindex)
})
{
txindex
} else {
return Err(Error::UnknownTxid);
};
let txoutindex = self
.vecs
.tx
.txindex_to_first_txoutindex
.get_pushed_or_read(prev_txindex, &self.readers.txindex_to_first_txoutindex)?
.ok_or(Error::Internal("Missing txoutindex"))?
+ vout;
let outpoint = OutPoint::new(prev_txindex, vout);
let txoutdata = self
.vecs
.txout
.txoutindex_to_txoutdata
.get_pushed_or_read(txoutindex, &self.readers.txoutindex_to_txoutdata)?
.ok_or(Error::Internal("Missing txout data"))?;
let value = txoutdata.value;
let outputtype = txoutdata.outputtype;
let typeindex = txoutdata.typeindex;
let height = self
.txindex_to_height
.get(prev_txindex)
.ok_or(Error::Internal("Missing height in txindex_to_height map"))?;
Ok((
txinindex,
InputSource::PreviousBlock {
vin,
value,
height,
txindex,
txoutindex,
outpoint,
outputtype,
typeindex,
},
))
},
)
.collect::<Result<Vec<_>>>()?;
Ok(txins)
}
/// Collect same-block spent outpoints.
pub fn collect_same_block_spent_outpoints(
txins: &[(TxInIndex, InputSource)],
) -> FxHashSet<OutPoint> {
txins
.iter()
.filter_map(|(_, input_source)| {
let InputSource::SameBlock { outpoint, .. } = input_source else {
return None;
};
if !outpoint.is_coinbase() {
Some(*outpoint)
} else {
None
}
})
.collect()
}
/// Finalize inputs sequentially (stores outpoints, updates address UTXOs).
pub fn finalize_inputs(
&mut self,
txins: Vec<(TxInIndex, InputSource)>,
same_block_output_info: &mut FxHashMap<OutPoint, SameBlockOutputInfo>,
) -> Result<()> {
let height = self.height;
for (txinindex, input_source) in txins {
let (prev_height, vin, txindex, value, outpoint, txoutindex, outputtype, typeindex) =
match input_source {
InputSource::PreviousBlock {
height,
vin,
txindex,
txoutindex,
value,
outpoint,
outputtype,
typeindex,
} => (
height, vin, txindex, value, outpoint, txoutindex, outputtype, typeindex,
),
InputSource::SameBlock {
txindex,
txin,
vin,
outpoint,
} => {
if outpoint.is_coinbase() {
(
height,
vin,
txindex,
Sats::COINBASE,
outpoint,
TxOutIndex::COINBASE,
OutputType::Unknown,
TypeIndex::COINBASE,
)
} else {
let info = same_block_output_info
.remove(&outpoint)
.ok_or(Error::Internal("Same-block output not found"))
.inspect_err(|_| {
dbg!(&same_block_output_info, txin);
})?;
(
height,
vin,
txindex,
info.value,
outpoint,
info.txoutindex,
info.outputtype,
info.typeindex,
)
}
}
};
if vin.is_zero() {
self.vecs
.tx
.txindex_to_first_txinindex
.checked_push(txindex, txinindex)?;
}
self.vecs
.txin
.txinindex_to_txindex
.checked_push(txinindex, txindex)?;
self.vecs
.txin
.txinindex_to_outpoint
.checked_push(txinindex, outpoint)?;
self.vecs
.txin
.txinindex_to_value
.checked_push(txinindex, value)?;
self.vecs
.txin
.txinindex_to_prev_height
.checked_push(txinindex, prev_height)?;
self.vecs
.txin
.txinindex_to_outputtype
.checked_push(txinindex, outputtype)?;
self.vecs
.txin
.txinindex_to_typeindex
.checked_push(txinindex, typeindex)?;
// Update txoutindex_to_txinindex for non-coinbase inputs
if !txoutindex.is_coinbase() {
self.vecs
.txout
.txoutindex_to_txinindex
.update(txoutindex, txinindex)?;
}
if !outputtype.is_address() {
continue;
}
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.insert_if_needed(
AddressIndexTxIndex::from((addressindex, txindex)),
Unit,
height,
);
self.stores
.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.remove_if_needed(AddressIndexOutPoint::from((addressindex, outpoint)), height);
}
Ok(())
}
}

275
crates/brk_indexer/src/processor/txout.rs Normal file
View File

@@ -0,0 +1,275 @@
//! Output processing for block indexing.
use brk_error::{Error, Result};
use brk_grouper::ByAddressType;
use brk_types::{
AddressBytes, AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, OutPoint, OutputType,
Sats, TxInIndex, TxIndex, TxOutData, TxOutIndex, TypeIndex, Unit, Vout,
};
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use vecdb::GenericStoredVec;
use super::{BlockProcessor, ProcessedOutput, SameBlockOutputInfo};
impl<'a> BlockProcessor<'a> {
/// Process outputs in parallel.
pub fn process_outputs(&self) -> Result<Vec<ProcessedOutput<'a>>> {
let height = self.height;
let check_collisions = self.check_collisions;
let base_txindex = self.indexes.txindex;
let base_txoutindex = self.indexes.txoutindex;
// Same pattern as inputs: collect then parallelize for maximum parallelism
self.block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.output
.iter()
.enumerate()
.map(move |(vout, txout)| (TxIndex::from(index), Vout::from(vout), txout, tx))
})
.collect::<Vec<_>>()
.into_par_iter()
.enumerate()
.map(
|(block_txoutindex, (block_txindex, vout, txout, tx))| -> Result<ProcessedOutput> {
let txindex = base_txindex + block_txindex;
let txoutindex = base_txoutindex + TxOutIndex::from(block_txoutindex);
let script = &txout.script_pubkey;
let outputtype = OutputType::from(script);
if outputtype.is_not_address() {
return Ok(ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info: None,
existing_typeindex: None,
});
}
let addresstype = outputtype;
let address_bytes = AddressBytes::try_from((script, addresstype)).unwrap();
let address_hash = AddressHash::from(&address_bytes);
let existing_typeindex = self
.stores
.addresstype_to_addresshash_to_addressindex
.get_unwrap(addresstype)
.get(&address_hash)
.unwrap()
.map(|v| *v)
.and_then(|typeindex_local| {
(typeindex_local < self.indexes.to_typeindex(addresstype))
.then_some(typeindex_local)
});
if check_collisions && let Some(typeindex) = existing_typeindex {
let prev_addressbytes_opt = self.vecs.get_addressbytes_by_type(
addresstype,
typeindex,
self.readers.addressbytes.get_unwrap(addresstype),
)?;
let prev_addressbytes = prev_addressbytes_opt
.as_ref()
.ok_or(Error::Internal("Missing addressbytes"))?;
if self
.stores
.addresstype_to_addresshash_to_addressindex
.get_unwrap(addresstype)
.needs(height)
&& prev_addressbytes != &address_bytes
{
let txid = tx.compute_txid();
dbg!(
height,
txid,
vout,
block_txindex,
addresstype,
prev_addressbytes,
&address_bytes,
&self.indexes,
typeindex,
txout,
AddressHash::from(&address_bytes),
);
panic!()
}
}
Ok(ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info: Some((address_bytes, address_hash)),
existing_typeindex,
})
},
)
.collect()
}
/// Finalize outputs sequentially (stores addresses, tracks UTXOs).
pub fn finalize_outputs(
&mut self,
txouts: Vec<ProcessedOutput>,
same_block_spent_outpoints: &FxHashSet<OutPoint>,
) -> Result<FxHashMap<OutPoint, SameBlockOutputInfo>> {
let height = self.height;
let mut already_added_addresshash: ByAddressType<FxHashMap<AddressHash, TypeIndex>> =
ByAddressType::default();
// Pre-size based on the number of same-block spent outpoints
let mut same_block_output_info: FxHashMap<OutPoint, SameBlockOutputInfo> =
FxHashMap::with_capacity_and_hasher(
same_block_spent_outpoints.len(),
Default::default(),
);
for ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info,
existing_typeindex,
} in txouts
{
let sats = Sats::from(txout.value);
if vout.is_zero() {
self.vecs
.tx
.txindex_to_first_txoutindex
.checked_push(txindex, txoutindex)?;
}
self.vecs
.txout
.txoutindex_to_txindex
.checked_push(txoutindex, txindex)?;
let typeindex = if let Some(ti) = existing_typeindex {
ti
} else if let Some((address_bytes, address_hash)) = address_info {
let addresstype = outputtype;
if let Some(&ti) = already_added_addresshash
.get_unwrap(addresstype)
.get(&address_hash)
{
ti
} else {
let ti = self.indexes.increment_address_index(addresstype);
already_added_addresshash
.get_mut_unwrap(addresstype)
.insert(address_hash, ti);
self.stores
.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(addresstype)
.insert_if_needed(address_hash, ti, height);
self.vecs.push_bytes_if_needed(ti, address_bytes)?;
ti
}
} else {
match outputtype {
OutputType::P2MS => {
self.vecs
.output
.p2msoutputindex_to_txindex
.checked_push(self.indexes.p2msoutputindex, txindex)?;
self.indexes.p2msoutputindex.copy_then_increment()
}
OutputType::OpReturn => {
self.vecs
.output
.opreturnindex_to_txindex
.checked_push(self.indexes.opreturnindex, txindex)?;
self.indexes.opreturnindex.copy_then_increment()
}
OutputType::Empty => {
self.vecs
.output
.emptyoutputindex_to_txindex
.checked_push(self.indexes.emptyoutputindex, txindex)?;
self.indexes.emptyoutputindex.copy_then_increment()
}
OutputType::Unknown => {
self.vecs
.output
.unknownoutputindex_to_txindex
.checked_push(self.indexes.unknownoutputindex, txindex)?;
self.indexes.unknownoutputindex.copy_then_increment()
}
_ => unreachable!(),
}
};
let txoutdata = TxOutData::new(sats, outputtype, typeindex);
self.vecs
.txout
.txoutindex_to_txoutdata
.checked_push(txoutindex, txoutdata)?;
self.vecs
.txout
.txoutindex_to_txinindex
.checked_push(txoutindex, TxInIndex::UNSPENT)?;
if outputtype.is_unspendable() {
continue;
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.insert_if_needed(
AddressIndexTxIndex::from((addressindex, txindex)),
Unit,
height,
);
}
let outpoint = OutPoint::new(txindex, vout);
if same_block_spent_outpoints.contains(&outpoint) {
same_block_output_info.insert(
outpoint,
SameBlockOutputInfo {
outputtype,
typeindex,
value: sats,
txoutindex,
},
);
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.insert_if_needed(
AddressIndexOutPoint::from((addressindex, outpoint)),
Unit,
height,
);
}
}
Ok(same_block_output_info)
}
}

57
crates/brk_indexer/src/processor/types.rs Normal file
View File

@@ -0,0 +1,57 @@
//! Type definitions for block processing.
use bitcoin::{Transaction, TxIn, TxOut};
use brk_types::{
AddressBytes, AddressHash, Height, OutPoint, OutputType, Sats, TxIndex, TxOutIndex, Txid,
TxidPrefix, TypeIndex, Vin, Vout,
};
/// Input source for tracking where an input came from.
#[derive(Debug)]
pub enum InputSource<'a> {
PreviousBlock {
vin: Vin,
value: Sats,
height: Height,
txindex: TxIndex,
txoutindex: TxOutIndex,
outpoint: OutPoint,
outputtype: OutputType,
typeindex: TypeIndex,
},
SameBlock {
txindex: TxIndex,
txin: &'a TxIn,
vin: Vin,
outpoint: OutPoint,
},
}
/// Output info for same-block spends (output created and spent in the same block).
#[derive(Debug, Clone, Copy)]
pub struct SameBlockOutputInfo {
pub outputtype: OutputType,
pub typeindex: TypeIndex,
pub value: Sats,
pub txoutindex: TxOutIndex,
}
/// Processed output data from parallel output processing.
pub struct ProcessedOutput<'a> {
pub txoutindex: TxOutIndex,
pub txout: &'a TxOut,
pub txindex: TxIndex,
pub vout: Vout,
pub outputtype: OutputType,
pub address_info: Option<(AddressBytes, AddressHash)>,
pub existing_typeindex: Option<TypeIndex>,
}
/// Computed transaction data from parallel TXID computation.
pub struct ComputedTx<'a> {
pub txindex: TxIndex,
pub tx: &'a Transaction,
pub txid: Txid,
pub txid_prefix: TxidPrefix,
pub prev_txindex_opt: Option<TxIndex>,
}

40
crates/brk_indexer/src/range_map.rs Normal file
View File

@@ -0,0 +1,40 @@
//! Range-based lookup map for efficient index -> value lookups.
//!
//! Uses the pattern that many indices share the same value (e.g., all txindexes
//! in a block have the same height) to provide O(log n) lookups via BTreeMap.
use std::collections::BTreeMap;
use vecdb::VecIndex;
/// Maps ranges of indices to values for efficient reverse lookups.
///
/// Instead of storing a value for every index, stores (first_index, value)
/// pairs and uses range search to find the value for any index.
#[derive(Debug, Default)]
pub struct RangeMap<I, V>(BTreeMap<I, V>);
impl<I: VecIndex, V: Copy> RangeMap<I, V> {
/// Create a new empty map.
pub fn new() -> Self {
Self(BTreeMap::new())
}
/// Insert a new (first_index, value) mapping.
#[inline]
pub fn insert(&mut self, first_index: I, value: V) {
self.0.insert(first_index, value);
}
/// Look up value for an index using range search.
/// Returns the value associated with the largest first_index <= given index.
#[inline]
pub fn get(&self, index: I) -> Option<V> {
self.0.range(..=index).next_back().map(|(_, &v)| v)
}
/// Clear all entries (for reset/rollback).
pub fn clear(&mut self) {
self.0.clear();
}
}

21
crates/brk_indexer/src/readers.rs
View File

@@ -7,8 +7,7 @@ use crate::Vecs;
/// These provide consistent snapshots for reading while the main vectors are being modified.
pub struct Readers {
pub txindex_to_first_txoutindex: Reader,
pub txoutindex_to_outputtype: Reader,
pub txoutindex_to_typeindex: Reader,
pub txoutindex_to_txoutdata: Reader,
pub addressbytes: ByAddressType<Reader>,
}
@@ -16,14 +15,22 @@ impl Readers {
pub fn new(vecs: &Vecs) -> Self {
Self {
txindex_to_first_txoutindex: vecs.tx.txindex_to_first_txoutindex.create_reader(),
txoutindex_to_outputtype: vecs.txout.txoutindex_to_outputtype.create_reader(),
txoutindex_to_typeindex: vecs.txout.txoutindex_to_typeindex.create_reader(),
txoutindex_to_txoutdata: vecs.txout.txoutindex_to_txoutdata.create_reader(),
addressbytes: ByAddressType {
p2pk65: vecs.address.p2pk65addressindex_to_p2pk65bytes.create_reader(),
p2pk33: vecs.address.p2pk33addressindex_to_p2pk33bytes.create_reader(),
p2pk65: vecs
.address
.p2pk65addressindex_to_p2pk65bytes
.create_reader(),
p2pk33: vecs
.address
.p2pk33addressindex_to_p2pk33bytes
.create_reader(),
p2pkh: vecs.address.p2pkhaddressindex_to_p2pkhbytes.create_reader(),
p2sh: vecs.address.p2shaddressindex_to_p2shbytes.create_reader(),
p2wpkh: vecs.address.p2wpkhaddressindex_to_p2wpkhbytes.create_reader(),
p2wpkh: vecs
.address
.p2wpkhaddressindex_to_p2wpkhbytes
.create_reader(),
p2wsh: vecs.address.p2wshaddressindex_to_p2wshbytes.create_reader(),
p2tr: vecs.address.p2traddressindex_to_p2trbytes.create_reader(),
p2a: vecs.address.p2aaddressindex_to_p2abytes.create_reader(),

84
crates/brk_indexer/src/stores.rs
View File

@@ -5,7 +5,8 @@ use brk_grouper::ByAddressType;
use brk_store::{AnyStore, Kind, Mode, Store};
use brk_types::{
AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, BlockHashPrefix, Height, OutPoint,
OutputType, StoredString, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, Unit, Version, Vout,
OutputType, StoredString, TxInIndex, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, Unit, Version,
Vout,
};
use fjall::{Database, PersistMode};
use log::info;
@@ -270,20 +271,14 @@ impl Stores {
let mut txindex_to_first_txoutindex_iter =
vecs.tx.txindex_to_first_txoutindex.iter()?;
vecs.txout
.txoutindex_to_outputtype
.txoutindex_to_txoutdata
.iter()?
.enumerate()
.skip(starting_indexes.txoutindex.to_usize())
.zip(
vecs.txout
.txoutindex_to_typeindex
.iter()?
.skip(starting_indexes.txoutindex.to_usize()),
)
.filter(|((_, outputtype), _): &((usize, OutputType), TypeIndex)| {
outputtype.is_address()
})
.for_each(|((txoutindex, addresstype), addressindex)| {
.filter(|(_, txoutdata)| txoutdata.outputtype.is_address())
.for_each(|(txoutindex, txoutdata)| {
let addresstype = txoutdata.outputtype;
let addressindex = txoutdata.typeindex;
let txindex = txoutindex_to_txindex_iter.get_at_unwrap(txoutindex);
self.addresstype_to_addressindex_and_txindex
@@ -303,20 +298,22 @@ impl Stores {
.remove(AddressIndexOutPoint::from((addressindex, outpoint)));
});
// Add back outputs that were spent after the rollback point
// Collect outputs that were spent after the rollback point
// We need to: 1) reset their spend status, 2) restore address stores
let mut txindex_to_first_txoutindex_iter =
vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut txoutindex_to_outputtype_iter = vecs.txout.txoutindex_to_outputtype.iter()?;
let mut txoutindex_to_typeindex_iter = vecs.txout.txoutindex_to_typeindex.iter()?;
let mut txoutindex_to_txoutdata_iter = vecs.txout.txoutindex_to_txoutdata.iter()?;
let mut txinindex_to_txindex_iter = vecs.txin.txinindex_to_txindex.iter()?;
vecs.txin
let outputs_to_unspend: Vec<_> = vecs
.txin
.txinindex_to_outpoint
.iter()?
.enumerate()
.skip(starting_indexes.txinindex.to_usize())
.for_each(|(txinindex, outpoint): (usize, OutPoint)| {
.filter_map(|(txinindex, outpoint): (usize, OutPoint)| {
if outpoint.is_coinbase() {
return;
return None;
}
let output_txindex = outpoint.txindex();
@@ -328,29 +325,38 @@ impl Stores {
// Only process if this output was created before the rollback point
if txoutindex < starting_indexes.txoutindex {
let outputtype = txoutindex_to_outputtype_iter.get_unwrap(txoutindex);
let txoutdata = txoutindex_to_txoutdata_iter.get_unwrap(txoutindex);
let spending_txindex =
txinindex_to_txindex_iter.get_at_unwrap(txinindex);
if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = txoutindex_to_typeindex_iter.get_unwrap(txoutindex);
// Get the SPENDING tx's index (not the output's tx)
let spending_txindex =
txinindex_to_txindex_iter.get_at_unwrap(txinindex);
self.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.remove(AddressIndexTxIndex::from((
addressindex,
spending_txindex,
)));
self.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.insert(AddressIndexOutPoint::from((addressindex, outpoint)), Unit);
}
Some((txoutindex, outpoint, txoutdata, spending_txindex))
} else {
None
}
});
})
.collect();
// Now process the collected outputs (iterators dropped, can mutate vecs)
for (txoutindex, outpoint, txoutdata, spending_txindex) in outputs_to_unspend {
// Reset spend status back to unspent
vecs.txout
.txoutindex_to_txinindex
.update(txoutindex, TxInIndex::UNSPENT)?;
// Restore address stores if this is an address output
if txoutdata.outputtype.is_address() {
let addresstype = txoutdata.outputtype;
let addressindex = txoutdata.typeindex;
self.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.remove(AddressIndexTxIndex::from((addressindex, spending_txindex)));
self.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.insert(AddressIndexOutPoint::from((addressindex, outpoint)), Unit);
}
}
} else {
unreachable!();
}

101
crates/brk_indexer/src/vecs/address.rs
View File

@@ -12,6 +12,8 @@ use vecdb::{
TypedVecIterator,
};
use crate::parallel_import;
#[derive(Clone, Traversable)]
pub struct AddressVecs {
// Height to first address index (per address type)
@@ -36,55 +38,58 @@ pub struct AddressVecs {
impl AddressVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
height_to_first_p2pk65addressindex,
height_to_first_p2pk33addressindex,
height_to_first_p2pkhaddressindex,
height_to_first_p2shaddressindex,
height_to_first_p2wpkhaddressindex,
height_to_first_p2wshaddressindex,
height_to_first_p2traddressindex,
height_to_first_p2aaddressindex,
p2pk65addressindex_to_p2pk65bytes,
p2pk33addressindex_to_p2pk33bytes,
p2pkhaddressindex_to_p2pkhbytes,
p2shaddressindex_to_p2shbytes,
p2wpkhaddressindex_to_p2wpkhbytes,
p2wshaddressindex_to_p2wshbytes,
p2traddressindex_to_p2trbytes,
p2aaddressindex_to_p2abytes,
) = parallel_import! {
height_to_first_p2pk65addressindex = PcoVec::forced_import(db, "first_p2pk65addressindex", version),
height_to_first_p2pk33addressindex = PcoVec::forced_import(db, "first_p2pk33addressindex", version),
height_to_first_p2pkhaddressindex = PcoVec::forced_import(db, "first_p2pkhaddressindex", version),
height_to_first_p2shaddressindex = PcoVec::forced_import(db, "first_p2shaddressindex", version),
height_to_first_p2wpkhaddressindex = PcoVec::forced_import(db, "first_p2wpkhaddressindex", version),
height_to_first_p2wshaddressindex = PcoVec::forced_import(db, "first_p2wshaddressindex", version),
height_to_first_p2traddressindex = PcoVec::forced_import(db, "first_p2traddressindex", version),
height_to_first_p2aaddressindex = PcoVec::forced_import(db, "first_p2aaddressindex", version),
p2pk65addressindex_to_p2pk65bytes = BytesVec::forced_import(db, "p2pk65bytes", version),
p2pk33addressindex_to_p2pk33bytes = BytesVec::forced_import(db, "p2pk33bytes", version),
p2pkhaddressindex_to_p2pkhbytes = BytesVec::forced_import(db, "p2pkhbytes", version),
p2shaddressindex_to_p2shbytes = BytesVec::forced_import(db, "p2shbytes", version),
p2wpkhaddressindex_to_p2wpkhbytes = BytesVec::forced_import(db, "p2wpkhbytes", version),
p2wshaddressindex_to_p2wshbytes = BytesVec::forced_import(db, "p2wshbytes", version),
p2traddressindex_to_p2trbytes = BytesVec::forced_import(db, "p2trbytes", version),
p2aaddressindex_to_p2abytes = BytesVec::forced_import(db, "p2abytes", version),
};
Ok(Self {
height_to_first_p2pk65addressindex: PcoVec::forced_import(
db,
"first_p2pk65addressindex",
version,
)?,
height_to_first_p2pk33addressindex: PcoVec::forced_import(
db,
"first_p2pk33addressindex",
version,
)?,
height_to_first_p2pkhaddressindex: PcoVec::forced_import(
db,
"first_p2pkhaddressindex",
version,
)?,
height_to_first_p2shaddressindex: PcoVec::forced_import(
db,
"first_p2shaddressindex",
version,
)?,
height_to_first_p2wpkhaddressindex: PcoVec::forced_import(
db,
"first_p2wpkhaddressindex",
version,
)?,
height_to_first_p2wshaddressindex: PcoVec::forced_import(
db,
"first_p2wshaddressindex",
version,
)?,
height_to_first_p2traddressindex: PcoVec::forced_import(
db,
"first_p2traddressindex",
version,
)?,
height_to_first_p2aaddressindex: PcoVec::forced_import(
db,
"first_p2aaddressindex",
version,
)?,
p2pk65addressindex_to_p2pk65bytes: BytesVec::forced_import(db, "p2pk65bytes", version)?,
p2pk33addressindex_to_p2pk33bytes: BytesVec::forced_import(db, "p2pk33bytes", version)?,
p2pkhaddressindex_to_p2pkhbytes: BytesVec::forced_import(db, "p2pkhbytes", version)?,
p2shaddressindex_to_p2shbytes: BytesVec::forced_import(db, "p2shbytes", version)?,
p2wpkhaddressindex_to_p2wpkhbytes: BytesVec::forced_import(db, "p2wpkhbytes", version)?,
p2wshaddressindex_to_p2wshbytes: BytesVec::forced_import(db, "p2wshbytes", version)?,
p2traddressindex_to_p2trbytes: BytesVec::forced_import(db, "p2trbytes", version)?,
p2aaddressindex_to_p2abytes: BytesVec::forced_import(db, "p2abytes", version)?,
height_to_first_p2pk65addressindex,
height_to_first_p2pk33addressindex,
height_to_first_p2pkhaddressindex,
height_to_first_p2shaddressindex,
height_to_first_p2wpkhaddressindex,
height_to_first_p2wshaddressindex,
height_to_first_p2traddressindex,
height_to_first_p2aaddressindex,
p2pk65addressindex_to_p2pk65bytes,
p2pk33addressindex_to_p2pk33bytes,
p2pkhaddressindex_to_p2pkhbytes,
p2shaddressindex_to_p2shbytes,
p2wpkhaddressindex_to_p2wpkhbytes,
p2wshaddressindex_to_p2wshbytes,
p2traddressindex_to_p2trbytes,
p2aaddressindex_to_p2abytes,
})
}

25
crates/brk_indexer/src/vecs/blocks.rs
View File

@@ -4,6 +4,8 @@ use brk_types::{BlockHash, Height, StoredF64, StoredU64, Timestamp, Version, Wei
use rayon::prelude::*;
use vecdb::{AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
use crate::parallel_import;
#[derive(Clone, Traversable)]
pub struct BlockVecs {
pub height_to_blockhash: BytesVec<Height, BlockHash>,
@@ -16,12 +18,25 @@ pub struct BlockVecs {
impl BlockVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
height_to_blockhash,
height_to_difficulty,
height_to_timestamp,
height_to_total_size,
height_to_weight,
) = parallel_import! {
height_to_blockhash = BytesVec::forced_import(db, "blockhash", version),
height_to_difficulty = PcoVec::forced_import(db, "difficulty", version),
height_to_timestamp = PcoVec::forced_import(db, "timestamp", version),
height_to_total_size = PcoVec::forced_import(db, "total_size", version),
height_to_weight = PcoVec::forced_import(db, "weight", version),
};
Ok(Self {
height_to_blockhash: BytesVec::forced_import(db, "blockhash", version)?,
height_to_difficulty: PcoVec::forced_import(db, "difficulty", version)?,
height_to_timestamp: PcoVec::forced_import(db, "timestamp", version)?,
height_to_total_size: PcoVec::forced_import(db, "total_size", version)?,
height_to_weight: PcoVec::forced_import(db, "weight", version)?,
height_to_blockhash,
height_to_difficulty,
height_to_timestamp,
height_to_total_size,
height_to_weight,
})
}

20
crates/brk_indexer/src/vecs/macros.rs Normal file
View File

@@ -0,0 +1,20 @@
/// Imports multiple items in parallel using thread::scope.
/// Each expression must return Result<T>.
///
/// # Example
/// ```ignore
/// let (a, b, c) = parallel_import! {
/// a = SomeVec::forced_import(&db, version),
/// b = OtherVec::forced_import(&db, version),
/// c = ThirdVec::forced_import(&db, version),
/// };
/// ```
#[macro_export]
macro_rules! parallel_import {
($($name:ident = $expr:expr),+ $(,)?) => {{
std::thread::scope(|s| -> brk_error::Result<_> {
$(let $name = s.spawn(|| $expr);)+
Ok(($($name.join().unwrap()?,)+))
})?
}};
}

54
crates/brk_indexer/src/vecs/mod.rs
View File

@@ -6,8 +6,11 @@ use brk_types::{AddressBytes, AddressHash, Height, OutputType, TypeIndex, Versio
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, PAGE_SIZE, Reader, Stamp};
use crate::parallel_import;
mod address;
mod blocks;
mod macros;
mod output;
mod tx;
mod txin;
@@ -35,15 +38,51 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(parent: &Path, version: Version) -> Result<Self> {
log::debug!("Opening vecs database...");
let db = Database::open(&parent.join("vecs"))?;
log::debug!("Setting min len...");
db.set_min_len(PAGE_SIZE * 50_000_000)?;
let block = BlockVecs::forced_import(&db, version)?;
let tx = TxVecs::forced_import(&db, version)?;
let txin = TxinVecs::forced_import(&db, version)?;
let txout = TxoutVecs::forced_import(&db, version)?;
let address = AddressVecs::forced_import(&db, version)?;
let output = OutputVecs::forced_import(&db, version)?;
log::debug!("Importing sub-vecs in parallel...");
let (block, tx, txin, txout, address, output) = parallel_import! {
block = {
log::debug!("Importing BlockVecs...");
let r = BlockVecs::forced_import(&db, version);
log::debug!("BlockVecs imported.");
r
},
tx = {
log::debug!("Importing TxVecs...");
let r = TxVecs::forced_import(&db, version);
log::debug!("TxVecs imported.");
r
},
txin = {
log::debug!("Importing TxinVecs...");
let r = TxinVecs::forced_import(&db, version);
log::debug!("TxinVecs imported.");
r
},
txout = {
log::debug!("Importing TxoutVecs...");
let r = TxoutVecs::forced_import(&db, version);
log::debug!("TxoutVecs imported.");
r
},
address = {
log::debug!("Importing AddressVecs...");
let r = AddressVecs::forced_import(&db, version);
log::debug!("AddressVecs imported.");
r
},
output = {
log::debug!("Importing OutputVecs...");
let r = OutputVecs::forced_import(&db, version);
log::debug!("OutputVecs imported.");
r
},
};
log::debug!("Sub-vecs imported.");
let this = Self {
db,
@@ -55,13 +94,16 @@ impl Vecs {
output,
};
log::debug!("Retaining regions...");
this.db.retain_regions(
this.iter_any_exportable()
.flat_map(|v| v.region_names())
.collect(),
)?;
log::debug!("Compacting database...");
this.db.compact()?;
log::debug!("Vecs import complete.");
Ok(this)
}

53
crates/brk_indexer/src/vecs/output.rs
View File

@@ -6,6 +6,8 @@ use brk_types::{
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
use crate::parallel_import;
#[derive(Clone, Traversable)]
pub struct OutputVecs {
// Height to first output index (per output type)
@@ -22,31 +24,34 @@ pub struct OutputVecs {
impl OutputVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
height_to_first_emptyoutputindex,
height_to_first_opreturnindex,
height_to_first_p2msoutputindex,
height_to_first_unknownoutputindex,
emptyoutputindex_to_txindex,
opreturnindex_to_txindex,
p2msoutputindex_to_txindex,
unknownoutputindex_to_txindex,
) = parallel_import! {
height_to_first_emptyoutputindex = PcoVec::forced_import(db, "first_emptyoutputindex", version),
height_to_first_opreturnindex = PcoVec::forced_import(db, "first_opreturnindex", version),
height_to_first_p2msoutputindex = PcoVec::forced_import(db, "first_p2msoutputindex", version),
height_to_first_unknownoutputindex = PcoVec::forced_import(db, "first_unknownoutputindex", version),
emptyoutputindex_to_txindex = PcoVec::forced_import(db, "txindex", version),
opreturnindex_to_txindex = PcoVec::forced_import(db, "txindex", version),
p2msoutputindex_to_txindex = PcoVec::forced_import(db, "txindex", version),
unknownoutputindex_to_txindex = PcoVec::forced_import(db, "txindex", version),
};
Ok(Self {
height_to_first_emptyoutputindex: PcoVec::forced_import(
db,
"first_emptyoutputindex",
version,
)?,
height_to_first_opreturnindex: PcoVec::forced_import(
db,
"first_opreturnindex",
version,
)?,
height_to_first_p2msoutputindex: PcoVec::forced_import(
db,
"first_p2msoutputindex",
version,
)?,
height_to_first_unknownoutputindex: PcoVec::forced_import(
db,
"first_unknownoutputindex",
version,
)?,
emptyoutputindex_to_txindex: PcoVec::forced_import(db, "txindex", version)?,
opreturnindex_to_txindex: PcoVec::forced_import(db, "txindex", version)?,
p2msoutputindex_to_txindex: PcoVec::forced_import(db, "txindex", version)?,
unknownoutputindex_to_txindex: PcoVec::forced_import(db, "txindex", version)?,
height_to_first_emptyoutputindex,
height_to_first_opreturnindex,
height_to_first_p2msoutputindex,
height_to_first_unknownoutputindex,
emptyoutputindex_to_txindex,
opreturnindex_to_txindex,
p2msoutputindex_to_txindex,
unknownoutputindex_to_txindex,
})
}

45
crates/brk_indexer/src/vecs/tx.rs
View File

@@ -7,6 +7,8 @@ use brk_types::{
use rayon::prelude::*;
use vecdb::{AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
use crate::parallel_import;
#[derive(Clone, Traversable)]
pub struct TxVecs {
pub height_to_first_txindex: PcoVec<Height, TxIndex>,
@@ -23,17 +25,40 @@ pub struct TxVecs {
impl TxVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
height_to_first_txindex,
txindex_to_height,
txindex_to_txid,
txindex_to_txversion,
txindex_to_rawlocktime,
txindex_to_base_size,
txindex_to_total_size,
txindex_to_is_explicitly_rbf,
txindex_to_first_txinindex,
txindex_to_first_txoutindex,
) = parallel_import! {
height_to_first_txindex = PcoVec::forced_import(db, "first_txindex", version),
txindex_to_height = PcoVec::forced_import(db, "height", version),
txindex_to_txid = BytesVec::forced_import(db, "txid", version),
txindex_to_txversion = PcoVec::forced_import(db, "txversion", version),
txindex_to_rawlocktime = PcoVec::forced_import(db, "rawlocktime", version),
txindex_to_base_size = PcoVec::forced_import(db, "base_size", version),
txindex_to_total_size = PcoVec::forced_import(db, "total_size", version),
txindex_to_is_explicitly_rbf = PcoVec::forced_import(db, "is_explicitly_rbf", version),
txindex_to_first_txinindex = PcoVec::forced_import(db, "first_txinindex", version),
txindex_to_first_txoutindex = BytesVec::forced_import(db, "first_txoutindex", version),
};
Ok(Self {
height_to_first_txindex: PcoVec::forced_import(db, "first_txindex", version)?,
txindex_to_height: PcoVec::forced_import(db, "height", version)?,
txindex_to_txid: BytesVec::forced_import(db, "txid", version)?,
txindex_to_txversion: PcoVec::forced_import(db, "txversion", version)?,
txindex_to_rawlocktime: PcoVec::forced_import(db, "rawlocktime", version)?,
txindex_to_base_size: PcoVec::forced_import(db, "base_size", version)?,
txindex_to_total_size: PcoVec::forced_import(db, "total_size", version)?,
txindex_to_is_explicitly_rbf: PcoVec::forced_import(db, "is_explicitly_rbf", version)?,
txindex_to_first_txinindex: PcoVec::forced_import(db, "first_txinindex", version)?,
txindex_to_first_txoutindex: BytesVec::forced_import(db, "first_txoutindex", version)?,
height_to_first_txindex,
txindex_to_height,
txindex_to_txid,
txindex_to_txversion,
txindex_to_rawlocktime,
txindex_to_base_size,
txindex_to_total_size,
txindex_to_is_explicitly_rbf,
txindex_to_first_txinindex,
txindex_to_first_txoutindex,
})
}

47
crates/brk_indexer/src/vecs/txin.rs
View File

@@ -1,22 +1,49 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, OutPoint, TxInIndex, TxIndex, Version};
use brk_types::{Height, OutPoint, OutputType, Sats, TxInIndex, TxIndex, TypeIndex, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
use crate::parallel_import;
#[derive(Clone, Traversable)]
pub struct TxinVecs {
pub height_to_first_txinindex: PcoVec<Height, TxInIndex>,
pub txinindex_to_outpoint: PcoVec<TxInIndex, OutPoint>,
pub txinindex_to_txindex: PcoVec<TxInIndex, TxIndex>,
pub txinindex_to_value: PcoVec<TxInIndex, Sats>,
pub txinindex_to_prev_height: PcoVec<TxInIndex, Height>,
pub txinindex_to_outputtype: PcoVec<TxInIndex, OutputType>,
pub txinindex_to_typeindex: PcoVec<TxInIndex, TypeIndex>,
}
impl TxinVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
height_to_first_txinindex,
txinindex_to_outpoint,
txinindex_to_txindex,
txinindex_to_value,
txinindex_to_prev_height,
txinindex_to_outputtype,
txinindex_to_typeindex,
) = parallel_import! {
height_to_first_txinindex = PcoVec::forced_import(db, "first_txinindex", version),
txinindex_to_outpoint = PcoVec::forced_import(db, "outpoint", version),
txinindex_to_txindex = PcoVec::forced_import(db, "txindex", version),
txinindex_to_value = PcoVec::forced_import(db, "value", version),
txinindex_to_prev_height = PcoVec::forced_import(db, "prev_height", version),
txinindex_to_outputtype = PcoVec::forced_import(db, "outputtype", version),
txinindex_to_typeindex = PcoVec::forced_import(db, "typeindex", version),
};
Ok(Self {
height_to_first_txinindex: PcoVec::forced_import(db, "first_txinindex", version)?,
txinindex_to_outpoint: PcoVec::forced_import(db, "outpoint", version)?,
txinindex_to_txindex: PcoVec::forced_import(db, "txindex", version)?,
height_to_first_txinindex,
txinindex_to_outpoint,
txinindex_to_txindex,
txinindex_to_value,
txinindex_to_prev_height,
txinindex_to_outputtype,
txinindex_to_typeindex,
})
}
@@ -27,6 +54,14 @@ impl TxinVecs {
.truncate_if_needed_with_stamp(txinindex, stamp)?;
self.txinindex_to_txindex
.truncate_if_needed_with_stamp(txinindex, stamp)?;
self.txinindex_to_value
.truncate_if_needed_with_stamp(txinindex, stamp)?;
self.txinindex_to_prev_height
.truncate_if_needed_with_stamp(txinindex, stamp)?;
self.txinindex_to_outputtype
.truncate_if_needed_with_stamp(txinindex, stamp)?;
self.txinindex_to_typeindex
.truncate_if_needed_with_stamp(txinindex, stamp)?;
Ok(())
}
@@ -35,6 +70,10 @@ impl TxinVecs {
&mut self.height_to_first_txinindex as &mut dyn AnyStoredVec,
&mut self.txinindex_to_outpoint,
&mut self.txinindex_to_txindex,
&mut self.txinindex_to_value,
&mut self.txinindex_to_prev_height,
&mut self.txinindex_to_outputtype,
&mut self.txinindex_to_typeindex,
]
.into_par_iter()
}

55
crates/brk_indexer/src/vecs/txout.rs
View File

@@ -1,50 +1,69 @@
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Height, OutputType, Sats, TxIndex, TxOutIndex, TypeIndex, Version};
use brk_types::{Height, Sats, TxInIndex, TxIndex, TxOutData, TxOutIndex, Version};
use rayon::prelude::*;
use vecdb::{AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
use vecdb::{
AnyStoredVec, AnyVec, BytesVec, Database, GenericStoredVec, ImportableVec, IterableCloneableVec,
LazyVecFrom1, PcoVec, Stamp,
};
use crate::parallel_import;
#[derive(Clone, Traversable)]
pub struct TxoutVecs {
pub height_to_first_txoutindex: PcoVec<Height, TxOutIndex>,
pub txoutindex_to_value: BytesVec<TxOutIndex, Sats>,
pub txoutindex_to_outputtype: BytesVec<TxOutIndex, OutputType>,
pub txoutindex_to_typeindex: BytesVec<TxOutIndex, TypeIndex>,
pub txoutindex_to_txoutdata: BytesVec<TxOutIndex, TxOutData>,
pub txoutindex_to_txindex: PcoVec<TxOutIndex, TxIndex>,
pub txoutindex_to_txinindex: BytesVec<TxOutIndex, TxInIndex>,
pub txoutindex_to_value: LazyVecFrom1<TxOutIndex, Sats, TxOutIndex, TxOutData>,
}
impl TxoutVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
height_to_first_txoutindex,
txoutindex_to_txoutdata,
txoutindex_to_txindex,
txoutindex_to_txinindex,
) = parallel_import! {
height_to_first_txoutindex = PcoVec::forced_import(db, "first_txoutindex", version),
txoutindex_to_txoutdata = BytesVec::forced_import(db, "txoutdata", version),
txoutindex_to_txindex = PcoVec::forced_import(db, "txindex", version),
txoutindex_to_txinindex = BytesVec::forced_import(db, "txinindex", version),
};
let txoutindex_to_value = LazyVecFrom1::init(
"value",
txoutindex_to_txoutdata.version(),
txoutindex_to_txoutdata.boxed_clone(),
|index, iter| iter.get(index).map(|txoutdata: TxOutData| txoutdata.value),
);
Ok(Self {
height_to_first_txoutindex: PcoVec::forced_import(db, "first_txoutindex", version)?,
txoutindex_to_value: BytesVec::forced_import(db, "value", version)?,
txoutindex_to_outputtype: BytesVec::forced_import(db, "outputtype", version)?,
txoutindex_to_typeindex: BytesVec::forced_import(db, "typeindex", version)?,
txoutindex_to_txindex: PcoVec::forced_import(db, "txindex", version)?,
height_to_first_txoutindex,
txoutindex_to_txoutdata,
txoutindex_to_txindex,
txoutindex_to_txinindex,
txoutindex_to_value,
})
}
pub fn truncate(&mut self, height: Height, txoutindex: TxOutIndex, stamp: Stamp) -> Result<()> {
self.height_to_first_txoutindex
.truncate_if_needed_with_stamp(height, stamp)?;
self.txoutindex_to_value
.truncate_if_needed_with_stamp(txoutindex, stamp)?;
self.txoutindex_to_outputtype
.truncate_if_needed_with_stamp(txoutindex, stamp)?;
self.txoutindex_to_typeindex
self.txoutindex_to_txoutdata
.truncate_if_needed_with_stamp(txoutindex, stamp)?;
self.txoutindex_to_txindex
.truncate_if_needed_with_stamp(txoutindex, stamp)?;
self.txoutindex_to_txinindex
.truncate_if_needed_with_stamp(txoutindex, stamp)?;
Ok(())
}
pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[
&mut self.height_to_first_txoutindex as &mut dyn AnyStoredVec,
&mut self.txoutindex_to_value,
&mut self.txoutindex_to_outputtype,
&mut self.txoutindex_to_typeindex,
&mut self.txoutindex_to_txoutdata,
&mut self.txoutindex_to_txindex,
&mut self.txoutindex_to_txinindex,
]
.into_par_iter()
}

7
crates/brk_mcp/Cargo.toml
View File

@@ -9,14 +9,15 @@ repository.workspace = true
build = "build.rs"
[dependencies]
aide = { workspace = true }
brk_query = { workspace = true }
axum = { workspace = true }
brk_rmcp = { version = "0.8.0", features = [
"transport-worker",
"transport-streamable-http-server",
] }
brk_types = { workspace = true }
log = { workspace = true }
minreq = { workspace = true }
schemars = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
[package.metadata.cargo-machete]

33
crates/brk_mcp/README.md
View File

@@ -4,42 +4,31 @@ Model Context Protocol (MCP) server for Bitcoin on-chain data.
## What It Enables
Expose BRK's query capabilities to AI assistants via the MCP standard. LLMs can browse metrics, fetch datasets, and analyze on-chain data through structured tool calls.
## Key Features
- **Tool-based API**: 8 tools for metric discovery and data retrieval
- **Pagination support**: Browse large metric catalogs in chunks
- **Self-documenting**: Built-in instructions explain available capabilities
- **Async**: Full tokio integration via `AsyncQuery`
Expose BRK's REST API to AI assistants via MCP. The LLM reads the OpenAPI spec and calls any endpoint through a generic fetch tool.
## Available Tools
| Tool | Description |
|------|-------------|
| `get_metric_count` | Count of unique metrics |
| `get_vec_count` | Total metric × index combinations |
| `get_indexes` | List all index types and variants |
| `get_vecids` | Paginated list of metric IDs |
| `get_index_to_vecids` | Metrics supporting a given index |
| `get_vecid_to_indexes` | Indexes supported by a metric |
| `get_vecs` | Fetch metric data with range selection |
| `get_version` | BRK version string |
| `get_openapi` | Get the OpenAPI specification for all REST endpoints |
| `fetch` | Call any REST API endpoint by path and query |
## Workflow
1. LLM calls `get_openapi` to discover available endpoints
2. LLM calls `fetch` with the desired path and query parameters
## Usage
```rust,ignore
let mcp = MCP::new(&async_query);
// The MCP server implements ServerHandler for use with rmcp
// Tools are auto-registered via the #[tool_router] macro
let mcp = MCP::new("http://127.0.0.1:3110", openapi_json);
```
## Integration
The MCP server is integrated into `brk_server` and exposed at `/mcp` endpoint for MCP transport.
The MCP server is integrated into `brk_server` and exposed at `/mcp` endpoint.
## Built On
- `brk_query` for data access
- `brk_rmcp` for MCP protocol implementation
- `minreq` for HTTP requests

166
crates/brk_mcp/src/lib.rs
View File

@@ -1,6 +1,7 @@
#![doc = include_str!("../README.md")]
use brk_query::{AsyncQuery, MetricSelection, Pagination, PaginationIndex};
use std::sync::Arc;
use brk_rmcp::{
ErrorData as McpError, RoleServer, ServerHandler,
handler::server::{router::tool::ToolRouter, wrapper::Parameters},
@@ -8,135 +9,68 @@ use brk_rmcp::{
service::RequestContext,
tool, tool_handler, tool_router,
};
use brk_types::Metric;
use log::info;
use schemars::JsonSchema;
use serde::Deserialize;
pub mod route;
#[derive(Clone)]
pub struct MCP {
query: AsyncQuery,
base_url: Arc<String>,
openapi_json: Arc<String>,
tool_router: ToolRouter<MCP>,
}
const VERSION: &str = env!("CARGO_PKG_VERSION");
/// Parameters for fetching from the REST API.
#[derive(Deserialize, JsonSchema)]
pub struct FetchParams {
/// API path (e.g., "/api/blocks" or "/api/metrics/list")
pub path: String,
/// Optional query string (e.g., "page=0" or "from=-1&to=-10")
pub query: Option<String>,
}
#[tool_router]
impl MCP {
pub fn new(query: &AsyncQuery) -> Self {
pub fn new(base_url: impl Into<String>, openapi_json: impl Into<String>) -> Self {
Self {
query: query.clone(),
base_url: Arc::new(base_url.into()),
openapi_json: Arc::new(openapi_json.into()),
tool_router: Self::tool_router(),
}
}
#[tool(description = "
Get the count of unique metrics.
")]
async fn get_metric_count(&self) -> Result<CallToolResult, McpError> {
info!("mcp: distinct_metric_count");
Ok(CallToolResult::success(vec![
Content::json(self.query.sync(|q| q.distinct_metric_count())).unwrap(),
]))
}
#[tool(description = "
Get the count of all metrics. (distinct metrics multiplied by the number of indexes supported by each one)
")]
async fn get_vec_count(&self) -> Result<CallToolResult, McpError> {
info!("mcp: total_metric_count");
Ok(CallToolResult::success(vec![
Content::json(self.query.sync(|q| q.total_metric_count())).unwrap(),
]))
}
#[tool(description = "
Get the list of all existing indexes and their accepted variants.
")]
async fn get_indexes(&self) -> Result<CallToolResult, McpError> {
info!("mcp: get_indexes");
Ok(CallToolResult::success(vec![
Content::json(self.query.inner().indexes()).unwrap(),
]))
}
#[tool(description = "
Get a paginated list of all existing vec ids.
There are up to 1,000 values per page.
If the `page` param is omitted, it will default to the first page.
")]
async fn get_vecids(
&self,
Parameters(pagination): Parameters<Pagination>,
) -> Result<CallToolResult, McpError> {
info!("mcp: get_metrics");
Ok(CallToolResult::success(vec![
Content::json(self.query.sync(|q| q.metrics(pagination))).unwrap(),
]))
}
#[tool(description = "
Get a paginated list of all vec ids which support a given index.
There are up to 1,000 values per page.
If the `page` param is omitted, it will default to the first page.
")]
async fn get_index_to_vecids(
&self,
Parameters(paginated_index): Parameters<PaginationIndex>,
) -> Result<CallToolResult, McpError> {
info!("mcp: get_index_to_vecids");
let result = self
.query
.inner()
.index_to_vecids(paginated_index)
.unwrap_or_default();
Ok(CallToolResult::success(vec![
Content::json(result).unwrap(),
]))
}
#[tool(description = "
Get a list of all indexes supported by a given vec id.
The list will be empty if the vec id isn't correct.
")]
async fn get_vecid_to_indexes(
&self,
Parameters(metric): Parameters<Metric>,
) -> Result<CallToolResult, McpError> {
info!("mcp: get_vecid_to_indexes");
Ok(CallToolResult::success(vec![
Content::json(self.query.inner().metric_to_indexes(metric)).unwrap(),
]))
}
#[tool(description = "
Get one or multiple vecs depending on given parameters.
The response's format will depend on the given parameters, it will be:
- A value: If requested only one vec and the given range returns one value (for example: `from=-1`)
- A list: If requested only one vec and the given range returns multiple values (for example: `from=-1000&count=100` or `from=-444&to=-333`)
- A matrix: When multiple vecs are requested, even if they each return one value.
")]
async fn get_vecs(
&self,
Parameters(params): Parameters<MetricSelection>,
) -> Result<CallToolResult, McpError> {
info!("mcp: get_vecs");
#[tool(description = "Get the OpenAPI specification describing all available REST API endpoints.")]
async fn get_openapi(&self) -> Result<CallToolResult, McpError> {
info!("mcp: get_openapi");
Ok(CallToolResult::success(vec![Content::text(
match self.query.run(move |q| q.search_and_format_legacy(params)).await {
Ok(output) => output.to_string(),
Err(e) => format!("Error:\n{e}"),
},
self.openapi_json.as_str(),
)]))
}
#[tool(description = "
Get the running version of the Bitcoin Research Kit.
")]
async fn get_version(&self) -> Result<CallToolResult, McpError> {
info!("mcp: get_version");
Ok(CallToolResult::success(vec![Content::text(format!(
"v{VERSION}"
))]))
#[tool(description = "Call a REST API endpoint. Use get_openapi first to discover available endpoints.")]
async fn fetch(
&self,
Parameters(params): Parameters<FetchParams>,
) -> Result<CallToolResult, McpError> {
info!("mcp: fetch {}", params.path);
let url = match &params.query {
Some(q) if !q.is_empty() => format!("{}{}?{}", self.base_url, params.path, q),
_ => format!("{}{}", self.base_url, params.path),
};
match minreq::get(&url).send() {
Ok(response) => {
let body = response.as_str().unwrap_or("").to_string();
Ok(CallToolResult::success(vec![Content::text(body)]))
}
Err(e) => Err(McpError::internal_error(
format!("HTTP request failed: {e}"),
None,
)),
}
}
}
@@ -149,17 +83,13 @@ impl ServerHandler for MCP {
server_info: Implementation::from_build_env(),
instructions: Some(
"
This server provides an interface to communicate with a running instance of the Bitcoin Research Kit (also called brk or BRK).
Bitcoin Research Kit (BRK) - Bitcoin on-chain metrics and market data.
Multiple tools are at your disposal including ones to fetch all sorts of Bitcoin on-chain metrics and market prices.
Workflow:
1. Call get_openapi to get the full API specification
2. Use fetch to call any endpoint described in the spec
If you're unsure which datasets are available, try out different tools before browsing the web. Each tool gives important information about BRK's capabilities.
Vectors can also be called 'Vecs', 'Arrays' or 'Datasets', they can all be used interchangeably.
An 'Index' (or indexes) is the timeframe of a dataset.
'VecId' (or vecids) are the name of the dataset and what it represents.
Example: fetch with path=\"/api/metrics/list\" to list metrics.
"
.to_string(),
),

47
crates/brk_mcp/src/route.rs
View File

@@ -1,39 +1,26 @@
use aide::axum::ApiRouter;
use brk_query::AsyncQuery;
use std::sync::Arc;
use axum::Router;
use brk_rmcp::transport::{
StreamableHttpServerConfig,
streamable_http_server::{StreamableHttpService, session::local::LocalSessionManager},
};
use log::info;
use crate::MCP;
pub trait MCPRoutes {
fn add_mcp_routes(self, query: &AsyncQuery, mcp: bool) -> Self;
}
impl<T> MCPRoutes for ApiRouter<T>
where
T: Clone + Send + Sync + 'static,
{
fn add_mcp_routes(self, query: &AsyncQuery, mcp: bool) -> Self {
if !mcp {
return self;
}
let query = query.clone();
let service = StreamableHttpService::new(
move || Ok(MCP::new(&query)),
LocalSessionManager::default().into(),
StreamableHttpServerConfig {
stateful_mode: false,
..Default::default()
},
);
info!("Setting MCP...");
self.nest_service("/mcp", service)
}
/// Create an MCP service router.
pub fn mcp_router(base_url: String, openapi_json: Arc<String>) -> Router {
info!("Setting up MCP...");
let service = StreamableHttpService::new(
move || Ok(MCP::new(base_url.clone(), openapi_json.as_str())),
LocalSessionManager::default().into(),
StreamableHttpServerConfig {
stateful_mode: false,
..Default::default()
},
);
Router::new().nest_service("/mcp", service)
}

4
crates/brk_query/src/impl/address.rs
View File

@@ -7,7 +7,7 @@ use brk_types::{
AddressIndexTxIndex, AddressStats, AnyAddressDataIndexEnum, OutputType, Sats, TxIndex,
TxStatus, Txid, TypeIndex, Unit, Utxo, Vout,
};
use vecdb::TypedVecIterator;
use vecdb::{IterableVec, TypedVecIterator};
use crate::Query;
@@ -169,7 +169,7 @@ impl Query {
let mut txindex_to_txid_iter = vecs.tx.txindex_to_txid.iter()?;
let mut txindex_to_height_iter = vecs.tx.txindex_to_height.iter()?;
let mut txindex_to_first_txoutindex_iter = vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut txoutindex_to_value_iter = vecs.txout.txoutindex_to_value.iter()?;
let mut txoutindex_to_value_iter = vecs.txout.txoutindex_to_value.iter();
let mut height_to_blockhash_iter = vecs.block.height_to_blockhash.iter()?;
let mut height_to_timestamp_iter = vecs.block.height_to_timestamp.iter()?;

14
crates/brk_query/src/impl/transaction.rs
View File

@@ -6,7 +6,7 @@ use brk_types::{
Sats, Transaction, TxIn, TxInIndex, TxIndex, TxOut, TxOutspend, TxStatus, Txid, TxidParam,
TxidPrefix, Vin, Vout, Weight,
};
use vecdb::{GenericStoredVec, TypedVecIterator};
use vecdb::{GenericStoredVec, IterableVec, TypedVecIterator};
use crate::Query;
@@ -119,9 +119,10 @@ impl Query {
let txoutindex = first_txoutindex + vout;
// Look up spend status
let computer = self.computer();
let txinindex = computer
.stateful
let indexer = self.indexer();
let txinindex = indexer
.vecs
.txout
.txoutindex_to_txinindex
.read_once(txoutindex)?;
@@ -167,8 +168,7 @@ impl Query {
let output_count = usize::from(next_first_txoutindex) - usize::from(first_txoutindex);
// Get spend status for each output
let computer = self.computer();
let mut txoutindex_to_txinindex_iter = computer.stateful.txoutindex_to_txinindex.iter()?;
let mut txoutindex_to_txinindex_iter = indexer.vecs.txout.txoutindex_to_txinindex.iter()?;
let mut outspends = Vec::with_capacity(output_count);
for i in 0..output_count {
@@ -220,7 +220,7 @@ impl Query {
let mut txindex_to_first_txoutindex_iter =
indexer.vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut txinindex_to_outpoint_iter = indexer.vecs.txin.txinindex_to_outpoint.iter()?;
let mut txoutindex_to_value_iter = indexer.vecs.txout.txoutindex_to_value.iter()?;
let mut txoutindex_to_value_iter = indexer.vecs.txout.txoutindex_to_value.iter();
// Build inputs with prevout information
let input: Vec<TxIn> = tx

39
crates/brk_server/src/lib.rs
View File

@@ -14,7 +14,7 @@ use axum::{
};
use brk_error::Result;
use brk_logger::OwoColorize;
use brk_mcp::route::MCPRoutes;
use brk_mcp::route::mcp_router;
use brk_query::AsyncQuery;
use log::{error, info};
use quick_cache::sync::Cache;
@@ -92,7 +92,6 @@ impl Server {
let vecs = state.query.inner().vecs();
let router = ApiRouter::new()
.add_api_routes()
.add_mcp_routes(&state.query, mcp)
.add_files_routes(state.path.as_ref())
.route(
"/discord",
@@ -136,24 +135,34 @@ impl Server {
let mut openapi = create_openapi();
let router = router.finish_api(&mut openapi);
let clients_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
let workspace_root: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent()
.and_then(|p| p.parent())
.unwrap()
.join("brk_binder")
.join("clients");
if clients_path.exists() {
let openapi_json = serde_json::to_string(&openapi).unwrap();
let result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
brk_binder::generate_clients(vecs, &openapi_json, &clients_path)
}));
.into();
let output_paths = brk_binder::ClientOutputPaths::new()
.rust(workspace_root.join("crates/brk_client/src/lib.rs"))
.javascript(workspace_root.join("modules/brk-client/index.js"))
.python(workspace_root.join("packages/brk_client/__init__.py"));
match result {
Ok(Ok(())) => info!("Generated clients at {}", clients_path.display()),
Ok(Err(e)) => error!("Failed to generate clients: {e}"),
Err(_) => error!("Client generation panicked"),
}
let openapi_json = Arc::new(serde_json::to_string(&openapi).unwrap());
let result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
brk_binder::generate_clients(vecs, &openapi_json, &output_paths)
}));
match result {
Ok(Ok(())) => info!("Generated clients"),
Ok(Err(e)) => error!("Failed to generate clients: {e}"),
Err(_) => error!("Client generation panicked"),
}
let router = if mcp {
let base_url = format!("http://127.0.0.1:{port}");
router.merge(mcp_router(base_url, openapi_json))
} else {
router
};
serve(
listener,
router

1
crates/brk_store/src/lib.rs
View File

@@ -277,6 +277,7 @@ where
ingestion.write(ByteView::from(key), ByteView::from(value))?;
}
Item::Tomb(key) => {
// TODO: switch to write_weak_tombstone when lsm-tree ingestion API supports it
ingestion.write_tombstone(ByteView::from(key))?;
}
}

18
crates/brk_store/src/meta.rs
View File

@@ -3,7 +3,7 @@ use std::{
path::{Path, PathBuf},
};
use brk_error::Result;
use brk_error::{Error, Result};
use brk_types::Version;
use fjall::{Database, Keyspace};
@@ -30,16 +30,14 @@ impl StoreMeta {
let partition = open_partition_handle()?;
if Version::try_from(Self::path_version_(path).as_path())
.is_ok_and(|prev_version| version != prev_version)
if let Ok(prev_version) = Version::try_from(Self::path_version_(path).as_path())
&& version != prev_version
{
todo!();
// fs::remove_dir_all(path)?;
// // Doesn't exist
// // database.delete_partition(partition)?;
// fs::create_dir(path)?;
// database.persist(PersistMode::SyncAll)?;
// partition = open_partition_handle()?;
return Err(Error::VersionMismatch {
path: path.to_path_buf(),
expected: u64::from(version) as usize,
found: u64::from(prev_version) as usize,
});
}
let slf = Self {

9
crates/brk_types/src/addressbytes.rs
View File

@@ -142,11 +142,10 @@ impl TryFrom<(&ScriptBuf, OutputType)> for AddressBytes {
let bytes = &script.as_bytes()[2..];
Ok(Self::P2A(Box::new(P2ABytes::from(bytes))))
}
OutputType::P2MS => Err(Error::WrongAddressType),
OutputType::Unknown => Err(Error::WrongAddressType),
OutputType::Empty => Err(Error::WrongAddressType),
OutputType::OpReturn => Err(Error::WrongAddressType),
_ => unreachable!(),
OutputType::P2MS
| OutputType::Unknown
| OutputType::Empty
| OutputType::OpReturn => Err(Error::WrongAddressType),
}
}
}

2
crates/brk_types/src/lib.rs
View File

@@ -138,6 +138,7 @@ mod txin;
mod txindex;
mod txinindex;
mod txout;
mod txoutdata;
mod txoutindex;
mod txoutspend;
mod txstatus;
@@ -292,6 +293,7 @@ pub use txin::*;
pub use txindex::*;
pub use txinindex::*;
pub use txout::*;
pub use txoutdata::*;
pub use txoutindex::*;
pub use txoutspend::*;
pub use txstatus::*;

787
crates/brk_types/src/outputtype.rs
View File

@@ -3,765 +3,35 @@ use brk_error::Error;
use schemars::JsonSchema;
use serde::Serialize;
use strum::Display;
use vecdb::{Bytes, Formattable};
use vecdb::{Bytes, Formattable, Pco, TransparentPco};
use crate::AddressBytes;
#[derive(
Debug, Clone, Copy, Display, PartialEq, Eq, PartialOrd, Ord, Serialize, JsonSchema, Hash,
)]
#[derive(Debug, Clone, Copy, Display, PartialEq, Eq, PartialOrd, Ord, Serialize, JsonSchema, Hash)]
#[serde(rename_all = "lowercase")]
#[strum(serialize_all = "lowercase")]
#[repr(u8)]
#[repr(u16)]
/// Type (P2PKH, P2WPKH, P2SH, P2TR, etc.)
pub enum OutputType {
P2PK65,
P2PK33,
P2PKH,
P2MS,
P2SH,
OpReturn,
P2WPKH,
P2WSH,
P2TR,
P2A,
#[doc(hidden)]
#[schemars(skip)]
Dummy10,
#[doc(hidden)]
#[schemars(skip)]
Dummy11,
#[doc(hidden)]
#[schemars(skip)]
Dummy12,
#[doc(hidden)]
#[schemars(skip)]
Dummy13,
#[doc(hidden)]
#[schemars(skip)]
Dummy14,
#[doc(hidden)]
#[schemars(skip)]
Dummy15,
#[doc(hidden)]
#[schemars(skip)]
Dummy16,
#[doc(hidden)]
#[schemars(skip)]
Dummy17,
#[doc(hidden)]
#[schemars(skip)]
Dummy18,
#[doc(hidden)]
#[schemars(skip)]
Dummy19,
#[doc(hidden)]
#[schemars(skip)]
Dummy20,
#[doc(hidden)]
#[schemars(skip)]
Dummy21,
#[doc(hidden)]
#[schemars(skip)]
Dummy22,
#[doc(hidden)]
#[schemars(skip)]
Dummy23,
#[doc(hidden)]
#[schemars(skip)]
Dummy24,
#[doc(hidden)]
#[schemars(skip)]
Dummy25,
#[doc(hidden)]
#[schemars(skip)]
Dummy26,
#[doc(hidden)]
#[schemars(skip)]
Dummy27,
#[doc(hidden)]
#[schemars(skip)]
Dummy28,
#[doc(hidden)]
#[schemars(skip)]
Dummy29,
#[doc(hidden)]
#[schemars(skip)]
Dummy30,
#[doc(hidden)]
#[schemars(skip)]
Dummy31,
#[doc(hidden)]
#[schemars(skip)]
Dummy32,
#[doc(hidden)]
#[schemars(skip)]
Dummy33,
#[doc(hidden)]
#[schemars(skip)]
Dummy34,
#[doc(hidden)]
#[schemars(skip)]
Dummy35,
#[doc(hidden)]
#[schemars(skip)]
Dummy36,
#[doc(hidden)]
#[schemars(skip)]
Dummy37,
#[doc(hidden)]
#[schemars(skip)]
Dummy38,
#[doc(hidden)]
#[schemars(skip)]
Dummy39,
#[doc(hidden)]
#[schemars(skip)]
Dummy40,
#[doc(hidden)]
#[schemars(skip)]
Dummy41,
#[doc(hidden)]
#[schemars(skip)]
Dummy42,
#[doc(hidden)]
#[schemars(skip)]
Dummy43,
#[doc(hidden)]
#[schemars(skip)]
Dummy44,
#[doc(hidden)]
#[schemars(skip)]
Dummy45,
#[doc(hidden)]
#[schemars(skip)]
Dummy46,
#[doc(hidden)]
#[schemars(skip)]
Dummy47,
#[doc(hidden)]
#[schemars(skip)]
Dummy48,
#[doc(hidden)]
#[schemars(skip)]
Dummy49,
#[doc(hidden)]
#[schemars(skip)]
Dummy50,
#[doc(hidden)]
#[schemars(skip)]
Dummy51,
#[doc(hidden)]
#[schemars(skip)]
Dummy52,
#[doc(hidden)]
#[schemars(skip)]
Dummy53,
#[doc(hidden)]
#[schemars(skip)]
Dummy54,
#[doc(hidden)]
#[schemars(skip)]
Dummy55,
#[doc(hidden)]
#[schemars(skip)]
Dummy56,
#[doc(hidden)]
#[schemars(skip)]
Dummy57,
#[doc(hidden)]
#[schemars(skip)]
Dummy58,
#[doc(hidden)]
#[schemars(skip)]
Dummy59,
#[doc(hidden)]
#[schemars(skip)]
Dummy60,
#[doc(hidden)]
#[schemars(skip)]
Dummy61,
#[doc(hidden)]
#[schemars(skip)]
Dummy62,
#[doc(hidden)]
#[schemars(skip)]
Dummy63,
#[doc(hidden)]
#[schemars(skip)]
Dummy64,
#[doc(hidden)]
#[schemars(skip)]
Dummy65,
#[doc(hidden)]
#[schemars(skip)]
Dummy66,
#[doc(hidden)]
#[schemars(skip)]
Dummy67,
#[doc(hidden)]
#[schemars(skip)]
Dummy68,
#[doc(hidden)]
#[schemars(skip)]
Dummy69,
#[doc(hidden)]
#[schemars(skip)]
Dummy70,
#[doc(hidden)]
#[schemars(skip)]
Dummy71,
#[doc(hidden)]
#[schemars(skip)]
Dummy72,
#[doc(hidden)]
#[schemars(skip)]
Dummy73,
#[doc(hidden)]
#[schemars(skip)]
Dummy74,
#[doc(hidden)]
#[schemars(skip)]
Dummy75,
#[doc(hidden)]
#[schemars(skip)]
Dummy76,
#[doc(hidden)]
#[schemars(skip)]
Dummy77,
#[doc(hidden)]
#[schemars(skip)]
Dummy78,
#[doc(hidden)]
#[schemars(skip)]
Dummy79,
#[doc(hidden)]
#[schemars(skip)]
Dummy80,
#[doc(hidden)]
#[schemars(skip)]
Dummy81,
#[doc(hidden)]
#[schemars(skip)]
Dummy82,
#[doc(hidden)]
#[schemars(skip)]
Dummy83,
#[doc(hidden)]
#[schemars(skip)]
Dummy84,
#[doc(hidden)]
#[schemars(skip)]
Dummy85,
#[doc(hidden)]
#[schemars(skip)]
Dummy86,
#[doc(hidden)]
#[schemars(skip)]
Dummy87,
#[doc(hidden)]
#[schemars(skip)]
Dummy88,
#[doc(hidden)]
#[schemars(skip)]
Dummy89,
#[doc(hidden)]
#[schemars(skip)]
Dummy90,
#[doc(hidden)]
#[schemars(skip)]
Dummy91,
#[doc(hidden)]
#[schemars(skip)]
Dummy92,
#[doc(hidden)]
#[schemars(skip)]
Dummy93,
#[doc(hidden)]
#[schemars(skip)]
Dummy94,
#[doc(hidden)]
#[schemars(skip)]
Dummy95,
#[doc(hidden)]
#[schemars(skip)]
Dummy96,
#[doc(hidden)]
#[schemars(skip)]
Dummy97,
#[doc(hidden)]
#[schemars(skip)]
Dummy98,
#[doc(hidden)]
#[schemars(skip)]
Dummy99,
#[doc(hidden)]
#[schemars(skip)]
Dummy100,
#[doc(hidden)]
#[schemars(skip)]
Dummy101,
#[doc(hidden)]
#[schemars(skip)]
Dummy102,
#[doc(hidden)]
#[schemars(skip)]
Dummy103,
#[doc(hidden)]
#[schemars(skip)]
Dummy104,
#[doc(hidden)]
#[schemars(skip)]
Dummy105,
#[doc(hidden)]
#[schemars(skip)]
Dummy106,
#[doc(hidden)]
#[schemars(skip)]
Dummy107,
#[doc(hidden)]
#[schemars(skip)]
Dummy108,
#[doc(hidden)]
#[schemars(skip)]
Dummy109,
#[doc(hidden)]
#[schemars(skip)]
Dummy110,
#[doc(hidden)]
#[schemars(skip)]
Dummy111,
#[doc(hidden)]
#[schemars(skip)]
Dummy112,
#[doc(hidden)]
#[schemars(skip)]
Dummy113,
#[doc(hidden)]
#[schemars(skip)]
Dummy114,
#[doc(hidden)]
#[schemars(skip)]
Dummy115,
#[doc(hidden)]
#[schemars(skip)]
Dummy116,
#[doc(hidden)]
#[schemars(skip)]
Dummy117,
#[doc(hidden)]
#[schemars(skip)]
Dummy118,
#[doc(hidden)]
#[schemars(skip)]
Dummy119,
#[doc(hidden)]
#[schemars(skip)]
Dummy120,
#[doc(hidden)]
#[schemars(skip)]
Dummy121,
#[doc(hidden)]
#[schemars(skip)]
Dummy122,
#[doc(hidden)]
#[schemars(skip)]
Dummy123,
#[doc(hidden)]
#[schemars(skip)]
Dummy124,
#[doc(hidden)]
#[schemars(skip)]
Dummy125,
#[doc(hidden)]
#[schemars(skip)]
Dummy126,
#[doc(hidden)]
#[schemars(skip)]
Dummy127,
#[doc(hidden)]
#[schemars(skip)]
Dummy128,
#[doc(hidden)]
#[schemars(skip)]
Dummy129,
#[doc(hidden)]
#[schemars(skip)]
Dummy130,
#[doc(hidden)]
#[schemars(skip)]
Dummy131,
#[doc(hidden)]
#[schemars(skip)]
Dummy132,
#[doc(hidden)]
#[schemars(skip)]
Dummy133,
#[doc(hidden)]
#[schemars(skip)]
Dummy134,
#[doc(hidden)]
#[schemars(skip)]
Dummy135,
#[doc(hidden)]
#[schemars(skip)]
Dummy136,
#[doc(hidden)]
#[schemars(skip)]
Dummy137,
#[doc(hidden)]
#[schemars(skip)]
Dummy138,
#[doc(hidden)]
#[schemars(skip)]
Dummy139,
#[doc(hidden)]
#[schemars(skip)]
Dummy140,
#[doc(hidden)]
#[schemars(skip)]
Dummy141,
#[doc(hidden)]
#[schemars(skip)]
Dummy142,
#[doc(hidden)]
#[schemars(skip)]
Dummy143,
#[doc(hidden)]
#[schemars(skip)]
Dummy144,
#[doc(hidden)]
#[schemars(skip)]
Dummy145,
#[doc(hidden)]
#[schemars(skip)]
Dummy146,
#[doc(hidden)]
#[schemars(skip)]
Dummy147,
#[doc(hidden)]
#[schemars(skip)]
Dummy148,
#[doc(hidden)]
#[schemars(skip)]
Dummy149,
#[doc(hidden)]
#[schemars(skip)]
Dummy150,
#[doc(hidden)]
#[schemars(skip)]
Dummy151,
#[doc(hidden)]
#[schemars(skip)]
Dummy152,
#[doc(hidden)]
#[schemars(skip)]
Dummy153,
#[doc(hidden)]
#[schemars(skip)]
Dummy154,
#[doc(hidden)]
#[schemars(skip)]
Dummy155,
#[doc(hidden)]
#[schemars(skip)]
Dummy156,
#[doc(hidden)]
#[schemars(skip)]
Dummy157,
#[doc(hidden)]
#[schemars(skip)]
Dummy158,
#[doc(hidden)]
#[schemars(skip)]
Dummy159,
#[doc(hidden)]
#[schemars(skip)]
Dummy160,
#[doc(hidden)]
#[schemars(skip)]
Dummy161,
#[doc(hidden)]
#[schemars(skip)]
Dummy162,
#[doc(hidden)]
#[schemars(skip)]
Dummy163,
#[doc(hidden)]
#[schemars(skip)]
Dummy164,
#[doc(hidden)]
#[schemars(skip)]
Dummy165,
#[doc(hidden)]
#[schemars(skip)]
Dummy166,
#[doc(hidden)]
#[schemars(skip)]
Dummy167,
#[doc(hidden)]
#[schemars(skip)]
Dummy168,
#[doc(hidden)]
#[schemars(skip)]
Dummy169,
#[doc(hidden)]
#[schemars(skip)]
Dummy170,
#[doc(hidden)]
#[schemars(skip)]
Dummy171,
#[doc(hidden)]
#[schemars(skip)]
Dummy172,
#[doc(hidden)]
#[schemars(skip)]
Dummy173,
#[doc(hidden)]
#[schemars(skip)]
Dummy174,
#[doc(hidden)]
#[schemars(skip)]
Dummy175,
#[doc(hidden)]
#[schemars(skip)]
Dummy176,
#[doc(hidden)]
#[schemars(skip)]
Dummy177,
#[doc(hidden)]
#[schemars(skip)]
Dummy178,
#[doc(hidden)]
#[schemars(skip)]
Dummy179,
#[doc(hidden)]
#[schemars(skip)]
Dummy180,
#[doc(hidden)]
#[schemars(skip)]
Dummy181,
#[doc(hidden)]
#[schemars(skip)]
Dummy182,
#[doc(hidden)]
#[schemars(skip)]
Dummy183,
#[doc(hidden)]
#[schemars(skip)]
Dummy184,
#[doc(hidden)]
#[schemars(skip)]
Dummy185,
#[doc(hidden)]
#[schemars(skip)]
Dummy186,
#[doc(hidden)]
#[schemars(skip)]
Dummy187,
#[doc(hidden)]
#[schemars(skip)]
Dummy188,
#[doc(hidden)]
#[schemars(skip)]
Dummy189,
#[doc(hidden)]
#[schemars(skip)]
Dummy190,
#[doc(hidden)]
#[schemars(skip)]
Dummy191,
#[doc(hidden)]
#[schemars(skip)]
Dummy192,
#[doc(hidden)]
#[schemars(skip)]
Dummy193,
#[doc(hidden)]
#[schemars(skip)]
Dummy194,
#[doc(hidden)]
#[schemars(skip)]
Dummy195,
#[doc(hidden)]
#[schemars(skip)]
Dummy196,
#[doc(hidden)]
#[schemars(skip)]
Dummy197,
#[doc(hidden)]
#[schemars(skip)]
Dummy198,
#[doc(hidden)]
#[schemars(skip)]
Dummy199,
#[doc(hidden)]
#[schemars(skip)]
Dummy200,
#[doc(hidden)]
#[schemars(skip)]
Dummy201,
#[doc(hidden)]
#[schemars(skip)]
Dummy202,
#[doc(hidden)]
#[schemars(skip)]
Dummy203,
#[doc(hidden)]
#[schemars(skip)]
Dummy204,
#[doc(hidden)]
#[schemars(skip)]
Dummy205,
#[doc(hidden)]
#[schemars(skip)]
Dummy206,
#[doc(hidden)]
#[schemars(skip)]
Dummy207,
#[doc(hidden)]
#[schemars(skip)]
Dummy208,
#[doc(hidden)]
#[schemars(skip)]
Dummy209,
#[doc(hidden)]
#[schemars(skip)]
Dummy210,
#[doc(hidden)]
#[schemars(skip)]
Dummy211,
#[doc(hidden)]
#[schemars(skip)]
Dummy212,
#[doc(hidden)]
#[schemars(skip)]
Dummy213,
#[doc(hidden)]
#[schemars(skip)]
Dummy214,
#[doc(hidden)]
#[schemars(skip)]
Dummy215,
#[doc(hidden)]
#[schemars(skip)]
Dummy216,
#[doc(hidden)]
#[schemars(skip)]
Dummy217,
#[doc(hidden)]
#[schemars(skip)]
Dummy218,
#[doc(hidden)]
#[schemars(skip)]
Dummy219,
#[doc(hidden)]
#[schemars(skip)]
Dummy220,
#[doc(hidden)]
#[schemars(skip)]
Dummy221,
#[doc(hidden)]
#[schemars(skip)]
Dummy222,
#[doc(hidden)]
#[schemars(skip)]
Dummy223,
#[doc(hidden)]
#[schemars(skip)]
Dummy224,
#[doc(hidden)]
#[schemars(skip)]
Dummy225,
#[doc(hidden)]
#[schemars(skip)]
Dummy226,
#[doc(hidden)]
#[schemars(skip)]
Dummy227,
#[doc(hidden)]
#[schemars(skip)]
Dummy228,
#[doc(hidden)]
#[schemars(skip)]
Dummy229,
#[doc(hidden)]
#[schemars(skip)]
Dummy230,
#[doc(hidden)]
#[schemars(skip)]
Dummy231,
#[doc(hidden)]
#[schemars(skip)]
Dummy232,
#[doc(hidden)]
#[schemars(skip)]
Dummy233,
#[doc(hidden)]
#[schemars(skip)]
Dummy234,
#[doc(hidden)]
#[schemars(skip)]
Dummy235,
#[doc(hidden)]
#[schemars(skip)]
Dummy236,
#[doc(hidden)]
#[schemars(skip)]
Dummy237,
#[doc(hidden)]
#[schemars(skip)]
Dummy238,
#[doc(hidden)]
#[schemars(skip)]
Dummy239,
#[doc(hidden)]
#[schemars(skip)]
Dummy240,
#[doc(hidden)]
#[schemars(skip)]
Dummy241,
#[doc(hidden)]
#[schemars(skip)]
Dummy242,
#[doc(hidden)]
#[schemars(skip)]
Dummy243,
#[doc(hidden)]
#[schemars(skip)]
Dummy244,
#[doc(hidden)]
#[schemars(skip)]
Dummy245,
#[doc(hidden)]
#[schemars(skip)]
Dummy246,
#[doc(hidden)]
#[schemars(skip)]
Dummy247,
#[doc(hidden)]
#[schemars(skip)]
Dummy248,
#[doc(hidden)]
#[schemars(skip)]
Dummy249,
#[doc(hidden)]
#[schemars(skip)]
Dummy250,
#[doc(hidden)]
#[schemars(skip)]
Dummy251,
#[doc(hidden)]
#[schemars(skip)]
Dummy252,
#[doc(hidden)]
#[schemars(skip)]
Dummy253,
Empty = 254,
Unknown = 255,
P2PK65 = 0,
P2PK33 = 1,
P2PKH = 2,
P2MS = 3,
P2SH = 4,
OpReturn = 5,
P2WPKH = 6,
P2WSH = 7,
P2TR = 8,
P2A = 9,
Empty = u16::MAX - 1,
Unknown = u16::MAX,
}
impl OutputType {
fn is_valid(value: u16) -> bool {
value <= Self::P2A as u16 || value >= Self::Empty as u16
}
pub fn is_spendable(&self) -> bool {
match self {
Self::P2PK65 => true,
@@ -776,7 +46,6 @@ impl OutputType {
Self::P2A => true,
Self::Empty => true,
Self::Unknown => true,
_ => unreachable!(),
}
}
@@ -794,7 +63,6 @@ impl OutputType {
Self::P2A => true,
Self::Empty => false,
Self::Unknown => false,
_ => unreachable!(),
}
}
@@ -924,7 +192,7 @@ impl Bytes for OutputType {
#[inline]
fn to_bytes(&self) -> Self::Array {
[*self as u8]
(*self as u16).to_le_bytes()
}
#[inline]
@@ -935,9 +203,18 @@ impl Bytes for OutputType {
received: bytes.len(),
});
};
// SAFETY: OutputType is repr(u8) and we're transmuting from u8
// All values 0-255 are valid (includes dummy variants)
let s: Self = unsafe { std::mem::transmute(bytes[0]) };
let value = u16::from_le_bytes([bytes[0], bytes[1]]);
if !Self::is_valid(value) {
return Err(vecdb::Error::InvalidArgument("invalid OutputType"));
}
// SAFETY: We validated that value is a valid variant
let s: Self = unsafe { std::mem::transmute(value) };
Ok(s)
}
}
impl Pco for OutputType {
type NumberType = u16;
}
impl TransparentPco<u16> for OutputType {}

1
crates/brk_types/src/sats.rs
View File

@@ -50,6 +50,7 @@ impl Sats {
pub const _100K_BTC: Self = Self(100_000_00_000_000);
pub const ONE_BTC: Self = Self(1_00_000_000);
pub const MAX: Self = Self(u64::MAX);
pub const COINBASE: Self = Self(u64::MAX);
pub const FIFTY_BTC: Self = Self(50_00_000_000);
pub const ONE_BTC_U128: u128 = 1_00_000_000;

100
crates/brk_types/src/txoutdata.rs Normal file
View File

@@ -0,0 +1,100 @@
//! Combined transaction output data for efficient access.
use std::fmt::{self, Display};
use std::mem::size_of;
use schemars::JsonSchema;
use serde::Serialize;
use vecdb::{Bytes, Formattable};
use crate::{OutputType, Sats, TypeIndex};
/// Core transaction output data: value, type, and type index.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)]
#[repr(C)]
pub struct TxOutData {
pub value: Sats,
pub typeindex: TypeIndex,
pub outputtype: OutputType,
_padding: u16,
}
impl TxOutData {
#[inline]
pub const fn new(value: Sats, outputtype: OutputType, typeindex: TypeIndex) -> Self {
Self {
value,
typeindex,
outputtype,
_padding: 0,
}
}
}
impl Bytes for TxOutData {
type Array = [u8; size_of::<Self>()];
#[inline]
fn to_bytes(&self) -> Self::Array {
let mut bytes = [0u8; 16];
bytes[0..8].copy_from_slice(&self.value.to_bytes());
bytes[8..12].copy_from_slice(&self.typeindex.to_bytes());
bytes[12..14].copy_from_slice(&self.outputtype.to_bytes());
// bytes[14..16] is padding, already zero
bytes
}
#[inline]
fn from_bytes(bytes: &[u8]) -> vecdb::Result<Self> {
if bytes.len() != size_of::<Self>() {
return Err(vecdb::Error::WrongLength {
expected: size_of::<Self>(),
received: bytes.len(),
});
}
Ok(Self {
value: Sats::from_bytes(&bytes[0..8])?,
typeindex: TypeIndex::from_bytes(&bytes[8..12])?,
outputtype: OutputType::from_bytes(&bytes[12..14])?,
_padding: 0,
})
}
}
impl Display for TxOutData {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"value: {}, outputtype: {}, typeindex: {}",
self.value, self.outputtype, self.typeindex
)
}
}
impl Formattable for TxOutData {
fn may_need_escaping() -> bool {
true
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_size() {
assert_eq!(size_of::<TxOutData>(), 16);
}
#[test]
fn test_roundtrip() {
let data = TxOutData::new(
Sats::from(123456789u64),
OutputType::P2TR,
TypeIndex::from(42u32),
);
let bytes = data.to_bytes();
let decoded = TxOutData::from_bytes(&bytes).unwrap();
assert_eq!(data, decoded);
}
}

2
crates/brk_types/src/typeindex.rs
View File

@@ -24,6 +24,8 @@ use vecdb::{CheckedSub, Formattable, Pco};
pub struct TypeIndex(u32);
impl TypeIndex {
pub const COINBASE: Self = Self(u32::MAX);
pub fn new(i: u32) -> Self {
Self(i)
}

1
modules/brk-client/.gitignore vendored
View File

@@ -1 +1,2 @@
generated
index.js

11
modules/brk-client/idle.js
View File

@@ -1,11 +0,0 @@
/**
* @param {VoidFunction} callback
* @param {number} [timeout = 1]
*/
export function runWhenIdle(callback, timeout = 1) {
if ("requestIdleCallback" in window) {
requestIdleCallback(callback);
} else {
setTimeout(callback, timeout);
}
}

4677
modules/brk-client/index.js
View File

File diff suppressed because it is too large Load Diff

111
modules/brk-client/metrics.js
View File

@@ -1,111 +0,0 @@
import {
INDEX_TO_WORD,
COMPRESSED_METRIC_TO_INDEXES,
} from "./generated/metrics";
/**
* @typedef {typeof import("./generated/metrics")["COMPRESSED_METRIC_TO_INDEXES"]} MetricToIndexes
* @typedef {string} Metric
*/
/** @type {Record<string, number>} */
const WORD_TO_INDEX = {};
INDEX_TO_WORD.forEach((word, index) => {
WORD_TO_INDEX[word] = index;
});
/**
* @param {Metric} metric
*/
export function getIndexesFromMetric(metric) {
return COMPRESSED_METRIC_TO_INDEXES[compressMetric(metric)];
}
/**
* @param {Metric} metric
*/
export function hasMetric(metric) {
return compressMetric(metric) in COMPRESSED_METRIC_TO_INDEXES;
}
/**
* @param {string} metric
*/
function compressMetric(metric) {
return metric
.split("_")
.map((word) => {
const index = WORD_TO_INDEX[word];
return index !== undefined ? indexToLetters(index) : word;
})
.join("_");
}
/**
* @param {string} compressedMetric
*/
function decompressMetric(compressedMetric) {
return compressedMetric
.split("_")
.map((code) => {
const index = lettersToIndex(code);
return INDEX_TO_WORD[index] || code; // Fallback to original if not found
})
.join("_");
}
/**
* @param {string} letters
*/
function lettersToIndex(letters) {
let result = 0;
for (let i = 0; i < letters.length; i++) {
const value = charToIndex(letters.charCodeAt(i));
result = result * 52 + value + 1;
}
return result - 1;
}
/**
* @param {number} byte
*/
function charToIndex(byte) {
if (byte >= 65 && byte <= 90) {
// 'A' to 'Z'
return byte - 65;
} else if (byte >= 97 && byte <= 122) {
// 'a' to 'z'
return byte - 97 + 26;
} else {
return 255; // Invalid
}
}
/**
* @param {number} index
*/
function indexToLetters(index) {
if (index < 52) {
return indexToChar(index);
}
let result = [];
while (true) {
result.push(indexToChar(index % 52));
index = Math.floor(index / 52);
if (index === 0) break;
index -= 1;
}
return result.reverse().join("");
}
/**
* @param {number} index
*/
function indexToChar(index) {
if (index <= 25) {
return String.fromCharCode(65 + index); // A-Z
} else {
return String.fromCharCode(97 + index - 26); // a-z
}
}