global: snapshot

This commit is contained in:
nym21
2025-12-25 22:21:12 +01:00
parent eadf93b804
commit bbb74b76c8
68 changed files with 6497 additions and 2659 deletions

31
Cargo.lock generated
View File

@@ -742,11 +742,12 @@ dependencies = [
name = "brk_mcp" name = "brk_mcp"
version = "0.1.0-alpha.1" version = "0.1.0-alpha.1"
dependencies = [ dependencies = [
"aide", "axum",
"brk_query",
"brk_rmcp", "brk_rmcp",
"brk_types",
"log", "log",
"minreq",
"schemars",
"serde",
"serde_json", "serde_json",
] ]
@@ -2780,9 +2781,9 @@ checksum = "7ee5b5339afb4c41626dde77b7a611bd4f2c202b897852b4bcf5d03eddc61010"
[[package]] [[package]]
name = "jiff" name = "jiff"
version = "0.2.16" version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" checksum = "a87d9b8105c23642f50cbbae03d1f75d8422c5cb98ce7ee9271f7ff7505be6b8"
dependencies = [ dependencies = [
"jiff-static", "jiff-static",
"jiff-tzdb-platform", "jiff-tzdb-platform",
@@ -2795,9 +2796,9 @@ dependencies = [
[[package]] [[package]]
name = "jiff-static" name = "jiff-static"
version = "0.2.16" version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" checksum = "b787bebb543f8969132630c51fd0afab173a86c6abae56ff3b9e5e3e3f9f6e58"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@@ -3635,14 +3636,15 @@ dependencies = [
[[package]] [[package]]
name = "oxc_resolver" name = "oxc_resolver"
version = "11.16.0" version = "11.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82835b74b32841714c1342b1636992d19622d4ec19666b55edb4c654fb6eb719" checksum = "5467a6fd6e1b2a0cc25f4f89a5ece8594213427e430ba8f0a8f900808553cb1e"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"fast-glob", "fast-glob",
"indexmap", "indexmap",
"json-strip-comments", "json-strip-comments",
"nodejs-built-in-modules",
"once_cell", "once_cell",
"papaya", "papaya",
"parking_lot", "parking_lot",
@@ -4235,8 +4237,6 @@ dependencies = [
[[package]] [[package]]
name = "rawdb" name = "rawdb"
version = "0.4.6" version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c73aead6409391fb8d52ca74985f75983c61a81247de5d78312b7134dd1818a"
dependencies = [ dependencies = [
"libc", "libc",
"log", "log",
@@ -5427,8 +5427,6 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]] [[package]]
name = "vecdb" name = "vecdb"
version = "0.4.6" version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8386c4148b31b9ba931394b0e97f5fc8d5f1644c5b55cdae52611e18227aee5"
dependencies = [ dependencies = [
"ctrlc", "ctrlc",
"log", "log",
@@ -5436,6 +5434,7 @@ dependencies = [
"parking_lot", "parking_lot",
"pco", "pco",
"rawdb", "rawdb",
"schemars",
"serde", "serde",
"serde_json", "serde_json",
"thiserror 2.0.17", "thiserror 2.0.17",
@@ -5447,8 +5446,6 @@ dependencies = [
[[package]] [[package]]
name = "vecdb_derive" name = "vecdb_derive"
version = "0.4.6" version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54a57c4efc56bf5aa76ccf39e52bc4d3154848c13996a1c0779dec4fd21eaf4a"
dependencies = [ dependencies = [
"quote", "quote",
"syn 2.0.111", "syn 2.0.111",
@@ -6042,9 +6039,9 @@ checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3"
[[package]] [[package]]
name = "zmij" name = "zmij"
version = "0.1.7" version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e404bcd8afdaf006e529269d3e85a743f9480c3cef60034d77860d02964f3ba" checksum = "d0095ecd462946aa3927d9297b63ef82fb9a5316d7a37d134eeb36e58228615a"
[[package]] [[package]]
name = "zopfli" name = "zopfli"

View File

@@ -66,7 +66,7 @@ byteview = "0.9.1"
color-eyre = "0.6.5" color-eyre = "0.6.5"
derive_deref = "1.1.1" derive_deref = "1.1.1"
fjall = "3.0.0-rc.6" fjall = "3.0.0-rc.6"
jiff = "0.2.16" jiff = "0.2.17"
log = "0.4.29" log = "0.4.29"
mimalloc = { version = "0.1.48", features = ["v3"] } mimalloc = { version = "0.1.48", features = ["v3"] }
minreq = { version = "2.14.1", features = ["https", "serde_json"] } minreq = { version = "2.14.1", features = ["https", "serde_json"] }
@@ -80,8 +80,8 @@ serde_derive = "1.0.228"
serde_json = { version = "1.0.147", features = ["float_roundtrip"] } serde_json = { version = "1.0.147", features = ["float_roundtrip"] }
smallvec = "1.15.1" smallvec = "1.15.1"
tokio = { version = "1.48.0", features = ["rt-multi-thread"] } tokio = { version = "1.48.0", features = ["rt-multi-thread"] }
vecdb = { version = "0.4.6", features = ["derive", "serde_json", "pco"] } # vecdb = { version = "0.4.6", features = ["derive", "serde_json", "pco"] }
# vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] } vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] }
# vecdb = { git = "https://github.com/anydb-rs/anydb", features = ["derive", "serde_json", "pco"] } # vecdb = { git = "https://github.com/anydb-rs/anydb", features = ["derive", "serde_json", "pco"] }
[workspace.metadata.release] [workspace.metadata.release]

View File

@@ -13,12 +13,14 @@ use crate::{
get_node_fields, get_pattern_instance_base, to_camel_case, to_pascal_case, get_node_fields, get_pattern_instance_base, to_camel_case, to_pascal_case,
}; };
/// Generate JavaScript + JSDoc client from metadata and OpenAPI endpoints /// Generate JavaScript + JSDoc client from metadata and OpenAPI endpoints.
///
/// `output_path` is the full path to the output file (e.g., "modules/brk-client/index.js").
pub fn generate_javascript_client( pub fn generate_javascript_client(
metadata: &ClientMetadata, metadata: &ClientMetadata,
endpoints: &[Endpoint], endpoints: &[Endpoint],
schemas: &TypeSchemas, schemas: &TypeSchemas,
output_dir: &Path, output_path: &Path,
) -> io::Result<()> { ) -> io::Result<()> {
let mut output = String::new(); let mut output = String::new();
@@ -44,7 +46,7 @@ pub fn generate_javascript_client(
// Generate the main client class with tree and API methods // Generate the main client class with tree and API methods
generate_main_client(&mut output, &metadata.catalog, metadata, endpoints); generate_main_client(&mut output, &metadata.catalog, metadata, endpoints);
fs::write(output_dir.join("client.js"), output)?; fs::write(output_path, output)?;
Ok(()) Ok(())
} }
@@ -446,13 +448,21 @@ fn generate_structural_patterns(
// Generate factory function // Generate factory function
writeln!(output, "/**").unwrap(); writeln!(output, "/**").unwrap();
writeln!(output, " * Create a {} pattern node", pattern.name).unwrap(); writeln!(output, " * Create a {} pattern node", pattern.name).unwrap();
if pattern.is_generic {
writeln!(output, " * @template T").unwrap();
}
writeln!(output, " * @param {{BrkClientBase}} client").unwrap(); writeln!(output, " * @param {{BrkClientBase}} client").unwrap();
if is_parameterizable { if is_parameterizable {
writeln!(output, " * @param {{string}} acc - Accumulated metric name").unwrap(); writeln!(output, " * @param {{string}} acc - Accumulated metric name").unwrap();
} else { } else {
writeln!(output, " * @param {{string}} basePath").unwrap(); writeln!(output, " * @param {{string}} basePath").unwrap();
} }
writeln!(output, " * @returns {{{}}}", pattern.name).unwrap(); let return_type = if pattern.is_generic {
format!("{}<T>", pattern.name)
} else {
pattern.name.clone()
};
writeln!(output, " * @returns {{{}}}", return_type).unwrap();
writeln!(output, " */").unwrap(); writeln!(output, " */").unwrap();
let param_name = if is_parameterizable { let param_name = if is_parameterizable {
@@ -613,11 +623,17 @@ fn field_to_js_type_with_generic_value(
}; };
if metadata.is_pattern_type(&field.rust_type) { if metadata.is_pattern_type(&field.rust_type) {
// Check if this pattern is generic and we have a value type // Check if this pattern is generic
if metadata.is_pattern_generic(&field.rust_type) if metadata.is_pattern_generic(&field.rust_type) {
&& let Some(vt) = generic_value_type if let Some(vt) = generic_value_type {
{ return format!("{}<{}>", field.rust_type, vt);
return format!("{}<{}>", field.rust_type, vt); } else if is_generic {
// Propagate T when inside a generic pattern
return format!("{}<T>", field.rust_type);
} else {
// Generic pattern without known type - use unknown
return format!("{}<unknown>", field.rust_type);
}
} }
field.rust_type.clone() field.rust_type.clone()
} else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) { } else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) {
@@ -629,7 +645,6 @@ fn field_to_js_type_with_generic_value(
} }
} }
/// Generate tree typedefs /// Generate tree typedefs
fn generate_tree_typedefs(output: &mut String, catalog: &TreeNode, metadata: &ClientMetadata) { fn generate_tree_typedefs(output: &mut String, catalog: &TreeNode, metadata: &ClientMetadata) {
writeln!(output, "// Catalog tree typedefs\n").unwrap(); writeln!(output, "// Catalog tree typedefs\n").unwrap();
@@ -666,7 +681,8 @@ fn generate_tree_typedef(
.collect(); .collect();
// Skip if this matches a pattern (already generated) // Skip if this matches a pattern (already generated)
if pattern_lookup.contains_key(&fields) && pattern_lookup.get(&fields) != Some(&name.to_string()) if pattern_lookup.contains_key(&fields)
&& pattern_lookup.get(&fields) != Some(&name.to_string())
{ {
return; return;
} }
@@ -680,15 +696,13 @@ fn generate_tree_typedef(
writeln!(output, " * @typedef {{Object}} {}", name).unwrap(); writeln!(output, " * @typedef {{Object}} {}", name).unwrap();
for (field, child_fields) in &fields_with_child_info { for (field, child_fields) in &fields_with_child_info {
// Look up type parameter for generic patterns
let generic_value_type = child_fields let generic_value_type = child_fields
.as_ref() .as_ref()
.and_then(|cf| metadata.get_generic_value_type(&field.rust_type, cf)); .and_then(|cf| metadata.get_type_param(cf))
let js_type = field_to_js_type_with_generic_value( .map(String::as_str);
field, let js_type =
metadata, field_to_js_type_with_generic_value(field, metadata, false, generic_value_type);
false,
generic_value_type.as_deref(),
);
writeln!( writeln!(
output, output,
" * @property {{{}}} {}", " * @property {{{}}} {}",
@@ -899,6 +913,11 @@ fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
if let Some(summary) = &endpoint.summary { if let Some(summary) = &endpoint.summary {
writeln!(output, " * {}", summary).unwrap(); writeln!(output, " * {}", summary).unwrap();
} }
if let Some(desc) = &endpoint.description
&& endpoint.summary.as_ref() != Some(desc)
{
writeln!(output, " * @description {}", desc).unwrap();
}
for param in &endpoint.path_params { for param in &endpoint.path_params {
let desc = param.description.as_deref().unwrap_or(""); let desc = param.description.as_deref().unwrap_or("");

View File

@@ -27,14 +27,57 @@
//! 2. **Schema collection** - Merges OpenAPI schemas with schemars-generated type schemas //! 2. **Schema collection** - Merges OpenAPI schemas with schemars-generated type schemas
//! //!
//! 3. **Code generation** - Produces language-specific clients: //! 3. **Code generation** - Produces language-specific clients:
//! - Rust: Uses `brk_types` directly, generates structs with lifetimes //! - Rust: Uses `brk_types` directly, generates structs with Arc-based sharing
//! - JavaScript: Generates JSDoc-typed ES modules with factory functions //! - JavaScript: Generates JSDoc-typed ES modules with factory functions
//! - Python: Generates typed classes with TypedDict and Generic support //! - Python: Generates typed classes with TypedDict and Generic support
use std::{collections::btree_map::Entry, fs::create_dir_all, io, path::Path}; use std::{collections::btree_map::Entry, fs::create_dir_all, io, path::PathBuf};
use brk_query::Vecs; use brk_query::Vecs;
/// Output path configuration for each language client.
///
/// Each path should be the full path to the output file, not just a directory.
/// Parent directories will be created automatically if they don't exist.
///
/// # Example
/// ```ignore
/// let paths = ClientOutputPaths::new()
/// .rust("crates/brk_client/src/lib.rs")
/// .javascript("modules/brk-client/index.js")
/// .python("packages/brk_client/__init__.py");
/// ```
#[derive(Debug, Clone, Default)]
pub struct ClientOutputPaths {
/// Full path to Rust client file (e.g., "crates/brk_client/src/lib.rs")
pub rust: Option<PathBuf>,
/// Full path to JavaScript client file (e.g., "modules/brk-client/index.js")
pub javascript: Option<PathBuf>,
/// Full path to Python client file (e.g., "packages/brk_client/__init__.py")
pub python: Option<PathBuf>,
}
impl ClientOutputPaths {
pub fn new() -> Self {
Self::default()
}
pub fn rust(mut self, path: impl Into<PathBuf>) -> Self {
self.rust = Some(path.into());
self
}
pub fn javascript(mut self, path: impl Into<PathBuf>) -> Self {
self.javascript = Some(path.into());
self
}
pub fn python(mut self, path: impl Into<PathBuf>) -> Self {
self.python = Some(path.into());
self
}
}
mod javascript; mod javascript;
mod js; mod js;
mod openapi; mod openapi;
@@ -51,8 +94,25 @@ pub use types::*;
pub const VERSION: &str = env!("CARGO_PKG_VERSION"); pub const VERSION: &str = env!("CARGO_PKG_VERSION");
/// Generate all client libraries from the query vecs and OpenAPI JSON /// Generate all client libraries from the query vecs and OpenAPI JSON.
pub fn generate_clients(vecs: &Vecs, openapi_json: &str, output_dir: &Path) -> io::Result<()> { ///
/// Uses `ClientOutputPaths` to specify the output file path for each language.
/// Only languages with a configured path will be generated.
///
/// # Example
/// ```ignore
/// let paths = ClientOutputPaths::new()
/// .rust("crates/brk_client/src/lib.rs")
/// .javascript("modules/brk-client/index.js")
/// .python("packages/brk_client/__init__.py");
///
/// generate_clients(&vecs, &openapi_json, &paths)?;
/// ```
pub fn generate_clients(
vecs: &Vecs,
openapi_json: &str,
output_paths: &ClientOutputPaths,
) -> io::Result<()> {
let metadata = ClientMetadata::from_vecs(vecs); let metadata = ClientMetadata::from_vecs(vecs);
// Parse OpenAPI spec // Parse OpenAPI spec
@@ -71,19 +131,28 @@ pub fn generate_clients(vecs: &Vecs, openapi_json: &str, output_dir: &Path) -> i
} }
// Generate Rust client (uses real brk_types, no schema conversion needed) // Generate Rust client (uses real brk_types, no schema conversion needed)
let rust_path = output_dir.join("rust"); if let Some(rust_path) = &output_paths.rust {
create_dir_all(&rust_path)?; if let Some(parent) = rust_path.parent() {
generate_rust_client(&metadata, &endpoints, &rust_path)?; create_dir_all(parent)?;
}
generate_rust_client(&metadata, &endpoints, rust_path)?;
}
// Generate JavaScript client (needs schemas for type definitions) // Generate JavaScript client (needs schemas for type definitions)
let js_path = output_dir.join("javascript"); if let Some(js_path) = &output_paths.javascript {
create_dir_all(&js_path)?; if let Some(parent) = js_path.parent() {
generate_javascript_client(&metadata, &endpoints, &schemas, &js_path)?; create_dir_all(parent)?;
}
generate_javascript_client(&metadata, &endpoints, &schemas, js_path)?;
}
// Generate Python client (needs schemas for type definitions) // Generate Python client (needs schemas for type definitions)
let python_path = output_dir.join("python"); if let Some(python_path) = &output_paths.python {
create_dir_all(&python_path)?; if let Some(parent) = python_path.parent() {
generate_python_client(&metadata, &endpoints, &schemas, &python_path)?; create_dir_all(parent)?;
}
generate_python_client(&metadata, &endpoints, &schemas, python_path)?;
}
Ok(()) Ok(())
} }

View File

@@ -17,8 +17,10 @@ pub struct Endpoint {
pub path: String, pub path: String,
/// Operation ID (e.g., "getBlockByHash") /// Operation ID (e.g., "getBlockByHash")
pub operation_id: Option<String>, pub operation_id: Option<String>,
/// Summary/description /// Short summary
pub summary: Option<String>, pub summary: Option<String>,
/// Detailed description
pub description: Option<String>,
/// Tags for grouping /// Tags for grouping
pub tags: Vec<String>, pub tags: Vec<String>,
/// Path parameters /// Path parameters
@@ -185,10 +187,8 @@ fn extract_endpoint(path: &str, method: &str, operation: &Operation) -> Option<E
method: method.to_string(), method: method.to_string(),
path: path.to_string(), path: path.to_string(),
operation_id: operation.operation_id.clone(), operation_id: operation.operation_id.clone(),
summary: operation summary: operation.summary.clone(),
.summary description: operation.description.clone(),
.clone()
.or_else(|| operation.description.clone()),
tags: operation.tags.clone(), tags: operation.tags.clone(),
path_params, path_params,
query_params, query_params,

View File

@@ -13,12 +13,14 @@ use crate::{
get_pattern_instance_base, is_enum_schema, to_pascal_case, to_snake_case, get_pattern_instance_base, is_enum_schema, to_pascal_case, to_snake_case,
}; };
/// Generate Python client from metadata and OpenAPI endpoints /// Generate Python client from metadata and OpenAPI endpoints.
///
/// `output_path` is the full path to the output file (e.g., "packages/brk_client/__init__.py").
pub fn generate_python_client( pub fn generate_python_client(
metadata: &ClientMetadata, metadata: &ClientMetadata,
endpoints: &[Endpoint], endpoints: &[Endpoint],
schemas: &TypeSchemas, schemas: &TypeSchemas,
output_dir: &Path, output_path: &Path,
) -> io::Result<()> { ) -> io::Result<()> {
let mut output = String::new(); let mut output = String::new();
@@ -57,7 +59,7 @@ pub fn generate_python_client(
// Generate main client with tree and API methods // Generate main client with tree and API methods
generate_main_client(&mut output, endpoints); generate_main_client(&mut output, endpoints);
fs::write(output_dir.join("client.py"), output)?; fs::write(output_path, output)?;
Ok(()) Ok(())
} }
@@ -720,14 +722,16 @@ fn generate_tree_class(
for ((field, child_fields_opt), (child_name, child_node)) in for ((field, child_fields_opt), (child_name, child_node)) in
fields_with_child_info.iter().zip(children.iter()) fields_with_child_info.iter().zip(children.iter())
{ {
// Look up type parameter for generic patterns
let generic_value_type = child_fields_opt let generic_value_type = child_fields_opt
.as_ref() .as_ref()
.and_then(|cf| metadata.get_generic_value_type(&field.rust_type, cf)); .and_then(|cf| metadata.get_type_param(cf))
.map(String::as_str);
let py_type = field_to_python_type_with_generic_value( let py_type = field_to_python_type_with_generic_value(
field, field,
metadata, metadata,
false, false,
generic_value_type.as_deref(), generic_value_type,
); );
let field_name_py = to_snake_case(&field.name); let field_name_py = to_snake_case(&field.name);
@@ -864,8 +868,19 @@ fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
.unwrap(); .unwrap();
// Docstring // Docstring
if let Some(summary) = &endpoint.summary { match (&endpoint.summary, &endpoint.description) {
writeln!(output, " \"\"\"{}\"\"\"", summary).unwrap(); (Some(summary), Some(desc)) if summary != desc => {
writeln!(output, " \"\"\"{}.", summary.trim_end_matches('.')).unwrap();
writeln!(output).unwrap();
writeln!(output, " {}\"\"\"", desc).unwrap();
}
(Some(summary), _) => {
writeln!(output, " \"\"\"{}\"\"\"", summary).unwrap();
}
(None, Some(desc)) => {
writeln!(output, " \"\"\"{}\"\"\"", desc).unwrap();
}
(None, None) => {}
} }
// Build path // Build path

View File

@@ -12,11 +12,13 @@ use crate::{
to_pascal_case, to_snake_case, to_pascal_case, to_snake_case,
}; };
/// Generate Rust client from metadata and OpenAPI endpoints /// Generate Rust client from metadata and OpenAPI endpoints.
///
/// `output_path` is the full path to the output file (e.g., "crates/brk_client/src/lib.rs").
pub fn generate_rust_client( pub fn generate_rust_client(
metadata: &ClientMetadata, metadata: &ClientMetadata,
endpoints: &[Endpoint], endpoints: &[Endpoint],
output_dir: &Path, output_path: &Path,
) -> io::Result<()> { ) -> io::Result<()> {
let mut output = String::new(); let mut output = String::new();
@@ -47,7 +49,7 @@ pub fn generate_rust_client(
// Generate main client with API methods // Generate main client with API methods
generate_main_client(&mut output, endpoints); generate_main_client(&mut output, endpoints);
fs::write(output_dir.join("client.rs"), output)?; fs::write(output_path, output)?;
Ok(()) Ok(())
} }
@@ -55,7 +57,7 @@ pub fn generate_rust_client(
fn generate_imports(output: &mut String) { fn generate_imports(output: &mut String) {
writeln!( writeln!(
output, output,
r#"use std::marker::PhantomData; r#"use std::sync::Arc;
use serde::de::DeserializeOwned; use serde::de::DeserializeOwned;
use brk_types::*; use brk_types::*;
@@ -88,14 +90,14 @@ pub type Result<T> = std::result::Result<T, BrkError>;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct BrkClientOptions {{ pub struct BrkClientOptions {{
pub base_url: String, pub base_url: String,
pub timeout_ms: u64, pub timeout_secs: u64,
}} }}
impl Default for BrkClientOptions {{ impl Default for BrkClientOptions {{
fn default() -> Self {{ fn default() -> Self {{
Self {{ Self {{
base_url: "http://localhost:3000".to_string(), base_url: "http://localhost:3000".to_string(),
timeout_ms: 30000, timeout_secs: 30,
}} }}
}} }}
}} }}
@@ -104,36 +106,41 @@ impl Default for BrkClientOptions {{
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct BrkClientBase {{ pub struct BrkClientBase {{
base_url: String, base_url: String,
client: reqwest::blocking::Client, timeout_secs: u64,
}} }}
impl BrkClientBase {{ impl BrkClientBase {{
/// Create a new client with the given base URL. /// Create a new client with the given base URL.
pub fn new(base_url: impl Into<String>) -> Result<Self> {{ pub fn new(base_url: impl Into<String>) -> Self {{
let base_url = base_url.into(); Self {{
let client = reqwest::blocking::Client::new(); base_url: base_url.into(),
Ok(Self {{ base_url, client }}) timeout_secs: 30,
}}
}} }}
/// Create a new client with options. /// Create a new client with options.
pub fn with_options(options: BrkClientOptions) -> Result<Self> {{ pub fn with_options(options: BrkClientOptions) -> Self {{
let client = reqwest::blocking::Client::builder() Self {{
.timeout(std::time::Duration::from_millis(options.timeout_ms))
.build()
.map_err(|e| BrkError {{ message: e.to_string() }})?;
Ok(Self {{
base_url: options.base_url, base_url: options.base_url,
client, timeout_secs: options.timeout_secs,
}}) }}
}} }}
/// Make a GET request. /// Make a GET request.
pub fn get<T: DeserializeOwned>(&self, path: &str) -> Result<T> {{ pub fn get<T: DeserializeOwned>(&self, path: &str) -> Result<T> {{
let url = format!("{{}}{{}}", self.base_url, path); let url = format!("{{}}{{}}", self.base_url, path);
self.client let response = minreq::get(&url)
.get(&url) .with_timeout(self.timeout_secs)
.send() .send()
.map_err(|e| BrkError {{ message: e.to_string() }})? .map_err(|e| BrkError {{ message: e.to_string() }})?;
if response.status_code >= 400 {{
return Err(BrkError {{
message: format!("HTTP {{}}", response.status_code),
}});
}}
response
.json() .json()
.map_err(|e| BrkError {{ message: e.to_string() }}) .map_err(|e| BrkError {{ message: e.to_string() }})
}} }}
@@ -148,18 +155,18 @@ fn generate_metric_node(output: &mut String) {
writeln!( writeln!(
output, output,
r#"/// A metric node that can fetch data for different indexes. r#"/// A metric node that can fetch data for different indexes.
pub struct MetricNode<'a, T> {{ pub struct MetricNode<T> {{
client: &'a BrkClientBase, client: Arc<BrkClientBase>,
path: String, path: String,
_marker: PhantomData<T>, _marker: std::marker::PhantomData<T>,
}} }}
impl<'a, T: DeserializeOwned> MetricNode<'a, T> {{ impl<T: DeserializeOwned> MetricNode<T> {{
pub fn new(client: &'a BrkClientBase, path: String) -> Self {{ pub fn new(client: Arc<BrkClientBase>, path: String) -> Self {{
Self {{ Self {{
client, client,
path, path,
_marker: PhantomData, _marker: std::marker::PhantomData,
}} }}
}} }}
@@ -168,7 +175,7 @@ impl<'a, T: DeserializeOwned> MetricNode<'a, T> {{
self.client.get(&self.path) self.client.get(&self.path)
}} }}
/// Fetch data points within a date range. /// Fetch data points within a range.
pub fn get_range(&self, from: &str, to: &str) -> Result<Vec<T>> {{ pub fn get_range(&self, from: &str, to: &str) -> Result<Vec<T>> {{
let path = format!("{{}}?from={{}}&to={{}}", self.path, from, to); let path = format!("{{}}?from={{}}&to={{}}", self.path, from, to);
self.client.get(&path) self.client.get(&path)
@@ -195,26 +202,20 @@ fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) {
pattern.indexes.len() pattern.indexes.len()
) )
.unwrap(); .unwrap();
writeln!(output, "pub struct {}<'a, T> {{", pattern.name).unwrap(); writeln!(output, "pub struct {}<T> {{", pattern.name).unwrap();
for index in &pattern.indexes { for index in &pattern.indexes {
let field_name = index_to_field_name(index); let field_name = index_to_field_name(index);
writeln!(output, " pub {}: MetricNode<'a, T>,", field_name).unwrap(); writeln!(output, " pub {}: MetricNode<T>,", field_name).unwrap();
} }
writeln!(output, " _marker: PhantomData<T>,").unwrap();
writeln!(output, "}}\n").unwrap(); writeln!(output, "}}\n").unwrap();
// Generate impl block with constructor // Generate impl block with constructor
writeln!(output, "impl<T: DeserializeOwned> {}<T> {{", pattern.name).unwrap();
writeln!( writeln!(
output, output,
"impl<'a, T: DeserializeOwned> {}<'a, T> {{", " pub fn new(client: Arc<BrkClientBase>, base_path: &str) -> Self {{"
pattern.name
)
.unwrap();
writeln!(
output,
" pub fn new(client: &'a BrkClientBase, base_path: &str) -> Self {{"
) )
.unwrap(); .unwrap();
writeln!(output, " Self {{").unwrap(); writeln!(output, " Self {{").unwrap();
@@ -224,13 +225,12 @@ fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) {
let path_segment = index.serialize_long(); let path_segment = index.serialize_long();
writeln!( writeln!(
output, output,
" {}: MetricNode::new(client, format!(\"{{base_path}}/{}\")),", " {}: MetricNode::new(client.clone(), format!(\"{{base_path}}/{}\")),",
field_name, path_segment field_name, path_segment
) )
.unwrap(); .unwrap();
} }
writeln!(output, " _marker: PhantomData,").unwrap();
writeln!(output, " }}").unwrap(); writeln!(output, " }}").unwrap();
writeln!(output, " }}").unwrap(); writeln!(output, " }}").unwrap();
writeln!(output, "}}\n").unwrap(); writeln!(output, "}}\n").unwrap();
@@ -256,11 +256,7 @@ fn generate_pattern_structs(
for pattern in patterns { for pattern in patterns {
let is_parameterizable = pattern.is_parameterizable(); let is_parameterizable = pattern.is_parameterizable();
let generic_params = if pattern.is_generic { let generic_params = if pattern.is_generic { "<T>" } else { "" };
"<'a, T>"
} else {
"<'a>"
};
writeln!(output, "/// Pattern struct for repeated tree structure.").unwrap(); writeln!(output, "/// Pattern struct for repeated tree structure.").unwrap();
writeln!(output, "pub struct {}{} {{", pattern.name, generic_params).unwrap(); writeln!(output, "pub struct {}{} {{", pattern.name, generic_params).unwrap();
@@ -275,10 +271,15 @@ fn generate_pattern_structs(
writeln!(output, "}}\n").unwrap(); writeln!(output, "}}\n").unwrap();
// Generate impl block with constructor // Generate impl block with constructor
let impl_generic = if pattern.is_generic {
"<T: DeserializeOwned>"
} else {
""
};
writeln!( writeln!(
output, output,
"impl{} {}{} {{", "impl{} {}{} {{",
generic_params, pattern.name, generic_params impl_generic, pattern.name, generic_params
) )
.unwrap(); .unwrap();
@@ -290,13 +291,13 @@ fn generate_pattern_structs(
.unwrap(); .unwrap();
writeln!( writeln!(
output, output,
" pub fn new(client: &'a BrkClientBase, acc: &str) -> Self {{" " pub fn new(client: Arc<BrkClientBase>, acc: &str) -> Self {{"
) )
.unwrap(); .unwrap();
} else { } else {
writeln!( writeln!(
output, output,
" pub fn new(client: &'a BrkClientBase, base_path: &str) -> Self {{" " pub fn new(client: Arc<BrkClientBase>, base_path: &str) -> Self {{"
) )
.unwrap(); .unwrap();
} }
@@ -340,7 +341,7 @@ fn generate_parameterized_rust_field(
writeln!( writeln!(
output, output,
" {}: {}::new(client, {}),", " {}: {}::new(client.clone(), {}),",
field_name, field.rust_type, child_acc field_name, field.rust_type, child_acc
) )
.unwrap(); .unwrap();
@@ -363,14 +364,14 @@ fn generate_parameterized_rust_field(
let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap(); let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap();
writeln!( writeln!(
output, output,
" {}: {}::new(client, &{}),", " {}: {}::new(client.clone(), &{}),",
field_name, accessor.name, metric_expr field_name, accessor.name, metric_expr
) )
.unwrap(); .unwrap();
} else { } else {
writeln!( writeln!(
output, output,
" {}: MetricNode::new(client, {}),", " {}: MetricNode::new(client.clone(), {}),",
field_name, metric_expr field_name, metric_expr
) )
.unwrap(); .unwrap();
@@ -388,7 +389,7 @@ fn generate_tree_path_rust_field(
if metadata.is_pattern_type(&field.rust_type) { if metadata.is_pattern_type(&field.rust_type) {
writeln!( writeln!(
output, output,
" {}: {}::new(client, &format!(\"{{base_path}}/{}\")),", " {}: {}::new(client.clone(), &format!(\"{{base_path}}/{}\")),",
field_name, field.rust_type, field.name field_name, field.rust_type, field.name
) )
.unwrap(); .unwrap();
@@ -396,14 +397,14 @@ fn generate_tree_path_rust_field(
let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap(); let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap();
writeln!( writeln!(
output, output,
" {}: {}::new(client, &format!(\"{{base_path}}/{}\")),", " {}: {}::new(client.clone(), &format!(\"{{base_path}}/{}\")),",
field_name, accessor.name, field.name field_name, accessor.name, field.name
) )
.unwrap(); .unwrap();
} else { } else {
writeln!( writeln!(
output, output,
" {}: MetricNode::new(client, format!(\"{{base_path}}/{}\")),", " {}: MetricNode::new(client.clone(), format!(\"{{base_path}}/{}\")),",
field_name, field.name field_name, field.name
) )
.unwrap(); .unwrap();
@@ -441,15 +442,16 @@ fn field_to_type_annotation_with_generic(
if metadata.is_pattern_generic(&field.rust_type) if metadata.is_pattern_generic(&field.rust_type)
&& let Some(vt) = generic_value_type && let Some(vt) = generic_value_type
{ {
return format!("{}<'a, {}>", field.rust_type, vt); return format!("{}<{}>", field.rust_type, vt);
} }
format!("{}<'a>", field.rust_type) // Non-generic pattern has no type params
field.rust_type.clone()
} else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) { } else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) {
// Leaf with a reusable accessor pattern // Leaf with a reusable accessor pattern
format!("{}<'a, {}>", accessor.name, value_type) format!("{}<{}>", accessor.name, value_type)
} else { } else {
// Leaf with unique index set - use MetricNode directly // Leaf with unique index set - use MetricNode directly
format!("MetricNode<'a, {}>", value_type) format!("MetricNode<{}>", value_type)
} }
} }
@@ -501,29 +503,27 @@ fn generate_tree_node(
generated.insert(name.to_string()); generated.insert(name.to_string());
writeln!(output, "/// Catalog tree node.").unwrap(); writeln!(output, "/// Catalog tree node.").unwrap();
writeln!(output, "pub struct {}<'a> {{", name).unwrap(); writeln!(output, "pub struct {} {{", name).unwrap();
for (field, child_fields) in &fields_with_child_info { for (field, child_fields) in &fields_with_child_info {
let field_name = to_snake_case(&field.name); let field_name = to_snake_case(&field.name);
// Look up type parameter for generic patterns
let generic_value_type = child_fields let generic_value_type = child_fields
.as_ref() .as_ref()
.and_then(|cf| metadata.get_generic_value_type(&field.rust_type, cf)); .and_then(|cf| metadata.get_type_param(cf))
let type_annotation = field_to_type_annotation_with_generic( .map(String::as_str);
field, let type_annotation =
metadata, field_to_type_annotation_with_generic(field, metadata, false, generic_value_type);
false,
generic_value_type.as_deref(),
);
writeln!(output, " pub {}: {},", field_name, type_annotation).unwrap(); writeln!(output, " pub {}: {},", field_name, type_annotation).unwrap();
} }
writeln!(output, "}}\n").unwrap(); writeln!(output, "}}\n").unwrap();
// Generate impl block // Generate impl block
writeln!(output, "impl<'a> {}<'a> {{", name).unwrap(); writeln!(output, "impl {} {{", name).unwrap();
writeln!( writeln!(
output, output,
" pub fn new(client: &'a BrkClientBase, base_path: &str) -> Self {{" " pub fn new(client: Arc<BrkClientBase>, base_path: &str) -> Self {{"
) )
.unwrap(); .unwrap();
writeln!(output, " Self {{").unwrap(); writeln!(output, " Self {{").unwrap();
@@ -538,14 +538,14 @@ fn generate_tree_node(
let metric_base = get_pattern_instance_base(child_node, child_name); let metric_base = get_pattern_instance_base(child_node, child_name);
writeln!( writeln!(
output, output,
" {}: {}::new(client, \"{}\"),", " {}: {}::new(client.clone(), \"{}\"),",
field_name, field.rust_type, metric_base field_name, field.rust_type, metric_base
) )
.unwrap(); .unwrap();
} else { } else {
writeln!( writeln!(
output, output,
" {}: {}::new(client, &format!(\"{{base_path}}/{}\")),", " {}: {}::new(client.clone(), &format!(\"{{base_path}}/{}\")),",
field_name, field.rust_type, field.name field_name, field.rust_type, field.name
) )
.unwrap(); .unwrap();
@@ -560,14 +560,14 @@ fn generate_tree_node(
if metric_path.contains("{base_path}") { if metric_path.contains("{base_path}") {
writeln!( writeln!(
output, output,
" {}: {}::new(client, &format!(\"{}\")),", " {}: {}::new(client.clone(), &format!(\"{}\")),",
field_name, accessor.name, metric_path field_name, accessor.name, metric_path
) )
.unwrap(); .unwrap();
} else { } else {
writeln!( writeln!(
output, output,
" {}: {}::new(client, \"{}\"),", " {}: {}::new(client.clone(), \"{}\"),",
field_name, accessor.name, metric_path field_name, accessor.name, metric_path
) )
.unwrap(); .unwrap();
@@ -581,14 +581,14 @@ fn generate_tree_node(
if metric_path.contains("{base_path}") { if metric_path.contains("{base_path}") {
writeln!( writeln!(
output, output,
" {}: MetricNode::new(client, format!(\"{}\")),", " {}: MetricNode::new(client.clone(), format!(\"{}\")),",
field_name, metric_path field_name, metric_path
) )
.unwrap(); .unwrap();
} else { } else {
writeln!( writeln!(
output, output,
" {}: MetricNode::new(client, \"{}\".to_string()),", " {}: MetricNode::new(client.clone(), \"{}\".to_string()),",
field_name, metric_path field_name, metric_path
) )
.unwrap(); .unwrap();
@@ -625,27 +625,28 @@ fn generate_main_client(output: &mut String, endpoints: &[Endpoint]) {
output, output,
r#"/// Main BRK client with catalog tree and API methods. r#"/// Main BRK client with catalog tree and API methods.
pub struct BrkClient {{ pub struct BrkClient {{
base: BrkClientBase, base: Arc<BrkClientBase>,
tree: CatalogTree,
}} }}
impl BrkClient {{ impl BrkClient {{
/// Create a new client with the given base URL. /// Create a new client with the given base URL.
pub fn new(base_url: impl Into<String>) -> Result<Self> {{ pub fn new(base_url: impl Into<String>) -> Self {{
Ok(Self {{ let base = Arc::new(BrkClientBase::new(base_url));
base: BrkClientBase::new(base_url)?, let tree = CatalogTree::new(base.clone(), "");
}}) Self {{ base, tree }}
}} }}
/// Create a new client with options. /// Create a new client with options.
pub fn with_options(options: BrkClientOptions) -> Result<Self> {{ pub fn with_options(options: BrkClientOptions) -> Self {{
Ok(Self {{ let base = Arc::new(BrkClientBase::with_options(options));
base: BrkClientBase::with_options(options)?, let tree = CatalogTree::new(base.clone(), "");
}}) Self {{ base, tree }}
}} }}
/// Get the catalog tree for navigating metrics. /// Get the catalog tree for navigating metrics.
pub fn tree(&self) -> CatalogTree<'_> {{ pub fn tree(&self) -> &CatalogTree {{
CatalogTree::new(&self.base, "") &self.tree
}} }}
"# "#
) )
@@ -678,6 +679,12 @@ fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
endpoint.summary.as_deref().unwrap_or(&method_name) endpoint.summary.as_deref().unwrap_or(&method_name)
) )
.unwrap(); .unwrap();
if let Some(desc) = &endpoint.description
&& endpoint.summary.as_ref() != Some(desc)
{
writeln!(output, " ///").unwrap();
writeln!(output, " /// {}", desc).unwrap();
}
// Build method signature // Build method signature
let params = build_method_params(endpoint); let params = build_method_params(endpoint);

View File

@@ -39,14 +39,16 @@ pub struct ClientMetadata {
/// Index set patterns - sets of indexes that appear together on metrics /// Index set patterns - sets of indexes that appear together on metrics
pub index_set_patterns: Vec<IndexSetPattern>, pub index_set_patterns: Vec<IndexSetPattern>,
/// Maps concrete field signatures to pattern names /// Maps concrete field signatures to pattern names
pub concrete_to_pattern: HashMap<Vec<PatternField>, String>, concrete_to_pattern: HashMap<Vec<PatternField>, String>,
/// Maps concrete field signatures to their type parameter (for generic patterns)
concrete_to_type_param: HashMap<Vec<PatternField>, String>,
} }
impl ClientMetadata { impl ClientMetadata {
/// Extract metadata from brk_query::Vecs. /// Extract metadata from brk_query::Vecs.
pub fn from_vecs(vecs: &Vecs) -> Self { pub fn from_vecs(vecs: &Vecs) -> Self {
let catalog = vecs.catalog().clone(); let catalog = vecs.catalog().clone();
let (structural_patterns, concrete_to_pattern) = let (structural_patterns, concrete_to_pattern, concrete_to_type_param) =
patterns::detect_structural_patterns(&catalog); patterns::detect_structural_patterns(&catalog);
let (used_indexes, index_set_patterns) = tree::detect_index_patterns(&catalog); let (used_indexes, index_set_patterns) = tree::detect_index_patterns(&catalog);
@@ -56,6 +58,7 @@ impl ClientMetadata {
used_indexes, used_indexes,
index_set_patterns, index_set_patterns,
concrete_to_pattern, concrete_to_pattern,
concrete_to_type_param,
} }
} }
@@ -81,19 +84,9 @@ impl ClientMetadata {
self.find_pattern(name).is_some_and(|p| p.is_generic) self.find_pattern(name).is_some_and(|p| p.is_generic)
} }
/// Extract the value type from concrete fields for a generic pattern. /// Get the type parameter for a generic pattern given its concrete fields.
pub fn get_generic_value_type( pub fn get_type_param(&self, fields: &[PatternField]) -> Option<&String> {
&self, self.concrete_to_type_param.get(fields)
pattern_name: &str,
fields: &[PatternField],
) -> Option<String> {
if !self.is_pattern_generic(pattern_name) {
return None;
}
fields
.iter()
.find(|f| f.is_leaf())
.map(|f| extract_inner_type(&f.rust_type))
} }
/// Build a lookup map from field signatures to pattern names. /// Build a lookup map from field signatures to pattern names.

View File

@@ -1,19 +1,24 @@
//! Pattern detection for structural patterns in the metric tree. //! Pattern detection for structural patterns in the metric tree.
use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::collections::{BTreeSet, HashMap};
use brk_types::TreeNode; use brk_types::TreeNode;
use super::{ use super::{
case::to_pascal_case, schema::schema_to_json_type, FieldNamePosition, PatternField, FieldNamePosition, PatternField, StructuralPattern, case::to_pascal_case,
StructuralPattern, schema::schema_to_json_type,
tree::{get_first_leaf_name, get_node_fields},
}; };
/// Detect structural patterns in the tree using a bottom-up approach. /// Detect structural patterns in the tree using a bottom-up approach.
/// Returns (patterns, concrete_to_pattern_mapping). /// Returns (patterns, concrete_to_pattern, concrete_to_type_param).
pub fn detect_structural_patterns( pub fn detect_structural_patterns(
tree: &TreeNode, tree: &TreeNode,
) -> (Vec<StructuralPattern>, HashMap<Vec<PatternField>, String>) { ) -> (
Vec<StructuralPattern>,
HashMap<Vec<PatternField>, String>,
HashMap<Vec<PatternField>, String>,
) {
let mut signature_to_pattern: HashMap<Vec<PatternField>, String> = HashMap::new(); let mut signature_to_pattern: HashMap<Vec<PatternField>, String> = HashMap::new();
let mut signature_counts: HashMap<Vec<PatternField>, usize> = HashMap::new(); let mut signature_counts: HashMap<Vec<PatternField>, usize> = HashMap::new();
let mut normalized_to_name: HashMap<Vec<PatternField>, String> = HashMap::new(); let mut normalized_to_name: HashMap<Vec<PatternField>, String> = HashMap::new();
@@ -29,8 +34,9 @@ pub fn detect_structural_patterns(
&mut name_counts, &mut name_counts,
); );
// Identify generic patterns // Identify generic patterns (also extracts type params)
let (generic_patterns, generic_mappings) = detect_generic_patterns(&signature_to_pattern); let (generic_patterns, generic_mappings, type_mappings) =
detect_generic_patterns(&signature_to_pattern);
// Build non-generic patterns: signatures appearing 2+ times that weren't merged into generics // Build non-generic patterns: signatures appearing 2+ times that weren't merged into generics
let mut patterns: Vec<StructuralPattern> = signature_to_pattern let mut patterns: Vec<StructuralPattern> = signature_to_pattern
@@ -64,33 +70,43 @@ pub fn detect_structural_patterns(
analyze_pattern_field_positions(tree, &mut patterns, &pattern_lookup); analyze_pattern_field_positions(tree, &mut patterns, &pattern_lookup);
patterns.sort_by(|a, b| b.fields.len().cmp(&a.fields.len())); patterns.sort_by(|a, b| b.fields.len().cmp(&a.fields.len()));
(patterns, concrete_to_pattern) (patterns, concrete_to_pattern, type_mappings)
} }
/// Detect generic patterns by grouping signatures by their normalized form. /// Detect generic patterns by grouping signatures by their normalized form.
/// Returns (patterns, concrete_to_pattern, concrete_to_type_param).
fn detect_generic_patterns( fn detect_generic_patterns(
signature_to_pattern: &HashMap<Vec<PatternField>, String>, signature_to_pattern: &HashMap<Vec<PatternField>, String>,
) -> (Vec<StructuralPattern>, HashMap<Vec<PatternField>, String>) { ) -> (
let mut normalized_groups: HashMap<Vec<PatternField>, Vec<(Vec<PatternField>, String)>> = Vec<StructuralPattern>,
HashMap::new(); HashMap<Vec<PatternField>, String>,
HashMap<Vec<PatternField>, String>,
) {
// Group by normalized form, tracking the extracted type for each concrete signature
let mut normalized_groups: HashMap<
Vec<PatternField>,
Vec<(Vec<PatternField>, String, String)>,
> = HashMap::new();
for (fields, name) in signature_to_pattern { for (fields, name) in signature_to_pattern {
if let Some(normalized) = normalize_fields_for_generic(fields) { if let Some((normalized, extracted_type)) = normalize_fields_for_generic(fields) {
normalized_groups normalized_groups
.entry(normalized) .entry(normalized)
.or_default() .or_default()
.push((fields.clone(), name.clone())); .push((fields.clone(), name.clone(), extracted_type));
} }
} }
let mut patterns = Vec::new(); let mut patterns = Vec::new();
let mut mappings: HashMap<Vec<PatternField>, String> = HashMap::new(); let mut pattern_mappings: HashMap<Vec<PatternField>, String> = HashMap::new();
let mut type_mappings: HashMap<Vec<PatternField>, String> = HashMap::new();
for (normalized_fields, group) in normalized_groups { for (normalized_fields, group) in normalized_groups {
if group.len() >= 2 { if group.len() >= 2 {
let generic_name = group[0].1.clone(); let generic_name = group[0].1.clone();
for (concrete_fields, _) in &group { for (concrete_fields, _, extracted_type) in &group {
mappings.insert(concrete_fields.clone(), generic_name.clone()); pattern_mappings.insert(concrete_fields.clone(), generic_name.clone());
type_mappings.insert(concrete_fields.clone(), extracted_type.clone());
} }
patterns.push(StructuralPattern { patterns.push(StructuralPattern {
name: generic_name, name: generic_name,
@@ -101,11 +117,12 @@ fn detect_generic_patterns(
} }
} }
(patterns, mappings) (patterns, pattern_mappings, type_mappings)
} }
/// Normalize fields by replacing concrete value types with "T". /// Normalize fields by replacing concrete value types with "T".
fn normalize_fields_for_generic(fields: &[PatternField]) -> Option<Vec<PatternField>> { /// Returns (normalized_fields, extracted_type) where extracted_type is the concrete type replaced.
fn normalize_fields_for_generic(fields: &[PatternField]) -> Option<(Vec<PatternField>, String)> {
let leaf_types: Vec<&str> = fields let leaf_types: Vec<&str> = fields
.iter() .iter()
.filter(|f| f.is_leaf()) .filter(|f| f.is_leaf())
@@ -137,7 +154,7 @@ fn normalize_fields_for_generic(fields: &[PatternField]) -> Option<Vec<PatternFi
}) })
.collect(); .collect();
Some(normalized) Some((normalized, super::extract_inner_type(first_type)))
} }
/// Recursively resolve branch patterns bottom-up. /// Recursively resolve branch patterns bottom-up.
@@ -266,7 +283,7 @@ fn collect_pattern_instances(
return; return;
}; };
let fields = get_node_fields_for_analysis(children, pattern_lookup); let fields = get_node_fields(children, pattern_lookup);
if let Some(pattern_name) = pattern_lookup.get(&fields) { if let Some(pattern_name) = pattern_lookup.get(&fields) {
for (field_name, child_node) in children { for (field_name, child_node) in children {
if let TreeNode::Leaf(leaf) = child_node { if let TreeNode::Leaf(leaf) = child_node {
@@ -283,7 +300,7 @@ fn collect_pattern_instances(
let child_accumulated = match child_node { let child_accumulated = match child_node {
TreeNode::Leaf(leaf) => leaf.name().to_string(), TreeNode::Leaf(leaf) => leaf.name().to_string(),
TreeNode::Branch(_) => { TreeNode::Branch(_) => {
if let Some(desc_leaf_name) = get_descendant_leaf_name(child_node) { if let Some(desc_leaf_name) = get_first_leaf_name(child_node) {
infer_accumulated_name(accumulated_name, field_name, &desc_leaf_name) infer_accumulated_name(accumulated_name, field_name, &desc_leaf_name)
} else if accumulated_name.is_empty() { } else if accumulated_name.is_empty() {
field_name.clone() field_name.clone()
@@ -296,13 +313,6 @@ fn collect_pattern_instances(
} }
} }
fn get_descendant_leaf_name(node: &TreeNode) -> Option<String> {
match node {
TreeNode::Leaf(leaf) => Some(leaf.name().to_string()),
TreeNode::Branch(children) => children.values().find_map(get_descendant_leaf_name),
}
}
fn infer_accumulated_name(parent_acc: &str, field_name: &str, descendant_leaf: &str) -> String { fn infer_accumulated_name(parent_acc: &str, field_name: &str, descendant_leaf: &str) -> String {
if let Some(pos) = descendant_leaf.find(field_name) { if let Some(pos) = descendant_leaf.find(field_name) {
if pos == 0 { if pos == 0 {
@@ -324,40 +334,6 @@ fn infer_accumulated_name(parent_acc: &str, field_name: &str, descendant_leaf: &
} }
} }
fn get_node_fields_for_analysis(
children: &BTreeMap<String, TreeNode>,
pattern_lookup: &HashMap<Vec<PatternField>, String>,
) -> Vec<PatternField> {
let mut fields: Vec<PatternField> = children
.iter()
.map(|(name, node)| {
let (rust_type, json_type, indexes) = match node {
TreeNode::Leaf(leaf) => (
leaf.value_type().to_string(),
schema_to_json_type(&leaf.schema),
leaf.indexes().clone(),
),
TreeNode::Branch(grandchildren) => {
let child_fields = get_node_fields_for_analysis(grandchildren, pattern_lookup);
let pattern_name = pattern_lookup
.get(&child_fields)
.cloned()
.unwrap_or_else(|| "Unknown".to_string());
(pattern_name.clone(), pattern_name, BTreeSet::new())
}
};
PatternField {
name: name.clone(),
rust_type,
json_type,
indexes,
}
})
.collect();
fields.sort_by(|a, b| a.name.cmp(&b.name));
fields
}
fn analyze_field_positions_from_instances( fn analyze_field_positions_from_instances(
instances: &[(String, String, String)], instances: &[(String, String, String)],
) -> HashMap<String, FieldNamePosition> { ) -> HashMap<String, FieldNamePosition> {

View File

@@ -59,42 +59,6 @@ pub fn get_node_fields(
fields fields
} }
/// Like get_node_fields but takes a parent name for generating child pattern names.
pub fn get_node_fields_with_parent(
children: &BTreeMap<String, TreeNode>,
parent_name: &str,
pattern_lookup: &HashMap<Vec<PatternField>, String>,
) -> Vec<PatternField> {
let mut fields: Vec<PatternField> = children
.iter()
.map(|(name, node)| {
let (rust_type, json_type, indexes) = match node {
TreeNode::Leaf(leaf) => (
leaf.value_type().to_string(),
schema_to_json_type(&leaf.schema),
leaf.indexes().clone(),
),
TreeNode::Branch(grandchildren) => {
let child_fields = get_node_fields(grandchildren, pattern_lookup);
let pattern_name = pattern_lookup
.get(&child_fields)
.cloned()
.unwrap_or_else(|| format!("{}_{}", parent_name, to_pascal_case(name)));
(pattern_name.clone(), pattern_name, BTreeSet::new())
}
};
PatternField {
name: name.clone(),
rust_type,
json_type,
indexes,
}
})
.collect();
fields.sort_by(|a, b| a.name.cmp(&b.name));
fields
}
/// Get fields with child field information for generic pattern lookup. /// Get fields with child field information for generic pattern lookup.
/// Returns (field, child_fields) pairs where child_fields is Some for branches. /// Returns (field, child_fields) pairs where child_fields is Some for branches.
pub fn get_fields_with_child_info( pub fn get_fields_with_child_info(

View File

@@ -35,7 +35,7 @@ fn run() -> Result<()> {
let computer = Computer::forced_import(&outputs_dir, &indexer, Some(fetcher))?; let computer = Computer::forced_import(&outputs_dir, &indexer, Some(fetcher))?;
let _a = dbg!(computer.chain.txinindex_to_value.region().meta()); let _a = dbg!(computer.chain.txinindex_to_value.region().meta());
let _b = dbg!(indexer.vecs.txout.txoutindex_to_value.region().meta()); let _b = dbg!(indexer.vecs.txout.txoutindex_to_txoutdata.region().meta());
Ok(()) Ok(())
} }

View File

@@ -4,7 +4,7 @@ use brk_types::{
CheckedSub, FeeRate, HalvingEpoch, Height, ONE_DAY_IN_SEC_F64, Sats, StoredF32, StoredF64, CheckedSub, FeeRate, HalvingEpoch, Height, ONE_DAY_IN_SEC_F64, Sats, StoredF32, StoredF64,
StoredU32, StoredU64, Timestamp, TxOutIndex, TxVersion, StoredU32, StoredU64, Timestamp, TxOutIndex, TxVersion,
}; };
use vecdb::{Exit, GenericStoredVec, IterableVec, TypedVecIterator, VecIndex, unlikely}; use vecdb::{Exit, IterableVec, TypedVecIterator, VecIndex, unlikely};
use crate::{grouped::ComputedVecsFromHeight, indexes, price, utils::OptionExt, Indexes}; use crate::{grouped::ComputedVecsFromHeight, indexes, price, utils::OptionExt, Indexes};
@@ -275,39 +275,11 @@ impl Vecs {
// TxInIndex // TxInIndex
// --- // ---
let txindex_to_first_txoutindex = &indexer.vecs.tx.txindex_to_first_txoutindex;
let txindex_to_first_txoutindex_reader = txindex_to_first_txoutindex.create_reader();
let txoutindex_to_value = &indexer.vecs.txout.txoutindex_to_value;
let txoutindex_to_value_reader = indexer.vecs.txout.txoutindex_to_value.create_reader();
self.txinindex_to_value.compute_transform(
starting_indexes.txinindex,
&indexer.vecs.txin.txinindex_to_outpoint,
|(txinindex, outpoint, ..)| {
if unlikely(outpoint.is_coinbase()) {
return (txinindex, Sats::MAX);
}
let txoutindex = txindex_to_first_txoutindex
.read_unwrap(outpoint.txindex(), &txindex_to_first_txoutindex_reader)
+ outpoint.vout();
let value = if unlikely(txoutindex == TxOutIndex::COINBASE) {
unreachable!()
} else {
txoutindex_to_value
.unchecked_read(txoutindex, &txoutindex_to_value_reader)
.unwrap()
};
(txinindex, value)
},
exit,
)?;
self.txindex_to_input_value.compute_sum_from_indexes( self.txindex_to_input_value.compute_sum_from_indexes(
starting_indexes.txindex, starting_indexes.txindex,
&indexer.vecs.tx.txindex_to_first_txinindex, &indexer.vecs.tx.txindex_to_first_txinindex,
&indexes.txindex_to_input_count, &indexes.txindex_to_input_count,
&self.txinindex_to_value, &indexer.vecs.txin.txinindex_to_value,
exit, exit,
)?; )?;
@@ -393,7 +365,8 @@ impl Vecs {
let mut txindex_to_first_txoutindex_iter = let mut txindex_to_first_txoutindex_iter =
indexer.vecs.tx.txindex_to_first_txoutindex.iter()?; indexer.vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut txindex_to_output_count_iter = indexes.txindex_to_output_count.iter(); let mut txindex_to_output_count_iter = indexes.txindex_to_output_count.iter();
let mut txoutindex_to_value_iter = indexer.vecs.txout.txoutindex_to_value.iter()?; let mut txoutindex_to_txoutdata_iter =
indexer.vecs.txout.txoutindex_to_txoutdata.iter()?;
vec.compute_transform( vec.compute_transform(
starting_indexes.height, starting_indexes.height,
&indexer.vecs.tx.height_to_first_txindex, &indexer.vecs.tx.height_to_first_txindex,
@@ -405,8 +378,9 @@ impl Vecs {
let mut sats = Sats::ZERO; let mut sats = Sats::ZERO;
(first_txoutindex..first_txoutindex + usize::from(output_count)).for_each( (first_txoutindex..first_txoutindex + usize::from(output_count)).for_each(
|txoutindex| { |txoutindex| {
sats += txoutindex_to_value_iter sats += txoutindex_to_txoutdata_iter
.get_unwrap(TxOutIndex::from(txoutindex)); .get_unwrap(TxOutIndex::from(txoutindex))
.value;
}, },
); );
(height, sats) (height, sats)

View File

@@ -126,9 +126,8 @@ impl Vecs {
let mut txindex_to_first_txoutindex_iter = let mut txindex_to_first_txoutindex_iter =
indexer.vecs.tx.txindex_to_first_txoutindex.iter()?; indexer.vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut txindex_to_output_count_iter = indexes.txindex_to_output_count.iter(); let mut txindex_to_output_count_iter = indexes.txindex_to_output_count.iter();
let mut txoutindex_to_outputtype_iter = let mut txoutindex_to_txoutdata_iter =
indexer.vecs.txout.txoutindex_to_outputtype.iter()?; indexer.vecs.txout.txoutindex_to_txoutdata.iter()?;
let mut txoutindex_to_typeindex_iter = indexer.vecs.txout.txoutindex_to_typeindex.iter()?;
let mut p2pk65addressindex_to_p2pk65bytes_iter = indexer let mut p2pk65addressindex_to_p2pk65bytes_iter = indexer
.vecs .vecs
.address .address
@@ -181,8 +180,9 @@ impl Vecs {
let pool = (*txoutindex..(*txoutindex + *outputcount)) let pool = (*txoutindex..(*txoutindex + *outputcount))
.map(TxOutIndex::from) .map(TxOutIndex::from)
.find_map(|txoutindex| { .find_map(|txoutindex| {
let outputtype = txoutindex_to_outputtype_iter.get_unwrap(txoutindex); let txoutdata = txoutindex_to_txoutdata_iter.get_unwrap(txoutindex);
let typeindex = txoutindex_to_typeindex_iter.get_unwrap(txoutindex); let outputtype = txoutdata.outputtype;
let typeindex = txoutdata.typeindex;
match outputtype { match outputtype {
OutputType::P2PK65 => Some(AddressBytes::from( OutputType::P2PK65 => Some(AddressBytes::from(

View File

@@ -24,8 +24,8 @@ use crate::{
address::AddressTypeToAddressCount, address::AddressTypeToAddressCount,
compute::write::{process_address_updates, write}, compute::write::{process_address_updates, write},
process::{ process::{
AddressCache, InputsResult, build_txoutindex_to_height_map, process_inputs, AddressCache, InputsResult, process_inputs, process_outputs, process_received,
process_outputs, process_received, process_sent, process_sent,
}, },
states::{BlockState, Transacted}, states::{BlockState, Transacted},
}, },
@@ -38,8 +38,8 @@ use super::{
vecs::Vecs, vecs::Vecs,
}, },
BIP30_DUPLICATE_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_2, BIP30_ORIGINAL_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_1, BIP30_DUPLICATE_HEIGHT_2, BIP30_ORIGINAL_HEIGHT_1,
BIP30_ORIGINAL_HEIGHT_2, ComputeContext, FLUSH_INTERVAL, IndexerReaders, TxInIterators, BIP30_ORIGINAL_HEIGHT_2, ComputeContext, FLUSH_INTERVAL, TxInIterators, TxOutIterators,
TxOutIterators, VecsReaders, build_txinindex_to_txindex, build_txoutindex_to_txindex, VecsReaders, build_txinindex_to_txindex, build_txoutindex_to_txindex,
}; };
/// Process all blocks from starting_height to last_height. /// Process all blocks from starting_height to last_height.
@@ -124,15 +124,8 @@ pub fn process_blocks(
let mut height_to_price_iter = height_to_price.map(|v| v.into_iter()); let mut height_to_price_iter = height_to_price.map(|v| v.into_iter());
let mut dateindex_to_price_iter = dateindex_to_price.map(|v| v.into_iter()); let mut dateindex_to_price_iter = dateindex_to_price.map(|v| v.into_iter());
info!("Building txoutindex_to_height map...");
// Build txoutindex -> height map for input processing
let txoutindex_to_height = build_txoutindex_to_height_map(height_to_first_txoutindex);
info!("Creating readers..."); info!("Creating readers...");
// Create readers for parallel data access
let ir = IndexerReaders::new(indexer);
let mut vr = VecsReaders::new(&vecs.any_address_indexes, &vecs.addresses_data); let mut vr = VecsReaders::new(&vecs.any_address_indexes, &vecs.addresses_data);
// Create reusable iterators for sequential txout/txin reads (16KB buffered) // Create reusable iterators for sequential txout/txin reads (16KB buffered)
@@ -273,14 +266,14 @@ pub fn process_blocks(
// Collect output/input data using reusable iterators (16KB buffered reads) // Collect output/input data using reusable iterators (16KB buffered reads)
// Must be done before thread::scope since iterators aren't Send // Must be done before thread::scope since iterators aren't Send
let (output_values, output_types, output_typeindexes) = let txoutdata_vec = txout_iters.collect_block_outputs(first_txoutindex, output_count);
txout_iters.collect_block_outputs(first_txoutindex, output_count);
let input_outpoints = if input_count > 1 { let (input_values, input_prev_heights, input_outputtypes, input_typeindexes) =
txin_iters.collect_block_outpoints(first_txinindex + 1, input_count - 1) if input_count > 1 {
} else { txin_iters.collect_block_inputs(first_txinindex + 1, input_count - 1)
Vec::new() } else {
}; (Vec::new(), Vec::new(), Vec::new(), Vec::new())
};
// Process outputs and inputs in parallel with tick-tock // Process outputs and inputs in parallel with tick-tock
let (outputs_result, inputs_result) = thread::scope(|scope| { let (outputs_result, inputs_result) = thread::scope(|scope| {
@@ -293,11 +286,8 @@ pub fn process_blocks(
let outputs_handle = scope.spawn(|| { let outputs_handle = scope.spawn(|| {
// Process outputs (receive) // Process outputs (receive)
process_outputs( process_outputs(
output_count,
&txoutindex_to_txindex, &txoutindex_to_txindex,
&output_values, &txoutdata_vec,
&output_types,
&output_typeindexes,
&first_addressindexes, &first_addressindexes,
&cache, &cache,
&vr, &vr,
@@ -309,16 +299,12 @@ pub fn process_blocks(
// Process inputs (send) - skip coinbase input // Process inputs (send) - skip coinbase input
let inputs_result = if input_count > 1 { let inputs_result = if input_count > 1 {
process_inputs( process_inputs(
first_txinindex + 1, // Skip coinbase
input_count - 1, input_count - 1,
&txinindex_to_txindex[1..], // Skip coinbase &txinindex_to_txindex[1..], // Skip coinbase
&input_outpoints, &input_values,
&indexer.vecs.tx.txindex_to_first_txoutindex, &input_outputtypes,
&indexer.vecs.txout.txoutindex_to_value, &input_typeindexes,
&indexer.vecs.txout.txoutindex_to_outputtype, &input_prev_heights,
&indexer.vecs.txout.txoutindex_to_typeindex,
&txoutindex_to_height,
&ir,
&first_addressindexes, &first_addressindexes,
&cache, &cache,
&vr, &vr,
@@ -331,7 +317,6 @@ pub fn process_blocks(
sent_data: Default::default(), sent_data: Default::default(),
address_data: Default::default(), address_data: Default::default(),
txindex_vecs: Default::default(), txindex_vecs: Default::default(),
txoutindex_to_txinindex_updates: Default::default(),
} }
}; };
@@ -426,12 +411,6 @@ pub fn process_blocks(
vecs.utxo_cohorts.send(height_to_sent, chain_state); vecs.utxo_cohorts.send(height_to_sent, chain_state);
}); });
// Update txoutindex_to_txinindex
vecs.update_txoutindex_to_txinindex(
output_count,
inputs_result.txoutindex_to_txinindex_updates,
)?;
// Push to height-indexed vectors // Push to height-indexed vectors
vecs.height_to_unspendable_supply vecs.height_to_unspendable_supply
.truncate_push(height, unspendable_supply)?; .truncate_push(height, unspendable_supply)?;

View File

@@ -17,7 +17,7 @@ mod write;
pub use block_loop::process_blocks; pub use block_loop::process_blocks;
pub use context::ComputeContext; pub use context::ComputeContext;
pub use readers::{ pub use readers::{
IndexerReaders, TxInIterators, TxOutIterators, VecsReaders, build_txinindex_to_txindex, TxInIterators, TxOutIterators, VecsReaders, build_txinindex_to_txindex,
build_txoutindex_to_txindex, build_txoutindex_to_txindex,
}; };
pub use recover::{StartMode, determine_start_mode, recover_state, reset_state}; pub use recover::{StartMode, determine_start_mode, recover_state, reset_state};

View File

@@ -4,7 +4,9 @@
use brk_grouper::{ByAddressType, ByAnyAddress}; use brk_grouper::{ByAddressType, ByAnyAddress};
use brk_indexer::Indexer; use brk_indexer::Indexer;
use brk_types::{OutPoint, OutputType, Sats, StoredU64, TxInIndex, TxIndex, TxOutIndex, TypeIndex}; use brk_types::{
Height, OutputType, Sats, StoredU64, TxInIndex, TxIndex, TxOutData, TxOutIndex, TypeIndex,
};
use vecdb::{ use vecdb::{
BoxedVecIterator, BytesVecIterator, GenericStoredVec, PcodecVecIterator, Reader, VecIndex, BoxedVecIterator, BytesVecIterator, GenericStoredVec, PcodecVecIterator, Reader, VecIndex,
VecIterator, VecIterator,
@@ -12,45 +14,18 @@ use vecdb::{
use crate::stateful::address::{AddressesDataVecs, AnyAddressIndexesVecs}; use crate::stateful::address::{AddressesDataVecs, AnyAddressIndexesVecs};
/// Cached readers for indexer vectors.
pub struct IndexerReaders {
pub txindex_to_first_txoutindex: Reader,
pub txoutindex_to_value: Reader,
pub txoutindex_to_outputtype: Reader,
pub txoutindex_to_typeindex: Reader,
}
impl IndexerReaders {
pub fn new(indexer: &Indexer) -> Self {
Self {
txindex_to_first_txoutindex: indexer
.vecs
.tx
.txindex_to_first_txoutindex
.create_reader(),
txoutindex_to_value: indexer.vecs.txout.txoutindex_to_value.create_reader(),
txoutindex_to_outputtype: indexer.vecs.txout.txoutindex_to_outputtype.create_reader(),
txoutindex_to_typeindex: indexer.vecs.txout.txoutindex_to_typeindex.create_reader(),
}
}
}
/// Reusable iterators for txout vectors (16KB buffered reads). /// Reusable iterators for txout vectors (16KB buffered reads).
/// ///
/// Iterators are created once and re-positioned each block to avoid /// Iterators are created once and re-positioned each block to avoid
/// creating new file handles repeatedly. /// creating new file handles repeatedly.
pub struct TxOutIterators<'a> { pub struct TxOutIterators<'a> {
value_iter: BytesVecIterator<'a, TxOutIndex, Sats>, txoutdata_iter: BytesVecIterator<'a, TxOutIndex, TxOutData>,
outputtype_iter: BytesVecIterator<'a, TxOutIndex, OutputType>,
typeindex_iter: BytesVecIterator<'a, TxOutIndex, TypeIndex>,
} }
impl<'a> TxOutIterators<'a> { impl<'a> TxOutIterators<'a> {
pub fn new(indexer: &'a Indexer) -> Self { pub fn new(indexer: &'a Indexer) -> Self {
Self { Self {
value_iter: indexer.vecs.txout.txoutindex_to_value.into_iter(), txoutdata_iter: indexer.vecs.txout.txoutindex_to_txoutdata.into_iter(),
outputtype_iter: indexer.vecs.txout.txoutindex_to_outputtype.into_iter(),
typeindex_iter: indexer.vecs.txout.txoutindex_to_typeindex.into_iter(),
} }
} }
@@ -59,43 +34,50 @@ impl<'a> TxOutIterators<'a> {
&mut self, &mut self,
first_txoutindex: usize, first_txoutindex: usize,
output_count: usize, output_count: usize,
) -> (Vec<Sats>, Vec<OutputType>, Vec<TypeIndex>) { ) -> Vec<TxOutData> {
let mut values = Vec::with_capacity(output_count); (first_txoutindex..first_txoutindex + output_count)
let mut output_types = Vec::with_capacity(output_count); .map(|i| self.txoutdata_iter.get_at_unwrap(i))
let mut type_indexes = Vec::with_capacity(output_count); .collect()
for i in first_txoutindex..first_txoutindex + output_count {
values.push(self.value_iter.get_at_unwrap(i));
output_types.push(self.outputtype_iter.get_at_unwrap(i));
type_indexes.push(self.typeindex_iter.get_at_unwrap(i));
}
(values, output_types, type_indexes)
} }
} }
/// Reusable iterator for txin outpoints (PcoVec - avoids repeated page decompression). /// Reusable iterators for txin vectors (PcoVec - avoids repeated page decompression).
pub struct TxInIterators<'a> { pub struct TxInIterators<'a> {
outpoint_iter: PcodecVecIterator<'a, TxInIndex, OutPoint>, value_iter: PcodecVecIterator<'a, TxInIndex, Sats>,
prev_height_iter: PcodecVecIterator<'a, TxInIndex, Height>,
outputtype_iter: PcodecVecIterator<'a, TxInIndex, OutputType>,
typeindex_iter: PcodecVecIterator<'a, TxInIndex, TypeIndex>,
} }
impl<'a> TxInIterators<'a> { impl<'a> TxInIterators<'a> {
pub fn new(indexer: &'a Indexer) -> Self { pub fn new(indexer: &'a Indexer) -> Self {
Self { Self {
outpoint_iter: indexer.vecs.txin.txinindex_to_outpoint.into_iter(), value_iter: indexer.vecs.txin.txinindex_to_value.into_iter(),
prev_height_iter: indexer.vecs.txin.txinindex_to_prev_height.into_iter(),
outputtype_iter: indexer.vecs.txin.txinindex_to_outputtype.into_iter(),
typeindex_iter: indexer.vecs.txin.txinindex_to_typeindex.into_iter(),
} }
} }
/// Collect outpoints for a block range using buffered iteration. /// Collect input data for a block range using buffered iteration.
/// This avoids repeated PcoVec page decompression (~1000x speedup). pub fn collect_block_inputs(
pub fn collect_block_outpoints(
&mut self, &mut self,
first_txinindex: usize, first_txinindex: usize,
input_count: usize, input_count: usize,
) -> Vec<OutPoint> { ) -> (Vec<Sats>, Vec<Height>, Vec<OutputType>, Vec<TypeIndex>) {
(first_txinindex..first_txinindex + input_count) let mut values = Vec::with_capacity(input_count);
.map(|i| self.outpoint_iter.get_at_unwrap(i)) let mut prev_heights = Vec::with_capacity(input_count);
.collect() let mut outputtypes = Vec::with_capacity(input_count);
let mut typeindexes = Vec::with_capacity(input_count);
for i in first_txinindex..first_txinindex + input_count {
values.push(self.value_iter.get_at_unwrap(i));
prev_heights.push(self.prev_height_iter.get_at_unwrap(i));
outputtypes.push(self.outputtype_iter.get_at_unwrap(i));
typeindexes.push(self.typeindex_iter.get_at_unwrap(i));
}
(values, prev_heights, outputtypes, typeindexes)
} }
} }

View File

@@ -27,7 +27,6 @@ pub struct RecoveredState {
pub fn recover_state( pub fn recover_state(
height: Height, height: Height,
chain_state_rollback: vecdb::Result<Stamp>, chain_state_rollback: vecdb::Result<Stamp>,
txoutindex_rollback: vecdb::Result<Stamp>,
any_address_indexes: &mut AnyAddressIndexesVecs, any_address_indexes: &mut AnyAddressIndexesVecs,
addresses_data: &mut AddressesDataVecs, addresses_data: &mut AddressesDataVecs,
utxo_cohorts: &mut UTXOCohorts, utxo_cohorts: &mut UTXOCohorts,
@@ -42,7 +41,6 @@ pub fn recover_state(
// Verify rollback consistency - all must agree on the same height // Verify rollback consistency - all must agree on the same height
let consistent_height = rollback_states( let consistent_height = rollback_states(
chain_state_rollback, chain_state_rollback,
txoutindex_rollback,
address_indexes_rollback, address_indexes_rollback,
address_data_rollback, address_data_rollback,
); );
@@ -127,7 +125,6 @@ pub enum StartMode {
/// otherwise returns Height::ZERO (need fresh start). /// otherwise returns Height::ZERO (need fresh start).
fn rollback_states( fn rollback_states(
chain_state_rollback: vecdb::Result<Stamp>, chain_state_rollback: vecdb::Result<Stamp>,
txoutindex_rollback: vecdb::Result<Stamp>,
address_indexes_rollbacks: Result<Vec<Stamp>>, address_indexes_rollbacks: Result<Vec<Stamp>>,
address_data_rollbacks: Result<[Stamp; 2]>, address_data_rollbacks: Result<[Stamp; 2]>,
) -> Height { ) -> Height {
@@ -139,11 +136,6 @@ fn rollback_states(
}; };
heights.insert(Height::from(s).incremented()); heights.insert(Height::from(s).incremented());
let Ok(s) = txoutindex_rollback else {
return Height::ZERO;
};
heights.insert(Height::from(s).incremented());
let Ok(stamps) = address_indexes_rollbacks else { let Ok(stamps) = address_indexes_rollbacks else {
return Height::ZERO; return Height::ZERO;
}; };

View File

@@ -89,9 +89,6 @@ pub fn write(
vecs.addresstype_to_height_to_empty_addr_count vecs.addresstype_to_height_to_empty_addr_count
.par_iter_mut(), .par_iter_mut(),
) )
.chain(rayon::iter::once(
&mut vecs.txoutindex_to_txinindex as &mut dyn AnyStoredVec,
))
.chain(rayon::iter::once( .chain(rayon::iter::once(
&mut vecs.chain_state as &mut dyn AnyStoredVec, &mut vecs.chain_state as &mut dyn AnyStoredVec,
)) ))

View File

@@ -39,8 +39,3 @@ pub use address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexe
// Cohort re-exports // Cohort re-exports
pub use cohorts::{AddressCohorts, CohortVecs, DynCohortVecs, UTXOCohorts}; pub use cohorts::{AddressCohorts, CohortVecs, DynCohortVecs, UTXOCohorts};
// Compute re-exports
pub use compute::IndexerReaders;
// Metrics re-exports

View File

@@ -1,24 +1,20 @@
//! Parallel input processing. //! Parallel input processing.
//!
//! Processes a block's inputs (spent UTXOs) in parallel, building:
//! - height_to_sent: map from creation height -> Transacted for sends
//! - Address data for address cohort tracking (optional)
use brk_grouper::ByAddressType; use brk_grouper::ByAddressType;
use brk_types::{Height, OutPoint, OutputType, Sats, TxInIndex, TxIndex, TxOutIndex, TypeIndex}; use brk_types::{Height, OutputType, Sats, TxIndex, TypeIndex};
use rayon::prelude::*; use rayon::prelude::*;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use vecdb::{BytesVec, GenericStoredVec};
use crate::stateful::address::{ use crate::stateful::{
AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs, address::{AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs},
compute::VecsReaders,
states::Transacted,
}; };
use crate::stateful::compute::VecsReaders;
use crate::stateful::states::Transacted;
use crate::stateful::{IndexerReaders, process::RangeMap};
use super::super::address::HeightToAddressTypeToVec; use super::{
use super::{load_uncached_address_data, AddressCache, LoadedAddressDataWithSource, TxIndexVec}; super::address::HeightToAddressTypeToVec, AddressCache, LoadedAddressDataWithSource,
TxIndexVec, load_uncached_address_data,
};
/// Result of processing inputs for a block. /// Result of processing inputs for a block.
pub struct InputsResult { pub struct InputsResult {
@@ -30,8 +26,6 @@ pub struct InputsResult {
pub address_data: AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>, pub address_data: AddressTypeToTypeIndexMap<LoadedAddressDataWithSource>,
/// Transaction indexes per address for tx_count tracking. /// Transaction indexes per address for tx_count tracking.
pub txindex_vecs: AddressTypeToTypeIndexMap<TxIndexVec>, pub txindex_vecs: AddressTypeToTypeIndexMap<TxIndexVec>,
/// Updates to txoutindex_to_txinindex: (spent txoutindex, spending txinindex).
pub txoutindex_to_txinindex_updates: Vec<(TxOutIndex, TxInIndex)>,
} }
/// Process inputs (spent UTXOs) for a block. /// Process inputs (spent UTXOs) for a block.
@@ -49,52 +43,32 @@ pub struct InputsResult {
/// expensive merge overhead from rayon's fold/reduce pattern. /// expensive merge overhead from rayon's fold/reduce pattern.
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn process_inputs( pub fn process_inputs(
first_txinindex: usize,
input_count: usize, input_count: usize,
txinindex_to_txindex: &[TxIndex], txinindex_to_txindex: &[TxIndex],
// Pre-collected outpoints (from reusable iterator with page caching) txinindex_to_value: &[Sats],
outpoints: &[OutPoint], txinindex_to_outputtype: &[OutputType],
txindex_to_first_txoutindex: &BytesVec<TxIndex, TxOutIndex>, txinindex_to_typeindex: &[TypeIndex],
txoutindex_to_value: &BytesVec<TxOutIndex, Sats>, txinindex_to_prev_height: &[Height],
txoutindex_to_outputtype: &BytesVec<TxOutIndex, OutputType>,
txoutindex_to_typeindex: &BytesVec<TxOutIndex, TypeIndex>,
txoutindex_to_height: &RangeMap<TxOutIndex, Height>,
ir: &IndexerReaders,
// Address lookup parameters
first_addressindexes: &ByAddressType<TypeIndex>, first_addressindexes: &ByAddressType<TypeIndex>,
cache: &AddressCache, cache: &AddressCache,
vr: &VecsReaders, vr: &VecsReaders,
any_address_indexes: &AnyAddressIndexesVecs, any_address_indexes: &AnyAddressIndexesVecs,
addresses_data: &AddressesDataVecs, addresses_data: &AddressesDataVecs,
) -> InputsResult { ) -> InputsResult {
// Parallel reads - collect all input data (outpoints already in memory)
let items: Vec<_> = (0..input_count) let items: Vec<_> = (0..input_count)
.into_par_iter() .into_par_iter()
.map(|local_idx| { .map(|local_idx| {
let txinindex = TxInIndex::from(first_txinindex + local_idx);
let txindex = txinindex_to_txindex[local_idx]; let txindex = txinindex_to_txindex[local_idx];
// Get outpoint from pre-collected vec and resolve to txoutindex let prev_height = *txinindex_to_prev_height.get(local_idx).unwrap();
let outpoint = outpoints[local_idx]; let value = *txinindex_to_value.get(local_idx).unwrap();
let first_txoutindex = txindex_to_first_txoutindex let input_type = *txinindex_to_outputtype.get(local_idx).unwrap();
.read_unwrap(outpoint.txindex(), &ir.txindex_to_first_txoutindex);
let txoutindex = first_txoutindex + outpoint.vout();
// Get creation height
let prev_height = *txoutindex_to_height.get(txoutindex).unwrap();
// Get value and type from the output being spent
let value = txoutindex_to_value.read_unwrap(txoutindex, &ir.txoutindex_to_value);
let input_type =
txoutindex_to_outputtype.read_unwrap(txoutindex, &ir.txoutindex_to_outputtype);
// Non-address inputs don't need typeindex or address lookup
if input_type.is_not_address() { if input_type.is_not_address() {
return (txinindex, txoutindex, prev_height, value, input_type, None); return (prev_height, value, input_type, None);
} }
let typeindex = let typeindex = *txinindex_to_typeindex.get(local_idx).unwrap();
txoutindex_to_typeindex.read_unwrap(txoutindex, &ir.txoutindex_to_typeindex);
// Look up address data // Look up address data
let addr_data_opt = load_uncached_address_data( let addr_data_opt = load_uncached_address_data(
@@ -108,8 +82,6 @@ pub fn process_inputs(
); );
( (
txinindex,
txoutindex,
prev_height, prev_height,
value, value,
input_type, input_type,
@@ -131,16 +103,13 @@ pub fn process_inputs(
AddressTypeToTypeIndexMap::<LoadedAddressDataWithSource>::with_capacity(estimated_per_type); AddressTypeToTypeIndexMap::<LoadedAddressDataWithSource>::with_capacity(estimated_per_type);
let mut txindex_vecs = let mut txindex_vecs =
AddressTypeToTypeIndexMap::<TxIndexVec>::with_capacity(estimated_per_type); AddressTypeToTypeIndexMap::<TxIndexVec>::with_capacity(estimated_per_type);
let mut txoutindex_to_txinindex_updates = Vec::with_capacity(input_count);
for (txinindex, txoutindex, prev_height, value, output_type, addr_info) in items { for (prev_height, value, output_type, addr_info) in items {
height_to_sent height_to_sent
.entry(prev_height) .entry(prev_height)
.or_default() .or_default()
.iterate(value, output_type); .iterate(value, output_type);
txoutindex_to_txinindex_updates.push((txoutindex, txinindex));
if let Some((typeindex, txindex, value, addr_data_opt)) = addr_info { if let Some((typeindex, txindex, value, addr_data_opt)) = addr_info {
sent_data sent_data
.entry(prev_height) .entry(prev_height)
@@ -167,7 +136,5 @@ pub fn process_inputs(
sent_data, sent_data,
address_data, address_data,
txindex_vecs, txindex_vecs,
txoutindex_to_txinindex_updates,
} }
} }

View File

@@ -3,7 +3,6 @@ mod cache;
mod inputs; mod inputs;
mod lookup; mod lookup;
mod outputs; mod outputs;
mod range_map;
mod received; mod received;
mod sent; mod sent;
mod tx_counts; mod tx_counts;
@@ -14,7 +13,6 @@ pub use cache::*;
pub use inputs::*; pub use inputs::*;
pub use lookup::*; pub use lookup::*;
pub use outputs::*; pub use outputs::*;
pub use range_map::*;
pub use received::*; pub use received::*;
pub use sent::*; pub use sent::*;
pub use tx_counts::*; pub use tx_counts::*;

View File

@@ -5,7 +5,7 @@
//! - Address data for address cohort tracking (optional) //! - Address data for address cohort tracking (optional)
use brk_grouper::ByAddressType; use brk_grouper::ByAddressType;
use brk_types::{OutputType, Sats, TxIndex, TypeIndex}; use brk_types::{Sats, TxIndex, TxOutData, TypeIndex};
use crate::stateful::address::{ use crate::stateful::address::{
AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs, AddressTypeToTypeIndexMap, AddressesDataVecs, AnyAddressIndexesVecs,
@@ -37,19 +37,16 @@ pub struct OutputsResult {
/// 4. Track address-specific data for address cohort processing /// 4. Track address-specific data for address cohort processing
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn process_outputs( pub fn process_outputs(
output_count: usize,
txoutindex_to_txindex: &[TxIndex], txoutindex_to_txindex: &[TxIndex],
// Pre-collected output data (from reusable iterators with 16KB buffered reads) txoutdata_vec: &[TxOutData],
values: &[Sats],
output_types: &[OutputType],
typeindexes: &[TypeIndex],
// Address lookup parameters
first_addressindexes: &ByAddressType<TypeIndex>, first_addressindexes: &ByAddressType<TypeIndex>,
cache: &AddressCache, cache: &AddressCache,
vr: &VecsReaders, vr: &VecsReaders,
any_address_indexes: &AnyAddressIndexesVecs, any_address_indexes: &AnyAddressIndexesVecs,
addresses_data: &AddressesDataVecs, addresses_data: &AddressesDataVecs,
) -> OutputsResult { ) -> OutputsResult {
let output_count = txoutdata_vec.len();
// Pre-allocate result structures // Pre-allocate result structures
let estimated_per_type = (output_count / 8).max(8); let estimated_per_type = (output_count / 8).max(8);
let mut transacted = Transacted::default(); let mut transacted = Transacted::default();
@@ -60,10 +57,10 @@ pub fn process_outputs(
AddressTypeToTypeIndexMap::<TxIndexVec>::with_capacity(estimated_per_type); AddressTypeToTypeIndexMap::<TxIndexVec>::with_capacity(estimated_per_type);
// Single pass: read from pre-collected vecs and accumulate // Single pass: read from pre-collected vecs and accumulate
for local_idx in 0..output_count { for (local_idx, txoutdata) in txoutdata_vec.iter().enumerate() {
let txindex = txoutindex_to_txindex[local_idx]; let txindex = txoutindex_to_txindex[local_idx];
let value = values[local_idx]; let value = txoutdata.value;
let output_type = output_types[local_idx]; let output_type = txoutdata.outputtype;
transacted.iterate(value, output_type); transacted.iterate(value, output_type);
@@ -71,7 +68,7 @@ pub fn process_outputs(
continue; continue;
} }
let typeindex = typeindexes[local_idx]; let typeindex = txoutdata.typeindex;
received_data received_data
.get_mut(output_type) .get_mut(output_type)

View File

@@ -1,65 +0,0 @@
//! Range-based lookup map.
//!
//! Maps ranges of indices to values for efficient reverse lookups.
use std::collections::BTreeMap;
use brk_types::{Height, TxOutIndex};
use vecdb::{BytesVec, BytesVecValue, PcoVec, PcoVecValue, VecIndex};
/// Maps ranges of indices to their corresponding height.
/// Used to efficiently look up which block a txoutindex belongs to.
#[derive(Debug)]
pub struct RangeMap<I, T>(BTreeMap<I, T>);
impl<I, T> RangeMap<I, T>
where
I: VecIndex,
T: VecIndex,
{
/// Look up value for a key using range search.
/// Returns the value associated with the largest key <= given key.
#[inline]
pub fn get(&self, key: I) -> Option<&T> {
self.0.range(..=key).next_back().map(|(_, value)| value)
}
}
impl<I, T> From<&BytesVec<I, T>> for RangeMap<T, I>
where
I: VecIndex,
T: VecIndex + BytesVecValue,
{
#[inline]
fn from(vec: &BytesVec<I, T>) -> Self {
Self(
vec.into_iter()
.enumerate()
.map(|(i, v)| (v, I::from(i)))
.collect(),
)
}
}
impl<I, T> From<&PcoVec<I, T>> for RangeMap<T, I>
where
I: VecIndex,
T: VecIndex + PcoVecValue,
{
#[inline]
fn from(vec: &PcoVec<I, T>) -> Self {
Self(
vec.into_iter()
.enumerate()
.map(|(i, v)| (v, I::from(i)))
.collect(),
)
}
}
/// Creates a RangeMap from height_to_first_txoutindex for fast txoutindex -> height lookups.
pub fn build_txoutindex_to_height_map(
height_to_first_txoutindex: &PcoVec<Height, TxOutIndex>,
) -> RangeMap<TxOutIndex, Height> {
RangeMap::from(height_to_first_txoutindex)
}

View File

@@ -7,11 +7,11 @@ use brk_indexer::Indexer;
use brk_traversable::Traversable; use brk_traversable::Traversable;
use brk_types::{ use brk_types::{
Dollars, EmptyAddressData, EmptyAddressIndex, Height, LoadedAddressData, LoadedAddressIndex, Dollars, EmptyAddressData, EmptyAddressIndex, Height, LoadedAddressData, LoadedAddressIndex,
Sats, StoredU64, TxInIndex, TxOutIndex, Version, Sats, StoredU64, Version,
}; };
use log::info; use log::info;
use vecdb::{ use vecdb::{
AnyStoredVec, AnyVec, BytesVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec, AnyVec, BytesVec, Database, EagerVec, Exit, GenericStoredVec, ImportableVec,
IterableCloneableVec, LazyVecFrom1, PAGE_SIZE, PcoVec, Stamp, TypedVecIterator, VecIndex, IterableCloneableVec, LazyVecFrom1, PAGE_SIZE, PcoVec, Stamp, TypedVecIterator, VecIndex,
}; };
@@ -47,7 +47,6 @@ pub struct Vecs {
// States // States
// --- // ---
pub chain_state: BytesVec<Height, SupplyState>, pub chain_state: BytesVec<Height, SupplyState>,
pub txoutindex_to_txinindex: BytesVec<TxOutIndex, TxInIndex>,
pub any_address_indexes: AnyAddressIndexesVecs, pub any_address_indexes: AnyAddressIndexesVecs,
pub addresses_data: AddressesDataVecs, pub addresses_data: AddressesDataVecs,
pub utxo_cohorts: UTXOCohorts, pub utxo_cohorts: UTXOCohorts,
@@ -126,10 +125,6 @@ impl Vecs {
vecdb::ImportOptions::new(&db, "chain", v0) vecdb::ImportOptions::new(&db, "chain", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES), .with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?, )?,
txoutindex_to_txinindex: BytesVec::forced_import_with(
vecdb::ImportOptions::new(&db, "txinindex", v0)
.with_saved_stamped_changes(SAVED_STAMPED_CHANGES),
)?,
height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?, height_to_unspendable_supply: EagerVec::forced_import(&db, "unspendable_supply", v0)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
@@ -265,12 +260,13 @@ impl Vecs {
let stateful_min = utxo_min let stateful_min = utxo_min
.min(address_min) .min(address_min)
.min(Height::from(self.chain_state.len())) .min(Height::from(self.chain_state.len()))
.min(Height::from(self.txoutindex_to_txinindex.stamp()).incremented())
.min(self.any_address_indexes.min_stamped_height()) .min(self.any_address_indexes.min_stamped_height())
.min(self.addresses_data.min_stamped_height()) .min(self.addresses_data.min_stamped_height())
.min(Height::from(self.height_to_unspendable_supply.len())) .min(Height::from(self.height_to_unspendable_supply.len()))
.min(Height::from(self.height_to_opreturn_supply.len())) .min(Height::from(self.height_to_opreturn_supply.len()))
.min(Height::from(self.addresstype_to_height_to_addr_count.min_len())) .min(Height::from(
self.addresstype_to_height_to_addr_count.min_len(),
))
.min(Height::from( .min(Height::from(
self.addresstype_to_height_to_empty_addr_count.min_len(), self.addresstype_to_height_to_empty_addr_count.min_len(),
)); ));
@@ -285,13 +281,11 @@ impl Vecs {
// Rollback BytesVec state and capture results for validation // Rollback BytesVec state and capture results for validation
let chain_state_rollback = self.chain_state.rollback_before(stamp); let chain_state_rollback = self.chain_state.rollback_before(stamp);
let txoutindex_rollback = self.txoutindex_to_txinindex.rollback_before(stamp);
// Validate all rollbacks and imports are consistent // Validate all rollbacks and imports are consistent
let recovered = recover_state( let recovered = recover_state(
height, height,
chain_state_rollback, chain_state_rollback,
txoutindex_rollback,
&mut self.any_address_indexes, &mut self.any_address_indexes,
&mut self.addresses_data, &mut self.addresses_data,
&mut self.utxo_cohorts, &mut self.utxo_cohorts,
@@ -309,7 +303,6 @@ impl Vecs {
// Fresh start: reset all state // Fresh start: reset all state
let (starting_height, mut chain_state) = if recovered_height.is_zero() { let (starting_height, mut chain_state) = if recovered_height.is_zero() {
self.chain_state.reset()?; self.chain_state.reset()?;
self.txoutindex_to_txinindex.reset()?;
self.height_to_unspendable_supply.reset()?; self.height_to_unspendable_supply.reset()?;
self.height_to_opreturn_supply.reset()?; self.height_to_opreturn_supply.reset()?;
self.addresstype_to_height_to_addr_count.reset()?; self.addresstype_to_height_to_addr_count.reset()?;
@@ -505,24 +498,4 @@ impl Vecs {
self.db.compact()?; self.db.compact()?;
Ok(()) Ok(())
} }
/// Update txoutindex_to_txinindex for a block.
///
/// 1. Push UNSPENT for all new outputs in the block
/// 2. Update spent outputs with their spending txinindex
pub fn update_txoutindex_to_txinindex(
&mut self,
output_count: usize,
updates: Vec<(TxOutIndex, TxInIndex)>,
) -> Result<()> {
// Push UNSPENT for all new outputs in this block
for _ in 0..output_count {
self.txoutindex_to_txinindex.push(TxInIndex::UNSPENT);
}
// Update spent outputs with their spending txinindex
for (txoutindex, txinindex) in updates {
self.txoutindex_to_txinindex.update(txoutindex, txinindex)?;
}
Ok(())
}
} }

View File

@@ -1,6 +1,6 @@
#![doc = include_str!("../README.md")] #![doc = include_str!("../README.md")]
use std::{io, result, time}; use std::{io, path::PathBuf, result, time};
use thiserror::Error; use thiserror::Error;
@@ -123,6 +123,13 @@ pub enum Error {
#[error("Fetch failed after retries: {0}")] #[error("Fetch failed after retries: {0}")]
FetchFailed(String), FetchFailed(String),
#[error("Version mismatch at {path:?}: expected {expected}, found {found}")]
VersionMismatch {
path: PathBuf,
expected: usize,
found: usize,
},
} }

View File

@@ -25,7 +25,6 @@ impl<T> GroupedByType<T> {
OutputType::Empty => &self.spendable.empty, OutputType::Empty => &self.spendable.empty,
OutputType::Unknown => &self.spendable.unknown, OutputType::Unknown => &self.spendable.unknown,
OutputType::OpReturn => &self.unspendable.opreturn, OutputType::OpReturn => &self.unspendable.opreturn,
_ => unreachable!(),
} }
} }
@@ -43,7 +42,6 @@ impl<T> GroupedByType<T> {
OutputType::Unknown => &mut self.spendable.unknown, OutputType::Unknown => &mut self.spendable.unknown,
OutputType::Empty => &mut self.spendable.empty, OutputType::Empty => &mut self.spendable.empty,
OutputType::OpReturn => &mut self.unspendable.opreturn, OutputType::OpReturn => &mut self.unspendable.opreturn,
_ => unreachable!(),
} }
} }
} }

View File

@@ -29,7 +29,7 @@ fn main() -> Result<()> {
indexer indexer
.vecs .vecs
.txout .txout
.txoutindex_to_value .txoutindex_to_txoutdata
.iter()? .iter()?
.enumerate() .enumerate()
.take(200) .take(200)

View File

@@ -13,9 +13,8 @@ fn run_benchmark(indexer: &Indexer) -> (Sats, std::time::Duration, usize) {
let mut sum = Sats::ZERO; let mut sum = Sats::ZERO;
let mut count = 0; let mut count = 0;
for value in indexer.vecs.txout.txoutindex_to_value.clean_iter().unwrap() { for txoutdata in indexer.vecs.txout.txoutindex_to_txoutdata.clean_iter().unwrap() {
// for value in indexer.vecs.txoutindex_to_value.values() { sum += txoutdata.value;
sum += value;
count += 1; count += 1;
} }

View File

@@ -4,7 +4,7 @@ use brk_types::{Height, TxIndex, Txid, TxidPrefix, Version};
// One version for all data sources // One version for all data sources
// Increment on **change _OR_ addition** // Increment on **change _OR_ addition**
pub const VERSION: Version = Version::new(23); pub const VERSION: Version = Version::new(24);
pub const SNAPSHOT_BLOCK_RANGE: usize = 1_000; pub const SNAPSHOT_BLOCK_RANGE: usize = 1_000;
pub const COLLISIONS_CHECKED_UP_TO: Height = Height::new(0); pub const COLLISIONS_CHECKED_UP_TO: Height = Height::new(0);

View File

@@ -45,7 +45,6 @@ impl Indexes {
OutputType::P2WPKH => *self.p2wpkhaddressindex, OutputType::P2WPKH => *self.p2wpkhaddressindex,
OutputType::P2WSH => *self.p2wshaddressindex, OutputType::P2WSH => *self.p2wshaddressindex,
OutputType::Unknown => *self.unknownoutputindex, OutputType::Unknown => *self.unknownoutputindex,
_ => unreachable!(),
} }
} }
@@ -225,7 +224,7 @@ impl From<(Height, &mut Vecs, &Stores)> for Indexes {
let txoutindex = starting_index( let txoutindex = starting_index(
&vecs.txout.height_to_first_txoutindex, &vecs.txout.height_to_first_txoutindex,
&vecs.txout.txoutindex_to_value, &vecs.txout.txoutindex_to_txoutdata,
height, height,
) )
.unwrap(); .unwrap();

View File

@@ -1,8 +1,8 @@
#![doc = include_str!("../README.md")] #![doc = include_str!("../README.md")]
use std::{path::Path, thread, time::Instant}; use std::{fs, path::Path, thread, time::Instant};
use brk_error::Result; use brk_error::{Error, Result};
use brk_iterator::Blocks; use brk_iterator::Blocks;
use brk_rpc::Client; use brk_rpc::Client;
use brk_types::Height; use brk_types::Height;
@@ -11,6 +11,7 @@ use vecdb::Exit;
mod constants; mod constants;
mod indexes; mod indexes;
mod processor; mod processor;
mod range_map;
mod readers; mod readers;
mod stores; mod stores;
mod vecs; mod vecs;
@@ -18,6 +19,7 @@ mod vecs;
use constants::*; use constants::*;
pub use indexes::*; pub use indexes::*;
pub use processor::*; pub use processor::*;
pub use range_map::*;
pub use readers::*; pub use readers::*;
pub use stores::*; pub use stores::*;
pub use vecs::*; pub use vecs::*;
@@ -30,6 +32,19 @@ pub struct Indexer {
impl Indexer { impl Indexer {
pub fn forced_import(outputs_dir: &Path) -> Result<Self> { pub fn forced_import(outputs_dir: &Path) -> Result<Self> {
match Self::forced_import_inner(outputs_dir) {
Ok(result) => Ok(result),
Err(Error::VersionMismatch { path, .. }) => {
let indexed_path = outputs_dir.join("indexed");
info!("Version mismatch at {path:?}, deleting {indexed_path:?} and retrying");
fs::remove_dir_all(&indexed_path)?;
Self::forced_import(outputs_dir)
}
Err(e) => Err(e),
}
}
fn forced_import_inner(outputs_dir: &Path) -> Result<Self> {
info!("Increasing number of open files limit..."); info!("Increasing number of open files limit...");
let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?; let no_file_limit = rlimit::getrlimit(rlimit::Resource::NOFILE)?;
rlimit::setrlimit( rlimit::setrlimit(
@@ -129,6 +144,13 @@ impl Indexer {
let mut readers = Readers::new(&self.vecs); let mut readers = Readers::new(&self.vecs);
// Build txindex -> height map from existing data for efficient lookups
let mut txindex_to_height = RangeMap::new();
for (height, first_txindex) in self.vecs.tx.height_to_first_txindex.into_iter().enumerate()
{
txindex_to_height.insert(first_txindex, Height::from(height));
}
let vecs = &mut self.vecs; let vecs = &mut self.vecs;
let stores = &mut self.stores; let stores = &mut self.stores;
@@ -139,6 +161,9 @@ impl Indexer {
indexes.height = height; indexes.height = height;
// Insert current block's first_txindex -> height before processing inputs
txindex_to_height.insert(indexes.txindex, height);
// Used to check rapidhash collisions // Used to check rapidhash collisions
let block_check_collisions = check_collisions && height > COLLISIONS_CHECKED_UP_TO; let block_check_collisions = check_collisions && height > COLLISIONS_CHECKED_UP_TO;
@@ -150,6 +175,7 @@ impl Indexer {
vecs, vecs,
stores, stores,
readers: &readers, readers: &readers,
txindex_to_height: &txindex_to_height,
}; };
// Phase 1: Process block metadata // Phase 1: Process block metadata

View File

@@ -1,717 +0,0 @@
use bitcoin::{Transaction, TxIn, TxOut};
use brk_error::{Error, Result};
use brk_grouper::ByAddressType;
use brk_types::{
AddressBytes, AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, Block, BlockHashPrefix,
Height, OutPoint, OutputType, Sats, StoredBool, Timestamp, TxInIndex, TxIndex, TxOutIndex,
Txid, TxidPrefix, TypeIndex, Unit, Vin, Vout,
};
use log::error;
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use vecdb::{AnyVec, GenericStoredVec, TypedVecIterator, likely};
use crate::{Indexes, Readers, Stores, Vecs, constants::*};
/// Input source for tracking where an input came from.
#[derive(Debug)]
pub enum InputSource<'a> {
PreviousBlock {
vin: Vin,
txindex: TxIndex,
outpoint: OutPoint,
address_info: Option<(OutputType, TypeIndex)>,
},
SameBlock {
txindex: TxIndex,
txin: &'a TxIn,
vin: Vin,
outpoint: OutPoint,
},
}
/// Processed output data from parallel output processing.
pub struct ProcessedOutput<'a> {
pub txoutindex: TxOutIndex,
pub txout: &'a TxOut,
pub txindex: TxIndex,
pub vout: Vout,
pub outputtype: OutputType,
pub address_info: Option<(AddressBytes, AddressHash)>,
pub existing_typeindex: Option<TypeIndex>,
}
/// Computed transaction data from parallel TXID computation.
pub struct ComputedTx<'a> {
pub txindex: TxIndex,
pub tx: &'a Transaction,
pub txid: Txid,
pub txid_prefix: TxidPrefix,
pub prev_txindex_opt: Option<TxIndex>,
}
/// Processes a single block, extracting and storing all indexed data.
pub struct BlockProcessor<'a> {
pub block: &'a Block,
pub height: Height,
pub check_collisions: bool,
pub indexes: &'a mut Indexes,
pub vecs: &'a mut Vecs,
pub stores: &'a mut Stores,
pub readers: &'a Readers,
}
impl<'a> BlockProcessor<'a> {
/// Process block metadata (blockhash, difficulty, timestamp, etc.)
pub fn process_block_metadata(&mut self) -> Result<()> {
let height = self.height;
let blockhash = self.block.hash();
let blockhash_prefix = BlockHashPrefix::from(blockhash);
// Check for blockhash prefix collision
if self
.stores
.blockhashprefix_to_height
.get(&blockhash_prefix)?
.is_some_and(|prev_height| *prev_height != height)
{
error!("BlockHash: {blockhash}");
return Err(Error::Internal("BlockHash prefix collision"));
}
self.indexes.checked_push(self.vecs)?;
self.stores
.blockhashprefix_to_height
.insert_if_needed(blockhash_prefix, height, height);
self.stores.height_to_coinbase_tag.insert_if_needed(
height,
self.block.coinbase_tag().into(),
height,
);
self.vecs
.block
.height_to_blockhash
.checked_push(height, blockhash.clone())?;
self.vecs
.block
.height_to_difficulty
.checked_push(height, self.block.header.difficulty_float().into())?;
self.vecs
.block
.height_to_timestamp
.checked_push(height, Timestamp::from(self.block.header.time))?;
self.vecs
.block
.height_to_total_size
.checked_push(height, self.block.total_size().into())?;
self.vecs
.block
.height_to_weight
.checked_push(height, self.block.weight().into())?;
Ok(())
}
/// Compute TXIDs in parallel (CPU-intensive operation).
pub fn compute_txids(&self) -> Result<Vec<ComputedTx<'a>>> {
let will_check_collisions =
self.check_collisions && self.stores.txidprefix_to_txindex.needs(self.height);
let base_txindex = self.indexes.txindex;
self.block
.txdata
.par_iter()
.enumerate()
.map(|(index, tx)| {
let txid = Txid::from(tx.compute_txid());
let txid_prefix = TxidPrefix::from(&txid);
let prev_txindex_opt = if will_check_collisions {
self.stores
.txidprefix_to_txindex
.get(&txid_prefix)?
.map(|v| *v)
} else {
None
};
Ok(ComputedTx {
txindex: base_txindex + TxIndex::from(index),
tx,
txid,
txid_prefix,
prev_txindex_opt,
})
})
.collect()
}
/// Process inputs in parallel.
///
/// Uses collect().into_par_iter() pattern because:
/// 1. The inner work (store lookups, vector reads) is expensive
/// 2. We want to parallelize across ALL inputs, not just per-transaction
/// 3. The intermediate allocation (~8KB per block) is negligible compared to parallelism gains
pub fn process_inputs<'c>(
&self,
txs: &[ComputedTx<'c>],
) -> Result<Vec<(TxInIndex, InputSource<'a>)>> {
let txid_prefix_to_txindex: FxHashMap<_, _> =
txs.iter().map(|ct| (ct.txid_prefix, &ct.txindex)).collect();
let base_txindex = self.indexes.txindex;
let base_txinindex = self.indexes.txinindex;
let txins = self
.block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.input
.iter()
.enumerate()
.map(move |(vin, txin)| (TxIndex::from(index), Vin::from(vin), txin, tx))
})
.collect::<Vec<_>>()
.into_par_iter()
.enumerate()
.map(
|(block_txinindex, (block_txindex, vin, txin, tx))| -> Result<(TxInIndex, InputSource)> {
let txindex = base_txindex + block_txindex;
let txinindex = base_txinindex + TxInIndex::from(block_txinindex);
if tx.is_coinbase() {
return Ok((
txinindex,
InputSource::SameBlock {
txindex,
txin,
vin,
outpoint: OutPoint::COINBASE,
},
));
}
let outpoint = txin.previous_output;
let txid = Txid::from(outpoint.txid);
let txid_prefix = TxidPrefix::from(&txid);
let vout = Vout::from(outpoint.vout);
if let Some(&&same_block_txindex) = txid_prefix_to_txindex
.get(&txid_prefix) {
let outpoint = OutPoint::new(same_block_txindex, vout);
return Ok((
txinindex,
InputSource::SameBlock {
txindex,
txin,
vin,
outpoint,
},
));
}
let prev_txindex = if let Some(txindex) = self
.stores
.txidprefix_to_txindex
.get(&txid_prefix)?
.map(|v| *v)
.and_then(|txindex| {
(txindex < self.indexes.txindex).then_some(txindex)
})
{
txindex
} else {
return Err(Error::UnknownTxid);
};
let txoutindex = self
.vecs
.tx
.txindex_to_first_txoutindex
.get_pushed_or_read(prev_txindex, &self.readers.txindex_to_first_txoutindex)?
.ok_or(Error::Internal("Missing txoutindex"))?
+ vout;
let outpoint = OutPoint::new(prev_txindex, vout);
let outputtype = self
.vecs
.txout
.txoutindex_to_outputtype
.get_pushed_or_read(txoutindex, &self.readers.txoutindex_to_outputtype)?
.ok_or(Error::Internal("Missing outputtype"))?;
let address_info = if outputtype.is_address() {
let typeindex = self
.vecs
.txout
.txoutindex_to_typeindex
.get_pushed_or_read(txoutindex, &self.readers.txoutindex_to_typeindex)?
.ok_or(Error::Internal("Missing typeindex"))?;
Some((outputtype, typeindex))
} else {
None
};
Ok((
txinindex,
InputSource::PreviousBlock {
vin,
txindex,
outpoint,
address_info,
},
))
},
)
.collect::<Result<Vec<_>>>()?;
Ok(txins)
}
/// Collect same-block spent outpoints.
pub fn collect_same_block_spent_outpoints(
txins: &[(TxInIndex, InputSource)],
) -> FxHashSet<OutPoint> {
txins
.iter()
.filter_map(|(_, input_source)| {
let InputSource::SameBlock { outpoint, .. } = input_source else {
return None;
};
if !outpoint.is_coinbase() {
Some(*outpoint)
} else {
None
}
})
.collect()
}
/// Process outputs in parallel.
pub fn process_outputs(&self) -> Result<Vec<ProcessedOutput<'a>>> {
let height = self.height;
let check_collisions = self.check_collisions;
let base_txindex = self.indexes.txindex;
let base_txoutindex = self.indexes.txoutindex;
// Same pattern as inputs: collect then parallelize for maximum parallelism
self.block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.output
.iter()
.enumerate()
.map(move |(vout, txout)| (TxIndex::from(index), Vout::from(vout), txout, tx))
})
.collect::<Vec<_>>()
.into_par_iter()
.enumerate()
.map(
|(block_txoutindex, (block_txindex, vout, txout, tx))| -> Result<ProcessedOutput> {
let txindex = base_txindex + block_txindex;
let txoutindex = base_txoutindex + TxOutIndex::from(block_txoutindex);
let script = &txout.script_pubkey;
let outputtype = OutputType::from(script);
if outputtype.is_not_address() {
return Ok(ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info: None,
existing_typeindex: None,
});
}
let addresstype = outputtype;
let address_bytes = AddressBytes::try_from((script, addresstype)).unwrap();
let address_hash = AddressHash::from(&address_bytes);
let existing_typeindex = self
.stores
.addresstype_to_addresshash_to_addressindex
.get_unwrap(addresstype)
.get(&address_hash)
.unwrap()
.map(|v| *v)
.and_then(|typeindex_local| {
(typeindex_local < self.indexes.to_typeindex(addresstype))
.then_some(typeindex_local)
});
if check_collisions && let Some(typeindex) = existing_typeindex {
let prev_addressbytes_opt = self.vecs.get_addressbytes_by_type(
addresstype,
typeindex,
self.readers.addressbytes.get_unwrap(addresstype),
)?;
let prev_addressbytes = prev_addressbytes_opt
.as_ref()
.ok_or(Error::Internal("Missing addressbytes"))?;
if self
.stores
.addresstype_to_addresshash_to_addressindex
.get_unwrap(addresstype)
.needs(height)
&& prev_addressbytes != &address_bytes
{
let txid = tx.compute_txid();
dbg!(
height,
txid,
vout,
block_txindex,
addresstype,
prev_addressbytes,
&address_bytes,
&self.indexes,
typeindex,
txout,
AddressHash::from(&address_bytes),
);
panic!()
}
}
Ok(ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info: Some((address_bytes, address_hash)),
existing_typeindex,
})
},
)
.collect()
}
/// Finalize outputs sequentially (stores addresses, tracks UTXOs).
pub fn finalize_outputs(
&mut self,
txouts: Vec<ProcessedOutput>,
same_block_spent_outpoints: &FxHashSet<OutPoint>,
) -> Result<FxHashMap<OutPoint, (OutputType, TypeIndex)>> {
let height = self.height;
let mut already_added_addresshash: ByAddressType<FxHashMap<AddressHash, TypeIndex>> =
ByAddressType::default();
// Pre-size based on the number of same-block spent outpoints
let mut same_block_output_info: FxHashMap<OutPoint, (OutputType, TypeIndex)> =
FxHashMap::with_capacity_and_hasher(
same_block_spent_outpoints.len(),
Default::default(),
);
for ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info,
existing_typeindex,
} in txouts
{
let sats = Sats::from(txout.value);
if vout.is_zero() {
self.vecs
.tx
.txindex_to_first_txoutindex
.checked_push(txindex, txoutindex)?;
}
self.vecs
.txout
.txoutindex_to_value
.checked_push(txoutindex, sats)?;
self.vecs
.txout
.txoutindex_to_txindex
.checked_push(txoutindex, txindex)?;
self.vecs
.txout
.txoutindex_to_outputtype
.checked_push(txoutindex, outputtype)?;
let typeindex = if let Some(ti) = existing_typeindex {
ti
} else if let Some((address_bytes, address_hash)) = address_info {
let addresstype = outputtype;
if let Some(&ti) = already_added_addresshash
.get_unwrap(addresstype)
.get(&address_hash)
{
ti
} else {
let ti = self.indexes.increment_address_index(addresstype);
already_added_addresshash
.get_mut_unwrap(addresstype)
.insert(address_hash, ti);
self.stores
.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(addresstype)
.insert_if_needed(address_hash, ti, height);
self.vecs.push_bytes_if_needed(ti, address_bytes)?;
ti
}
} else {
match outputtype {
OutputType::P2MS => {
self.vecs
.output
.p2msoutputindex_to_txindex
.checked_push(self.indexes.p2msoutputindex, txindex)?;
self.indexes.p2msoutputindex.copy_then_increment()
}
OutputType::OpReturn => {
self.vecs
.output
.opreturnindex_to_txindex
.checked_push(self.indexes.opreturnindex, txindex)?;
self.indexes.opreturnindex.copy_then_increment()
}
OutputType::Empty => {
self.vecs
.output
.emptyoutputindex_to_txindex
.checked_push(self.indexes.emptyoutputindex, txindex)?;
self.indexes.emptyoutputindex.copy_then_increment()
}
OutputType::Unknown => {
self.vecs
.output
.unknownoutputindex_to_txindex
.checked_push(self.indexes.unknownoutputindex, txindex)?;
self.indexes.unknownoutputindex.copy_then_increment()
}
_ => unreachable!(),
}
};
self.vecs
.txout
.txoutindex_to_typeindex
.checked_push(txoutindex, typeindex)?;
if outputtype.is_unspendable() {
continue;
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.insert_if_needed(
AddressIndexTxIndex::from((addressindex, txindex)),
Unit,
height,
);
}
let outpoint = OutPoint::new(txindex, vout);
if same_block_spent_outpoints.contains(&outpoint) {
same_block_output_info.insert(outpoint, (outputtype, typeindex));
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.insert_if_needed(
AddressIndexOutPoint::from((addressindex, outpoint)),
Unit,
height,
);
}
}
Ok(same_block_output_info)
}
/// Finalize inputs sequentially (stores outpoints, updates address UTXOs).
pub fn finalize_inputs(
&mut self,
txins: Vec<(TxInIndex, InputSource)>,
same_block_output_info: &mut FxHashMap<OutPoint, (OutputType, TypeIndex)>,
) -> Result<()> {
let height = self.height;
for (txinindex, input_source) in txins {
let (vin, txindex, outpoint, address_info) = match input_source {
InputSource::PreviousBlock {
vin,
txindex,
outpoint,
address_info,
} => (vin, txindex, outpoint, address_info),
InputSource::SameBlock {
txindex,
txin,
vin,
outpoint,
} => {
if outpoint.is_coinbase() {
(vin, txindex, outpoint, None)
} else {
let outputtype_typeindex = same_block_output_info
.remove(&outpoint)
.ok_or(Error::Internal("Same-block addressindex not found"))
.inspect_err(|_| {
dbg!(&same_block_output_info, txin);
})?;
let address_info = if outputtype_typeindex.0.is_address() {
Some(outputtype_typeindex)
} else {
None
};
(vin, txindex, outpoint, address_info)
}
}
};
if vin.is_zero() {
self.vecs
.tx
.txindex_to_first_txinindex
.checked_push(txindex, txinindex)?;
}
self.vecs
.txin
.txinindex_to_txindex
.checked_push(txinindex, txindex)?;
self.vecs
.txin
.txinindex_to_outpoint
.checked_push(txinindex, outpoint)?;
let Some((addresstype, addressindex)) = address_info else {
continue;
};
self.stores
.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.insert_if_needed(
AddressIndexTxIndex::from((addressindex, txindex)),
Unit,
height,
);
self.stores
.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.remove_if_needed(AddressIndexOutPoint::from((addressindex, outpoint)), height);
}
Ok(())
}
/// Check for TXID collisions (only for known duplicate TXIDs).
pub fn check_txid_collisions(&self, txs: &[ComputedTx]) -> Result<()> {
if likely(!self.check_collisions) {
return Ok(());
}
let mut txindex_to_txid_iter = self.vecs.tx.txindex_to_txid.into_iter();
for ct in txs.iter() {
let Some(prev_txindex) = ct.prev_txindex_opt else {
continue;
};
// In case if we start at an already parsed height
if ct.txindex == prev_txindex {
continue;
}
let len = self.vecs.tx.txindex_to_txid.len();
let prev_txid = txindex_to_txid_iter
.get(prev_txindex)
.ok_or(Error::Internal("Missing txid for txindex"))
.inspect_err(|_| {
dbg!(ct.txindex, len);
})?;
let is_dup = DUPLICATE_TXIDS.contains(&prev_txid);
if !is_dup {
dbg!(self.height, ct.txindex, prev_txid, prev_txindex);
return Err(Error::Internal("Unexpected TXID collision"));
}
}
Ok(())
}
/// Store transaction metadata.
pub fn store_transaction_metadata(&mut self, txs: Vec<ComputedTx>) -> Result<()> {
let height = self.height;
for ct in txs {
if ct.prev_txindex_opt.is_none() {
self.stores.txidprefix_to_txindex.insert_if_needed(
ct.txid_prefix,
ct.txindex,
height,
);
}
self.vecs
.tx
.txindex_to_height
.checked_push(ct.txindex, height)?;
self.vecs
.tx
.txindex_to_txversion
.checked_push(ct.txindex, ct.tx.version.into())?;
self.vecs
.tx
.txindex_to_txid
.checked_push(ct.txindex, ct.txid)?;
self.vecs
.tx
.txindex_to_rawlocktime
.checked_push(ct.txindex, ct.tx.lock_time.into())?;
self.vecs
.tx
.txindex_to_base_size
.checked_push(ct.txindex, ct.tx.base_size().into())?;
self.vecs
.tx
.txindex_to_total_size
.checked_push(ct.txindex, ct.tx.total_size().into())?;
self.vecs
.tx
.txindex_to_is_explicitly_rbf
.checked_push(ct.txindex, StoredBool::from(ct.tx.is_explicitly_rbf()))?;
}
Ok(())
}
/// Update global indexes after processing a block.
pub fn update_indexes(&mut self, tx_count: usize, input_count: usize, output_count: usize) {
self.indexes.txindex += TxIndex::from(tx_count);
self.indexes.txinindex += TxInIndex::from(input_count);
self.indexes.txoutindex += TxOutIndex::from(output_count);
}
}

View File

@@ -0,0 +1,63 @@
//! Block metadata processing.
use brk_error::{Error, Result};
use brk_types::{BlockHashPrefix, Timestamp};
use log::error;
use vecdb::GenericStoredVec;
use super::BlockProcessor;
impl BlockProcessor<'_> {
/// Process block metadata (blockhash, difficulty, timestamp, etc.)
pub fn process_block_metadata(&mut self) -> Result<()> {
let height = self.height;
let blockhash = self.block.hash();
let blockhash_prefix = BlockHashPrefix::from(blockhash);
// Check for blockhash prefix collision
if self
.stores
.blockhashprefix_to_height
.get(&blockhash_prefix)?
.is_some_and(|prev_height| *prev_height != height)
{
error!("BlockHash: {blockhash}");
return Err(Error::Internal("BlockHash prefix collision"));
}
self.indexes.checked_push(self.vecs)?;
self.stores
.blockhashprefix_to_height
.insert_if_needed(blockhash_prefix, height, height);
self.stores.height_to_coinbase_tag.insert_if_needed(
height,
self.block.coinbase_tag().into(),
height,
);
self.vecs
.block
.height_to_blockhash
.checked_push(height, blockhash.clone())?;
self.vecs
.block
.height_to_difficulty
.checked_push(height, self.block.header.difficulty_float().into())?;
self.vecs
.block
.height_to_timestamp
.checked_push(height, Timestamp::from(self.block.header.time))?;
self.vecs
.block
.height_to_total_size
.checked_push(height, self.block.total_size().into())?;
self.vecs
.block
.height_to_weight
.checked_push(height, self.block.weight().into())?;
Ok(())
}
}

View File

@@ -0,0 +1,37 @@
//! Block processing for indexing.
//!
//! This module handles the extraction and storage of all indexed data from blocks.
//! Processing is split into phases that can be parallelized where possible.
mod metadata;
mod tx;
mod txin;
mod txout;
mod types;
pub use types::*;
use brk_types::{Block, Height, TxInIndex, TxIndex, TxOutIndex};
use crate::{Indexes, RangeMap, Readers, Stores, Vecs};
/// Processes a single block, extracting and storing all indexed data.
pub struct BlockProcessor<'a> {
pub block: &'a Block,
pub height: Height,
pub check_collisions: bool,
pub indexes: &'a mut Indexes,
pub vecs: &'a mut Vecs,
pub stores: &'a mut Stores,
pub readers: &'a Readers,
pub txindex_to_height: &'a RangeMap<TxIndex, Height>,
}
impl BlockProcessor<'_> {
/// Update global indexes after processing a block.
pub fn update_indexes(&mut self, tx_count: usize, input_count: usize, output_count: usize) {
self.indexes.txindex += TxIndex::from(tx_count);
self.indexes.txinindex += TxInIndex::from(input_count);
self.indexes.txoutindex += TxOutIndex::from(output_count);
}
}

View File

@@ -0,0 +1,128 @@
//! TXID computation and collision checking.
use brk_error::{Error, Result};
use brk_types::{StoredBool, TxIndex, Txid, TxidPrefix};
use rayon::prelude::*;
use vecdb::{AnyVec, GenericStoredVec, TypedVecIterator, likely};
use crate::constants::DUPLICATE_TXIDS;
use super::{BlockProcessor, ComputedTx};
impl<'a> BlockProcessor<'a> {
/// Compute TXIDs in parallel (CPU-intensive operation).
pub fn compute_txids(&self) -> Result<Vec<ComputedTx<'a>>> {
let will_check_collisions =
self.check_collisions && self.stores.txidprefix_to_txindex.needs(self.height);
let base_txindex = self.indexes.txindex;
self.block
.txdata
.par_iter()
.enumerate()
.map(|(index, tx)| {
let txid = Txid::from(tx.compute_txid());
let txid_prefix = TxidPrefix::from(&txid);
let prev_txindex_opt = if will_check_collisions {
self.stores
.txidprefix_to_txindex
.get(&txid_prefix)?
.map(|v| *v)
} else {
None
};
Ok(ComputedTx {
txindex: base_txindex + TxIndex::from(index),
tx,
txid,
txid_prefix,
prev_txindex_opt,
})
})
.collect()
}
/// Check for TXID collisions (only for known duplicate TXIDs).
pub fn check_txid_collisions(&self, txs: &[ComputedTx]) -> Result<()> {
if likely(!self.check_collisions) {
return Ok(());
}
let mut txindex_to_txid_iter = self.vecs.tx.txindex_to_txid.into_iter();
for ct in txs.iter() {
let Some(prev_txindex) = ct.prev_txindex_opt else {
continue;
};
// In case if we start at an already parsed height
if ct.txindex == prev_txindex {
continue;
}
let len = self.vecs.tx.txindex_to_txid.len();
let prev_txid = txindex_to_txid_iter
.get(prev_txindex)
.ok_or(Error::Internal("Missing txid for txindex"))
.inspect_err(|_| {
dbg!(ct.txindex, len);
})?;
let is_dup = DUPLICATE_TXIDS.contains(&prev_txid);
if !is_dup {
dbg!(self.height, ct.txindex, prev_txid, prev_txindex);
return Err(Error::Internal("Unexpected TXID collision"));
}
}
Ok(())
}
/// Store transaction metadata.
pub fn store_transaction_metadata(&mut self, txs: Vec<ComputedTx>) -> Result<()> {
let height = self.height;
for ct in txs {
if ct.prev_txindex_opt.is_none() {
self.stores.txidprefix_to_txindex.insert_if_needed(
ct.txid_prefix,
ct.txindex,
height,
);
}
self.vecs
.tx
.txindex_to_height
.checked_push(ct.txindex, height)?;
self.vecs
.tx
.txindex_to_txversion
.checked_push(ct.txindex, ct.tx.version.into())?;
self.vecs
.tx
.txindex_to_txid
.checked_push(ct.txindex, ct.txid)?;
self.vecs
.tx
.txindex_to_rawlocktime
.checked_push(ct.txindex, ct.tx.lock_time.into())?;
self.vecs
.tx
.txindex_to_base_size
.checked_push(ct.txindex, ct.tx.base_size().into())?;
self.vecs
.tx
.txindex_to_total_size
.checked_push(ct.txindex, ct.tx.total_size().into())?;
self.vecs
.tx
.txindex_to_is_explicitly_rbf
.checked_push(ct.txindex, StoredBool::from(ct.tx.is_explicitly_rbf()))?;
}
Ok(())
}
}

View File

@@ -0,0 +1,284 @@
//! Input processing for block indexing.
use brk_error::{Error, Result};
use brk_types::{
AddressIndexOutPoint, AddressIndexTxIndex, OutPoint, OutputType, Sats, TxInIndex, TxIndex,
TxOutIndex, Txid, TxidPrefix, TypeIndex, Unit, Vin, Vout,
};
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use vecdb::GenericStoredVec;
use super::{BlockProcessor, ComputedTx, InputSource, SameBlockOutputInfo};
impl<'a> BlockProcessor<'a> {
/// Process inputs in parallel.
///
/// Uses collect().into_par_iter() pattern because:
/// 1. The inner work (store lookups, vector reads) is expensive
/// 2. We want to parallelize across ALL inputs, not just per-transaction
/// 3. The intermediate allocation (~8KB per block) is negligible compared to parallelism gains
pub fn process_inputs<'c>(
&self,
txs: &[ComputedTx<'c>],
) -> Result<Vec<(TxInIndex, InputSource<'a>)>> {
let txid_prefix_to_txindex: FxHashMap<_, _> =
txs.iter().map(|ct| (ct.txid_prefix, &ct.txindex)).collect();
let base_txindex = self.indexes.txindex;
let base_txinindex = self.indexes.txinindex;
let txins = self
.block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.input
.iter()
.enumerate()
.map(move |(vin, txin)| (TxIndex::from(index), Vin::from(vin), txin, tx))
})
.collect::<Vec<_>>()
.into_par_iter()
.enumerate()
.map(
|(block_txinindex, (block_txindex, vin, txin, tx))| -> Result<(TxInIndex, InputSource)> {
let txindex = base_txindex + block_txindex;
let txinindex = base_txinindex + TxInIndex::from(block_txinindex);
if tx.is_coinbase() {
return Ok((
txinindex,
InputSource::SameBlock {
txindex,
txin,
vin,
outpoint: OutPoint::COINBASE,
},
));
}
let outpoint = txin.previous_output;
let txid = Txid::from(outpoint.txid);
let txid_prefix = TxidPrefix::from(&txid);
let vout = Vout::from(outpoint.vout);
if let Some(&&same_block_txindex) = txid_prefix_to_txindex
.get(&txid_prefix) {
let outpoint = OutPoint::new(same_block_txindex, vout);
return Ok((
txinindex,
InputSource::SameBlock {
txindex,
txin,
vin,
outpoint,
},
));
}
let prev_txindex = if let Some(txindex) = self
.stores
.txidprefix_to_txindex
.get(&txid_prefix)?
.map(|v| *v)
.and_then(|txindex| {
(txindex < self.indexes.txindex).then_some(txindex)
})
{
txindex
} else {
return Err(Error::UnknownTxid);
};
let txoutindex = self
.vecs
.tx
.txindex_to_first_txoutindex
.get_pushed_or_read(prev_txindex, &self.readers.txindex_to_first_txoutindex)?
.ok_or(Error::Internal("Missing txoutindex"))?
+ vout;
let outpoint = OutPoint::new(prev_txindex, vout);
let txoutdata = self
.vecs
.txout
.txoutindex_to_txoutdata
.get_pushed_or_read(txoutindex, &self.readers.txoutindex_to_txoutdata)?
.ok_or(Error::Internal("Missing txout data"))?;
let value = txoutdata.value;
let outputtype = txoutdata.outputtype;
let typeindex = txoutdata.typeindex;
let height = self
.txindex_to_height
.get(prev_txindex)
.ok_or(Error::Internal("Missing height in txindex_to_height map"))?;
Ok((
txinindex,
InputSource::PreviousBlock {
vin,
value,
height,
txindex,
txoutindex,
outpoint,
outputtype,
typeindex,
},
))
},
)
.collect::<Result<Vec<_>>>()?;
Ok(txins)
}
/// Collect same-block spent outpoints.
pub fn collect_same_block_spent_outpoints(
txins: &[(TxInIndex, InputSource)],
) -> FxHashSet<OutPoint> {
txins
.iter()
.filter_map(|(_, input_source)| {
let InputSource::SameBlock { outpoint, .. } = input_source else {
return None;
};
if !outpoint.is_coinbase() {
Some(*outpoint)
} else {
None
}
})
.collect()
}
/// Finalize inputs sequentially (stores outpoints, updates address UTXOs).
pub fn finalize_inputs(
&mut self,
txins: Vec<(TxInIndex, InputSource)>,
same_block_output_info: &mut FxHashMap<OutPoint, SameBlockOutputInfo>,
) -> Result<()> {
let height = self.height;
for (txinindex, input_source) in txins {
let (prev_height, vin, txindex, value, outpoint, txoutindex, outputtype, typeindex) =
match input_source {
InputSource::PreviousBlock {
height,
vin,
txindex,
txoutindex,
value,
outpoint,
outputtype,
typeindex,
} => (
height, vin, txindex, value, outpoint, txoutindex, outputtype, typeindex,
),
InputSource::SameBlock {
txindex,
txin,
vin,
outpoint,
} => {
if outpoint.is_coinbase() {
(
height,
vin,
txindex,
Sats::COINBASE,
outpoint,
TxOutIndex::COINBASE,
OutputType::Unknown,
TypeIndex::COINBASE,
)
} else {
let info = same_block_output_info
.remove(&outpoint)
.ok_or(Error::Internal("Same-block output not found"))
.inspect_err(|_| {
dbg!(&same_block_output_info, txin);
})?;
(
height,
vin,
txindex,
info.value,
outpoint,
info.txoutindex,
info.outputtype,
info.typeindex,
)
}
}
};
if vin.is_zero() {
self.vecs
.tx
.txindex_to_first_txinindex
.checked_push(txindex, txinindex)?;
}
self.vecs
.txin
.txinindex_to_txindex
.checked_push(txinindex, txindex)?;
self.vecs
.txin
.txinindex_to_outpoint
.checked_push(txinindex, outpoint)?;
self.vecs
.txin
.txinindex_to_value
.checked_push(txinindex, value)?;
self.vecs
.txin
.txinindex_to_prev_height
.checked_push(txinindex, prev_height)?;
self.vecs
.txin
.txinindex_to_outputtype
.checked_push(txinindex, outputtype)?;
self.vecs
.txin
.txinindex_to_typeindex
.checked_push(txinindex, typeindex)?;
// Update txoutindex_to_txinindex for non-coinbase inputs
if !txoutindex.is_coinbase() {
self.vecs
.txout
.txoutindex_to_txinindex
.update(txoutindex, txinindex)?;
}
if !outputtype.is_address() {
continue;
}
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.insert_if_needed(
AddressIndexTxIndex::from((addressindex, txindex)),
Unit,
height,
);
self.stores
.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.remove_if_needed(AddressIndexOutPoint::from((addressindex, outpoint)), height);
}
Ok(())
}
}

View File

@@ -0,0 +1,275 @@
//! Output processing for block indexing.
use brk_error::{Error, Result};
use brk_grouper::ByAddressType;
use brk_types::{
AddressBytes, AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, OutPoint, OutputType,
Sats, TxInIndex, TxIndex, TxOutData, TxOutIndex, TypeIndex, Unit, Vout,
};
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use vecdb::GenericStoredVec;
use super::{BlockProcessor, ProcessedOutput, SameBlockOutputInfo};
impl<'a> BlockProcessor<'a> {
/// Process outputs in parallel.
pub fn process_outputs(&self) -> Result<Vec<ProcessedOutput<'a>>> {
let height = self.height;
let check_collisions = self.check_collisions;
let base_txindex = self.indexes.txindex;
let base_txoutindex = self.indexes.txoutindex;
// Same pattern as inputs: collect then parallelize for maximum parallelism
self.block
.txdata
.iter()
.enumerate()
.flat_map(|(index, tx)| {
tx.output
.iter()
.enumerate()
.map(move |(vout, txout)| (TxIndex::from(index), Vout::from(vout), txout, tx))
})
.collect::<Vec<_>>()
.into_par_iter()
.enumerate()
.map(
|(block_txoutindex, (block_txindex, vout, txout, tx))| -> Result<ProcessedOutput> {
let txindex = base_txindex + block_txindex;
let txoutindex = base_txoutindex + TxOutIndex::from(block_txoutindex);
let script = &txout.script_pubkey;
let outputtype = OutputType::from(script);
if outputtype.is_not_address() {
return Ok(ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info: None,
existing_typeindex: None,
});
}
let addresstype = outputtype;
let address_bytes = AddressBytes::try_from((script, addresstype)).unwrap();
let address_hash = AddressHash::from(&address_bytes);
let existing_typeindex = self
.stores
.addresstype_to_addresshash_to_addressindex
.get_unwrap(addresstype)
.get(&address_hash)
.unwrap()
.map(|v| *v)
.and_then(|typeindex_local| {
(typeindex_local < self.indexes.to_typeindex(addresstype))
.then_some(typeindex_local)
});
if check_collisions && let Some(typeindex) = existing_typeindex {
let prev_addressbytes_opt = self.vecs.get_addressbytes_by_type(
addresstype,
typeindex,
self.readers.addressbytes.get_unwrap(addresstype),
)?;
let prev_addressbytes = prev_addressbytes_opt
.as_ref()
.ok_or(Error::Internal("Missing addressbytes"))?;
if self
.stores
.addresstype_to_addresshash_to_addressindex
.get_unwrap(addresstype)
.needs(height)
&& prev_addressbytes != &address_bytes
{
let txid = tx.compute_txid();
dbg!(
height,
txid,
vout,
block_txindex,
addresstype,
prev_addressbytes,
&address_bytes,
&self.indexes,
typeindex,
txout,
AddressHash::from(&address_bytes),
);
panic!()
}
}
Ok(ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info: Some((address_bytes, address_hash)),
existing_typeindex,
})
},
)
.collect()
}
/// Finalize outputs sequentially (stores addresses, tracks UTXOs).
pub fn finalize_outputs(
&mut self,
txouts: Vec<ProcessedOutput>,
same_block_spent_outpoints: &FxHashSet<OutPoint>,
) -> Result<FxHashMap<OutPoint, SameBlockOutputInfo>> {
let height = self.height;
let mut already_added_addresshash: ByAddressType<FxHashMap<AddressHash, TypeIndex>> =
ByAddressType::default();
// Pre-size based on the number of same-block spent outpoints
let mut same_block_output_info: FxHashMap<OutPoint, SameBlockOutputInfo> =
FxHashMap::with_capacity_and_hasher(
same_block_spent_outpoints.len(),
Default::default(),
);
for ProcessedOutput {
txoutindex,
txout,
txindex,
vout,
outputtype,
address_info,
existing_typeindex,
} in txouts
{
let sats = Sats::from(txout.value);
if vout.is_zero() {
self.vecs
.tx
.txindex_to_first_txoutindex
.checked_push(txindex, txoutindex)?;
}
self.vecs
.txout
.txoutindex_to_txindex
.checked_push(txoutindex, txindex)?;
let typeindex = if let Some(ti) = existing_typeindex {
ti
} else if let Some((address_bytes, address_hash)) = address_info {
let addresstype = outputtype;
if let Some(&ti) = already_added_addresshash
.get_unwrap(addresstype)
.get(&address_hash)
{
ti
} else {
let ti = self.indexes.increment_address_index(addresstype);
already_added_addresshash
.get_mut_unwrap(addresstype)
.insert(address_hash, ti);
self.stores
.addresstype_to_addresshash_to_addressindex
.get_mut_unwrap(addresstype)
.insert_if_needed(address_hash, ti, height);
self.vecs.push_bytes_if_needed(ti, address_bytes)?;
ti
}
} else {
match outputtype {
OutputType::P2MS => {
self.vecs
.output
.p2msoutputindex_to_txindex
.checked_push(self.indexes.p2msoutputindex, txindex)?;
self.indexes.p2msoutputindex.copy_then_increment()
}
OutputType::OpReturn => {
self.vecs
.output
.opreturnindex_to_txindex
.checked_push(self.indexes.opreturnindex, txindex)?;
self.indexes.opreturnindex.copy_then_increment()
}
OutputType::Empty => {
self.vecs
.output
.emptyoutputindex_to_txindex
.checked_push(self.indexes.emptyoutputindex, txindex)?;
self.indexes.emptyoutputindex.copy_then_increment()
}
OutputType::Unknown => {
self.vecs
.output
.unknownoutputindex_to_txindex
.checked_push(self.indexes.unknownoutputindex, txindex)?;
self.indexes.unknownoutputindex.copy_then_increment()
}
_ => unreachable!(),
}
};
let txoutdata = TxOutData::new(sats, outputtype, typeindex);
self.vecs
.txout
.txoutindex_to_txoutdata
.checked_push(txoutindex, txoutdata)?;
self.vecs
.txout
.txoutindex_to_txinindex
.checked_push(txoutindex, TxInIndex::UNSPENT)?;
if outputtype.is_unspendable() {
continue;
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.insert_if_needed(
AddressIndexTxIndex::from((addressindex, txindex)),
Unit,
height,
);
}
let outpoint = OutPoint::new(txindex, vout);
if same_block_spent_outpoints.contains(&outpoint) {
same_block_output_info.insert(
outpoint,
SameBlockOutputInfo {
outputtype,
typeindex,
value: sats,
txoutindex,
},
);
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
self.stores
.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.insert_if_needed(
AddressIndexOutPoint::from((addressindex, outpoint)),
Unit,
height,
);
}
}
Ok(same_block_output_info)
}
}

View File

@@ -0,0 +1,57 @@
//! Type definitions for block processing.
use bitcoin::{Transaction, TxIn, TxOut};
use brk_types::{
AddressBytes, AddressHash, Height, OutPoint, OutputType, Sats, TxIndex, TxOutIndex, Txid,
TxidPrefix, TypeIndex, Vin, Vout,
};
/// Input source for tracking where an input came from.
#[derive(Debug)]
pub enum InputSource<'a> {
PreviousBlock {
vin: Vin,
value: Sats,
height: Height,
txindex: TxIndex,
txoutindex: TxOutIndex,
outpoint: OutPoint,
outputtype: OutputType,
typeindex: TypeIndex,
},
SameBlock {
txindex: TxIndex,
txin: &'a TxIn,
vin: Vin,
outpoint: OutPoint,
},
}
/// Output info for same-block spends (output created and spent in the same block).
#[derive(Debug, Clone, Copy)]
pub struct SameBlockOutputInfo {
pub outputtype: OutputType,
pub typeindex: TypeIndex,
pub value: Sats,
pub txoutindex: TxOutIndex,
}
/// Processed output data from parallel output processing.
pub struct ProcessedOutput<'a> {
pub txoutindex: TxOutIndex,
pub txout: &'a TxOut,
pub txindex: TxIndex,
pub vout: Vout,
pub outputtype: OutputType,
pub address_info: Option<(AddressBytes, AddressHash)>,
pub existing_typeindex: Option<TypeIndex>,
}
/// Computed transaction data from parallel TXID computation.
pub struct ComputedTx<'a> {
pub txindex: TxIndex,
pub tx: &'a Transaction,
pub txid: Txid,
pub txid_prefix: TxidPrefix,
pub prev_txindex_opt: Option<TxIndex>,
}

View File

@@ -0,0 +1,40 @@
//! Range-based lookup map for efficient index -> value lookups.
//!
//! Uses the pattern that many indices share the same value (e.g., all txindexes
//! in a block have the same height) to provide O(log n) lookups via BTreeMap.
use std::collections::BTreeMap;
use vecdb::VecIndex;
/// Maps ranges of indices to values for efficient reverse lookups.
///
/// Instead of storing a value for every index, stores (first_index, value)
/// pairs and uses range search to find the value for any index.
#[derive(Debug, Default)]
pub struct RangeMap<I, V>(BTreeMap<I, V>);
impl<I: VecIndex, V: Copy> RangeMap<I, V> {
/// Create a new empty map.
pub fn new() -> Self {
Self(BTreeMap::new())
}
/// Insert a new (first_index, value) mapping.
#[inline]
pub fn insert(&mut self, first_index: I, value: V) {
self.0.insert(first_index, value);
}
/// Look up value for an index using range search.
/// Returns the value associated with the largest first_index <= given index.
#[inline]
pub fn get(&self, index: I) -> Option<V> {
self.0.range(..=index).next_back().map(|(_, &v)| v)
}
/// Clear all entries (for reset/rollback).
pub fn clear(&mut self) {
self.0.clear();
}
}

View File

@@ -7,8 +7,7 @@ use crate::Vecs;
/// These provide consistent snapshots for reading while the main vectors are being modified. /// These provide consistent snapshots for reading while the main vectors are being modified.
pub struct Readers { pub struct Readers {
pub txindex_to_first_txoutindex: Reader, pub txindex_to_first_txoutindex: Reader,
pub txoutindex_to_outputtype: Reader, pub txoutindex_to_txoutdata: Reader,
pub txoutindex_to_typeindex: Reader,
pub addressbytes: ByAddressType<Reader>, pub addressbytes: ByAddressType<Reader>,
} }
@@ -16,14 +15,22 @@ impl Readers {
pub fn new(vecs: &Vecs) -> Self { pub fn new(vecs: &Vecs) -> Self {
Self { Self {
txindex_to_first_txoutindex: vecs.tx.txindex_to_first_txoutindex.create_reader(), txindex_to_first_txoutindex: vecs.tx.txindex_to_first_txoutindex.create_reader(),
txoutindex_to_outputtype: vecs.txout.txoutindex_to_outputtype.create_reader(), txoutindex_to_txoutdata: vecs.txout.txoutindex_to_txoutdata.create_reader(),
txoutindex_to_typeindex: vecs.txout.txoutindex_to_typeindex.create_reader(),
addressbytes: ByAddressType { addressbytes: ByAddressType {
p2pk65: vecs.address.p2pk65addressindex_to_p2pk65bytes.create_reader(), p2pk65: vecs
p2pk33: vecs.address.p2pk33addressindex_to_p2pk33bytes.create_reader(), .address
.p2pk65addressindex_to_p2pk65bytes
.create_reader(),
p2pk33: vecs
.address
.p2pk33addressindex_to_p2pk33bytes
.create_reader(),
p2pkh: vecs.address.p2pkhaddressindex_to_p2pkhbytes.create_reader(), p2pkh: vecs.address.p2pkhaddressindex_to_p2pkhbytes.create_reader(),
p2sh: vecs.address.p2shaddressindex_to_p2shbytes.create_reader(), p2sh: vecs.address.p2shaddressindex_to_p2shbytes.create_reader(),
p2wpkh: vecs.address.p2wpkhaddressindex_to_p2wpkhbytes.create_reader(), p2wpkh: vecs
.address
.p2wpkhaddressindex_to_p2wpkhbytes
.create_reader(),
p2wsh: vecs.address.p2wshaddressindex_to_p2wshbytes.create_reader(), p2wsh: vecs.address.p2wshaddressindex_to_p2wshbytes.create_reader(),
p2tr: vecs.address.p2traddressindex_to_p2trbytes.create_reader(), p2tr: vecs.address.p2traddressindex_to_p2trbytes.create_reader(),
p2a: vecs.address.p2aaddressindex_to_p2abytes.create_reader(), p2a: vecs.address.p2aaddressindex_to_p2abytes.create_reader(),

View File

@@ -5,7 +5,8 @@ use brk_grouper::ByAddressType;
use brk_store::{AnyStore, Kind, Mode, Store}; use brk_store::{AnyStore, Kind, Mode, Store};
use brk_types::{ use brk_types::{
AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, BlockHashPrefix, Height, OutPoint, AddressHash, AddressIndexOutPoint, AddressIndexTxIndex, BlockHashPrefix, Height, OutPoint,
OutputType, StoredString, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, Unit, Version, Vout, OutputType, StoredString, TxInIndex, TxIndex, TxOutIndex, TxidPrefix, TypeIndex, Unit, Version,
Vout,
}; };
use fjall::{Database, PersistMode}; use fjall::{Database, PersistMode};
use log::info; use log::info;
@@ -270,20 +271,14 @@ impl Stores {
let mut txindex_to_first_txoutindex_iter = let mut txindex_to_first_txoutindex_iter =
vecs.tx.txindex_to_first_txoutindex.iter()?; vecs.tx.txindex_to_first_txoutindex.iter()?;
vecs.txout vecs.txout
.txoutindex_to_outputtype .txoutindex_to_txoutdata
.iter()? .iter()?
.enumerate() .enumerate()
.skip(starting_indexes.txoutindex.to_usize()) .skip(starting_indexes.txoutindex.to_usize())
.zip( .filter(|(_, txoutdata)| txoutdata.outputtype.is_address())
vecs.txout .for_each(|(txoutindex, txoutdata)| {
.txoutindex_to_typeindex let addresstype = txoutdata.outputtype;
.iter()? let addressindex = txoutdata.typeindex;
.skip(starting_indexes.txoutindex.to_usize()),
)
.filter(|((_, outputtype), _): &((usize, OutputType), TypeIndex)| {
outputtype.is_address()
})
.for_each(|((txoutindex, addresstype), addressindex)| {
let txindex = txoutindex_to_txindex_iter.get_at_unwrap(txoutindex); let txindex = txoutindex_to_txindex_iter.get_at_unwrap(txoutindex);
self.addresstype_to_addressindex_and_txindex self.addresstype_to_addressindex_and_txindex
@@ -303,20 +298,22 @@ impl Stores {
.remove(AddressIndexOutPoint::from((addressindex, outpoint))); .remove(AddressIndexOutPoint::from((addressindex, outpoint)));
}); });
// Add back outputs that were spent after the rollback point // Collect outputs that were spent after the rollback point
// We need to: 1) reset their spend status, 2) restore address stores
let mut txindex_to_first_txoutindex_iter = let mut txindex_to_first_txoutindex_iter =
vecs.tx.txindex_to_first_txoutindex.iter()?; vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut txoutindex_to_outputtype_iter = vecs.txout.txoutindex_to_outputtype.iter()?; let mut txoutindex_to_txoutdata_iter = vecs.txout.txoutindex_to_txoutdata.iter()?;
let mut txoutindex_to_typeindex_iter = vecs.txout.txoutindex_to_typeindex.iter()?;
let mut txinindex_to_txindex_iter = vecs.txin.txinindex_to_txindex.iter()?; let mut txinindex_to_txindex_iter = vecs.txin.txinindex_to_txindex.iter()?;
vecs.txin
let outputs_to_unspend: Vec<_> = vecs
.txin
.txinindex_to_outpoint .txinindex_to_outpoint
.iter()? .iter()?
.enumerate() .enumerate()
.skip(starting_indexes.txinindex.to_usize()) .skip(starting_indexes.txinindex.to_usize())
.for_each(|(txinindex, outpoint): (usize, OutPoint)| { .filter_map(|(txinindex, outpoint): (usize, OutPoint)| {
if outpoint.is_coinbase() { if outpoint.is_coinbase() {
return; return None;
} }
let output_txindex = outpoint.txindex(); let output_txindex = outpoint.txindex();
@@ -328,29 +325,38 @@ impl Stores {
// Only process if this output was created before the rollback point // Only process if this output was created before the rollback point
if txoutindex < starting_indexes.txoutindex { if txoutindex < starting_indexes.txoutindex {
let outputtype = txoutindex_to_outputtype_iter.get_unwrap(txoutindex); let txoutdata = txoutindex_to_txoutdata_iter.get_unwrap(txoutindex);
let spending_txindex =
txinindex_to_txindex_iter.get_at_unwrap(txinindex);
if outputtype.is_address() { Some((txoutindex, outpoint, txoutdata, spending_txindex))
let addresstype = outputtype; } else {
let addressindex = txoutindex_to_typeindex_iter.get_unwrap(txoutindex); None
// Get the SPENDING tx's index (not the output's tx)
let spending_txindex =
txinindex_to_txindex_iter.get_at_unwrap(txinindex);
self.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.remove(AddressIndexTxIndex::from((
addressindex,
spending_txindex,
)));
self.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.insert(AddressIndexOutPoint::from((addressindex, outpoint)), Unit);
}
} }
}); })
.collect();
// Now process the collected outputs (iterators dropped, can mutate vecs)
for (txoutindex, outpoint, txoutdata, spending_txindex) in outputs_to_unspend {
// Reset spend status back to unspent
vecs.txout
.txoutindex_to_txinindex
.update(txoutindex, TxInIndex::UNSPENT)?;
// Restore address stores if this is an address output
if txoutdata.outputtype.is_address() {
let addresstype = txoutdata.outputtype;
let addressindex = txoutdata.typeindex;
self.addresstype_to_addressindex_and_txindex
.get_mut_unwrap(addresstype)
.remove(AddressIndexTxIndex::from((addressindex, spending_txindex)));
self.addresstype_to_addressindex_and_unspentoutpoint
.get_mut_unwrap(addresstype)
.insert(AddressIndexOutPoint::from((addressindex, outpoint)), Unit);
}
}
} else { } else {
unreachable!(); unreachable!();
} }

View File

@@ -12,6 +12,8 @@ use vecdb::{
TypedVecIterator, TypedVecIterator,
}; };
use crate::parallel_import;
#[derive(Clone, Traversable)] #[derive(Clone, Traversable)]
pub struct AddressVecs { pub struct AddressVecs {
// Height to first address index (per address type) // Height to first address index (per address type)
@@ -36,55 +38,58 @@ pub struct AddressVecs {
impl AddressVecs { impl AddressVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> { pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
height_to_first_p2pk65addressindex,
height_to_first_p2pk33addressindex,
height_to_first_p2pkhaddressindex,
height_to_first_p2shaddressindex,
height_to_first_p2wpkhaddressindex,
height_to_first_p2wshaddressindex,
height_to_first_p2traddressindex,
height_to_first_p2aaddressindex,
p2pk65addressindex_to_p2pk65bytes,
p2pk33addressindex_to_p2pk33bytes,
p2pkhaddressindex_to_p2pkhbytes,
p2shaddressindex_to_p2shbytes,
p2wpkhaddressindex_to_p2wpkhbytes,
p2wshaddressindex_to_p2wshbytes,
p2traddressindex_to_p2trbytes,
p2aaddressindex_to_p2abytes,
) = parallel_import! {
height_to_first_p2pk65addressindex = PcoVec::forced_import(db, "first_p2pk65addressindex", version),
height_to_first_p2pk33addressindex = PcoVec::forced_import(db, "first_p2pk33addressindex", version),
height_to_first_p2pkhaddressindex = PcoVec::forced_import(db, "first_p2pkhaddressindex", version),
height_to_first_p2shaddressindex = PcoVec::forced_import(db, "first_p2shaddressindex", version),
height_to_first_p2wpkhaddressindex = PcoVec::forced_import(db, "first_p2wpkhaddressindex", version),
height_to_first_p2wshaddressindex = PcoVec::forced_import(db, "first_p2wshaddressindex", version),
height_to_first_p2traddressindex = PcoVec::forced_import(db, "first_p2traddressindex", version),
height_to_first_p2aaddressindex = PcoVec::forced_import(db, "first_p2aaddressindex", version),
p2pk65addressindex_to_p2pk65bytes = BytesVec::forced_import(db, "p2pk65bytes", version),
p2pk33addressindex_to_p2pk33bytes = BytesVec::forced_import(db, "p2pk33bytes", version),
p2pkhaddressindex_to_p2pkhbytes = BytesVec::forced_import(db, "p2pkhbytes", version),
p2shaddressindex_to_p2shbytes = BytesVec::forced_import(db, "p2shbytes", version),
p2wpkhaddressindex_to_p2wpkhbytes = BytesVec::forced_import(db, "p2wpkhbytes", version),
p2wshaddressindex_to_p2wshbytes = BytesVec::forced_import(db, "p2wshbytes", version),
p2traddressindex_to_p2trbytes = BytesVec::forced_import(db, "p2trbytes", version),
p2aaddressindex_to_p2abytes = BytesVec::forced_import(db, "p2abytes", version),
};
Ok(Self { Ok(Self {
height_to_first_p2pk65addressindex: PcoVec::forced_import( height_to_first_p2pk65addressindex,
db, height_to_first_p2pk33addressindex,
"first_p2pk65addressindex", height_to_first_p2pkhaddressindex,
version, height_to_first_p2shaddressindex,
)?, height_to_first_p2wpkhaddressindex,
height_to_first_p2pk33addressindex: PcoVec::forced_import( height_to_first_p2wshaddressindex,
db, height_to_first_p2traddressindex,
"first_p2pk33addressindex", height_to_first_p2aaddressindex,
version, p2pk65addressindex_to_p2pk65bytes,
)?, p2pk33addressindex_to_p2pk33bytes,
height_to_first_p2pkhaddressindex: PcoVec::forced_import( p2pkhaddressindex_to_p2pkhbytes,
db, p2shaddressindex_to_p2shbytes,
"first_p2pkhaddressindex", p2wpkhaddressindex_to_p2wpkhbytes,
version, p2wshaddressindex_to_p2wshbytes,
)?, p2traddressindex_to_p2trbytes,
height_to_first_p2shaddressindex: PcoVec::forced_import( p2aaddressindex_to_p2abytes,
db,
"first_p2shaddressindex",
version,
)?,
height_to_first_p2wpkhaddressindex: PcoVec::forced_import(
db,
"first_p2wpkhaddressindex",
version,
)?,
height_to_first_p2wshaddressindex: PcoVec::forced_import(
db,
"first_p2wshaddressindex",
version,
)?,
height_to_first_p2traddressindex: PcoVec::forced_import(
db,
"first_p2traddressindex",
version,
)?,
height_to_first_p2aaddressindex: PcoVec::forced_import(
db,
"first_p2aaddressindex",
version,
)?,
p2pk65addressindex_to_p2pk65bytes: BytesVec::forced_import(db, "p2pk65bytes", version)?,
p2pk33addressindex_to_p2pk33bytes: BytesVec::forced_import(db, "p2pk33bytes", version)?,
p2pkhaddressindex_to_p2pkhbytes: BytesVec::forced_import(db, "p2pkhbytes", version)?,
p2shaddressindex_to_p2shbytes: BytesVec::forced_import(db, "p2shbytes", version)?,
p2wpkhaddressindex_to_p2wpkhbytes: BytesVec::forced_import(db, "p2wpkhbytes", version)?,
p2wshaddressindex_to_p2wshbytes: BytesVec::forced_import(db, "p2wshbytes", version)?,
p2traddressindex_to_p2trbytes: BytesVec::forced_import(db, "p2trbytes", version)?,
p2aaddressindex_to_p2abytes: BytesVec::forced_import(db, "p2abytes", version)?,
}) })
} }

View File

@@ -4,6 +4,8 @@ use brk_types::{BlockHash, Height, StoredF64, StoredU64, Timestamp, Version, Wei
use rayon::prelude::*; use rayon::prelude::*;
use vecdb::{AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp}; use vecdb::{AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
use crate::parallel_import;
#[derive(Clone, Traversable)] #[derive(Clone, Traversable)]
pub struct BlockVecs { pub struct BlockVecs {
pub height_to_blockhash: BytesVec<Height, BlockHash>, pub height_to_blockhash: BytesVec<Height, BlockHash>,
@@ -16,12 +18,25 @@ pub struct BlockVecs {
impl BlockVecs { impl BlockVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> { pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
height_to_blockhash,
height_to_difficulty,
height_to_timestamp,
height_to_total_size,
height_to_weight,
) = parallel_import! {
height_to_blockhash = BytesVec::forced_import(db, "blockhash", version),
height_to_difficulty = PcoVec::forced_import(db, "difficulty", version),
height_to_timestamp = PcoVec::forced_import(db, "timestamp", version),
height_to_total_size = PcoVec::forced_import(db, "total_size", version),
height_to_weight = PcoVec::forced_import(db, "weight", version),
};
Ok(Self { Ok(Self {
height_to_blockhash: BytesVec::forced_import(db, "blockhash", version)?, height_to_blockhash,
height_to_difficulty: PcoVec::forced_import(db, "difficulty", version)?, height_to_difficulty,
height_to_timestamp: PcoVec::forced_import(db, "timestamp", version)?, height_to_timestamp,
height_to_total_size: PcoVec::forced_import(db, "total_size", version)?, height_to_total_size,
height_to_weight: PcoVec::forced_import(db, "weight", version)?, height_to_weight,
}) })
} }

View File

@@ -0,0 +1,20 @@
/// Imports multiple items in parallel using thread::scope.
/// Each expression must return Result<T>.
///
/// # Example
/// ```ignore
/// let (a, b, c) = parallel_import! {
/// a = SomeVec::forced_import(&db, version),
/// b = OtherVec::forced_import(&db, version),
/// c = ThirdVec::forced_import(&db, version),
/// };
/// ```
#[macro_export]
macro_rules! parallel_import {
($($name:ident = $expr:expr),+ $(,)?) => {{
std::thread::scope(|s| -> brk_error::Result<_> {
$(let $name = s.spawn(|| $expr);)+
Ok(($($name.join().unwrap()?,)+))
})?
}};
}

View File

@@ -6,8 +6,11 @@ use brk_types::{AddressBytes, AddressHash, Height, OutputType, TypeIndex, Versio
use rayon::prelude::*; use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, PAGE_SIZE, Reader, Stamp}; use vecdb::{AnyStoredVec, Database, PAGE_SIZE, Reader, Stamp};
use crate::parallel_import;
mod address; mod address;
mod blocks; mod blocks;
mod macros;
mod output; mod output;
mod tx; mod tx;
mod txin; mod txin;
@@ -35,15 +38,51 @@ pub struct Vecs {
impl Vecs { impl Vecs {
pub fn forced_import(parent: &Path, version: Version) -> Result<Self> { pub fn forced_import(parent: &Path, version: Version) -> Result<Self> {
log::debug!("Opening vecs database...");
let db = Database::open(&parent.join("vecs"))?; let db = Database::open(&parent.join("vecs"))?;
log::debug!("Setting min len...");
db.set_min_len(PAGE_SIZE * 50_000_000)?; db.set_min_len(PAGE_SIZE * 50_000_000)?;
let block = BlockVecs::forced_import(&db, version)?; log::debug!("Importing sub-vecs in parallel...");
let tx = TxVecs::forced_import(&db, version)?; let (block, tx, txin, txout, address, output) = parallel_import! {
let txin = TxinVecs::forced_import(&db, version)?; block = {
let txout = TxoutVecs::forced_import(&db, version)?; log::debug!("Importing BlockVecs...");
let address = AddressVecs::forced_import(&db, version)?; let r = BlockVecs::forced_import(&db, version);
let output = OutputVecs::forced_import(&db, version)?; log::debug!("BlockVecs imported.");
r
},
tx = {
log::debug!("Importing TxVecs...");
let r = TxVecs::forced_import(&db, version);
log::debug!("TxVecs imported.");
r
},
txin = {
log::debug!("Importing TxinVecs...");
let r = TxinVecs::forced_import(&db, version);
log::debug!("TxinVecs imported.");
r
},
txout = {
log::debug!("Importing TxoutVecs...");
let r = TxoutVecs::forced_import(&db, version);
log::debug!("TxoutVecs imported.");
r
},
address = {
log::debug!("Importing AddressVecs...");
let r = AddressVecs::forced_import(&db, version);
log::debug!("AddressVecs imported.");
r
},
output = {
log::debug!("Importing OutputVecs...");
let r = OutputVecs::forced_import(&db, version);
log::debug!("OutputVecs imported.");
r
},
};
log::debug!("Sub-vecs imported.");
let this = Self { let this = Self {
db, db,
@@ -55,13 +94,16 @@ impl Vecs {
output, output,
}; };
log::debug!("Retaining regions...");
this.db.retain_regions( this.db.retain_regions(
this.iter_any_exportable() this.iter_any_exportable()
.flat_map(|v| v.region_names()) .flat_map(|v| v.region_names())
.collect(), .collect(),
)?; )?;
log::debug!("Compacting database...");
this.db.compact()?; this.db.compact()?;
log::debug!("Vecs import complete.");
Ok(this) Ok(this)
} }

View File

@@ -6,6 +6,8 @@ use brk_types::{
use rayon::prelude::*; use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp}; use vecdb::{AnyStoredVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
use crate::parallel_import;
#[derive(Clone, Traversable)] #[derive(Clone, Traversable)]
pub struct OutputVecs { pub struct OutputVecs {
// Height to first output index (per output type) // Height to first output index (per output type)
@@ -22,31 +24,34 @@ pub struct OutputVecs {
impl OutputVecs { impl OutputVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> { pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
height_to_first_emptyoutputindex,
height_to_first_opreturnindex,
height_to_first_p2msoutputindex,
height_to_first_unknownoutputindex,
emptyoutputindex_to_txindex,
opreturnindex_to_txindex,
p2msoutputindex_to_txindex,
unknownoutputindex_to_txindex,
) = parallel_import! {
height_to_first_emptyoutputindex = PcoVec::forced_import(db, "first_emptyoutputindex", version),
height_to_first_opreturnindex = PcoVec::forced_import(db, "first_opreturnindex", version),
height_to_first_p2msoutputindex = PcoVec::forced_import(db, "first_p2msoutputindex", version),
height_to_first_unknownoutputindex = PcoVec::forced_import(db, "first_unknownoutputindex", version),
emptyoutputindex_to_txindex = PcoVec::forced_import(db, "txindex", version),
opreturnindex_to_txindex = PcoVec::forced_import(db, "txindex", version),
p2msoutputindex_to_txindex = PcoVec::forced_import(db, "txindex", version),
unknownoutputindex_to_txindex = PcoVec::forced_import(db, "txindex", version),
};
Ok(Self { Ok(Self {
height_to_first_emptyoutputindex: PcoVec::forced_import( height_to_first_emptyoutputindex,
db, height_to_first_opreturnindex,
"first_emptyoutputindex", height_to_first_p2msoutputindex,
version, height_to_first_unknownoutputindex,
)?, emptyoutputindex_to_txindex,
height_to_first_opreturnindex: PcoVec::forced_import( opreturnindex_to_txindex,
db, p2msoutputindex_to_txindex,
"first_opreturnindex", unknownoutputindex_to_txindex,
version,
)?,
height_to_first_p2msoutputindex: PcoVec::forced_import(
db,
"first_p2msoutputindex",
version,
)?,
height_to_first_unknownoutputindex: PcoVec::forced_import(
db,
"first_unknownoutputindex",
version,
)?,
emptyoutputindex_to_txindex: PcoVec::forced_import(db, "txindex", version)?,
opreturnindex_to_txindex: PcoVec::forced_import(db, "txindex", version)?,
p2msoutputindex_to_txindex: PcoVec::forced_import(db, "txindex", version)?,
unknownoutputindex_to_txindex: PcoVec::forced_import(db, "txindex", version)?,
}) })
} }

View File

@@ -7,6 +7,8 @@ use brk_types::{
use rayon::prelude::*; use rayon::prelude::*;
use vecdb::{AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp}; use vecdb::{AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
use crate::parallel_import;
#[derive(Clone, Traversable)] #[derive(Clone, Traversable)]
pub struct TxVecs { pub struct TxVecs {
pub height_to_first_txindex: PcoVec<Height, TxIndex>, pub height_to_first_txindex: PcoVec<Height, TxIndex>,
@@ -23,17 +25,40 @@ pub struct TxVecs {
impl TxVecs { impl TxVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> { pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
height_to_first_txindex,
txindex_to_height,
txindex_to_txid,
txindex_to_txversion,
txindex_to_rawlocktime,
txindex_to_base_size,
txindex_to_total_size,
txindex_to_is_explicitly_rbf,
txindex_to_first_txinindex,
txindex_to_first_txoutindex,
) = parallel_import! {
height_to_first_txindex = PcoVec::forced_import(db, "first_txindex", version),
txindex_to_height = PcoVec::forced_import(db, "height", version),
txindex_to_txid = BytesVec::forced_import(db, "txid", version),
txindex_to_txversion = PcoVec::forced_import(db, "txversion", version),
txindex_to_rawlocktime = PcoVec::forced_import(db, "rawlocktime", version),
txindex_to_base_size = PcoVec::forced_import(db, "base_size", version),
txindex_to_total_size = PcoVec::forced_import(db, "total_size", version),
txindex_to_is_explicitly_rbf = PcoVec::forced_import(db, "is_explicitly_rbf", version),
txindex_to_first_txinindex = PcoVec::forced_import(db, "first_txinindex", version),
txindex_to_first_txoutindex = BytesVec::forced_import(db, "first_txoutindex", version),
};
Ok(Self { Ok(Self {
height_to_first_txindex: PcoVec::forced_import(db, "first_txindex", version)?, height_to_first_txindex,
txindex_to_height: PcoVec::forced_import(db, "height", version)?, txindex_to_height,
txindex_to_txid: BytesVec::forced_import(db, "txid", version)?, txindex_to_txid,
txindex_to_txversion: PcoVec::forced_import(db, "txversion", version)?, txindex_to_txversion,
txindex_to_rawlocktime: PcoVec::forced_import(db, "rawlocktime", version)?, txindex_to_rawlocktime,
txindex_to_base_size: PcoVec::forced_import(db, "base_size", version)?, txindex_to_base_size,
txindex_to_total_size: PcoVec::forced_import(db, "total_size", version)?, txindex_to_total_size,
txindex_to_is_explicitly_rbf: PcoVec::forced_import(db, "is_explicitly_rbf", version)?, txindex_to_is_explicitly_rbf,
txindex_to_first_txinindex: PcoVec::forced_import(db, "first_txinindex", version)?, txindex_to_first_txinindex,
txindex_to_first_txoutindex: BytesVec::forced_import(db, "first_txoutindex", version)?, txindex_to_first_txoutindex,
}) })
} }

View File

@@ -1,22 +1,49 @@
use brk_error::Result; use brk_error::Result;
use brk_traversable::Traversable; use brk_traversable::Traversable;
use brk_types::{Height, OutPoint, TxInIndex, TxIndex, Version}; use brk_types::{Height, OutPoint, OutputType, Sats, TxInIndex, TxIndex, TypeIndex, Version};
use rayon::prelude::*; use rayon::prelude::*;
use vecdb::{AnyStoredVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp}; use vecdb::{AnyStoredVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp};
use crate::parallel_import;
#[derive(Clone, Traversable)] #[derive(Clone, Traversable)]
pub struct TxinVecs { pub struct TxinVecs {
pub height_to_first_txinindex: PcoVec<Height, TxInIndex>, pub height_to_first_txinindex: PcoVec<Height, TxInIndex>,
pub txinindex_to_outpoint: PcoVec<TxInIndex, OutPoint>, pub txinindex_to_outpoint: PcoVec<TxInIndex, OutPoint>,
pub txinindex_to_txindex: PcoVec<TxInIndex, TxIndex>, pub txinindex_to_txindex: PcoVec<TxInIndex, TxIndex>,
pub txinindex_to_value: PcoVec<TxInIndex, Sats>,
pub txinindex_to_prev_height: PcoVec<TxInIndex, Height>,
pub txinindex_to_outputtype: PcoVec<TxInIndex, OutputType>,
pub txinindex_to_typeindex: PcoVec<TxInIndex, TypeIndex>,
} }
impl TxinVecs { impl TxinVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> { pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
height_to_first_txinindex,
txinindex_to_outpoint,
txinindex_to_txindex,
txinindex_to_value,
txinindex_to_prev_height,
txinindex_to_outputtype,
txinindex_to_typeindex,
) = parallel_import! {
height_to_first_txinindex = PcoVec::forced_import(db, "first_txinindex", version),
txinindex_to_outpoint = PcoVec::forced_import(db, "outpoint", version),
txinindex_to_txindex = PcoVec::forced_import(db, "txindex", version),
txinindex_to_value = PcoVec::forced_import(db, "value", version),
txinindex_to_prev_height = PcoVec::forced_import(db, "prev_height", version),
txinindex_to_outputtype = PcoVec::forced_import(db, "outputtype", version),
txinindex_to_typeindex = PcoVec::forced_import(db, "typeindex", version),
};
Ok(Self { Ok(Self {
height_to_first_txinindex: PcoVec::forced_import(db, "first_txinindex", version)?, height_to_first_txinindex,
txinindex_to_outpoint: PcoVec::forced_import(db, "outpoint", version)?, txinindex_to_outpoint,
txinindex_to_txindex: PcoVec::forced_import(db, "txindex", version)?, txinindex_to_txindex,
txinindex_to_value,
txinindex_to_prev_height,
txinindex_to_outputtype,
txinindex_to_typeindex,
}) })
} }
@@ -27,6 +54,14 @@ impl TxinVecs {
.truncate_if_needed_with_stamp(txinindex, stamp)?; .truncate_if_needed_with_stamp(txinindex, stamp)?;
self.txinindex_to_txindex self.txinindex_to_txindex
.truncate_if_needed_with_stamp(txinindex, stamp)?; .truncate_if_needed_with_stamp(txinindex, stamp)?;
self.txinindex_to_value
.truncate_if_needed_with_stamp(txinindex, stamp)?;
self.txinindex_to_prev_height
.truncate_if_needed_with_stamp(txinindex, stamp)?;
self.txinindex_to_outputtype
.truncate_if_needed_with_stamp(txinindex, stamp)?;
self.txinindex_to_typeindex
.truncate_if_needed_with_stamp(txinindex, stamp)?;
Ok(()) Ok(())
} }
@@ -35,6 +70,10 @@ impl TxinVecs {
&mut self.height_to_first_txinindex as &mut dyn AnyStoredVec, &mut self.height_to_first_txinindex as &mut dyn AnyStoredVec,
&mut self.txinindex_to_outpoint, &mut self.txinindex_to_outpoint,
&mut self.txinindex_to_txindex, &mut self.txinindex_to_txindex,
&mut self.txinindex_to_value,
&mut self.txinindex_to_prev_height,
&mut self.txinindex_to_outputtype,
&mut self.txinindex_to_typeindex,
] ]
.into_par_iter() .into_par_iter()
} }

View File

@@ -1,50 +1,69 @@
use brk_error::Result; use brk_error::Result;
use brk_traversable::Traversable; use brk_traversable::Traversable;
use brk_types::{Height, OutputType, Sats, TxIndex, TxOutIndex, TypeIndex, Version}; use brk_types::{Height, Sats, TxInIndex, TxIndex, TxOutData, TxOutIndex, Version};
use rayon::prelude::*; use rayon::prelude::*;
use vecdb::{AnyStoredVec, BytesVec, Database, GenericStoredVec, ImportableVec, PcoVec, Stamp}; use vecdb::{
AnyStoredVec, AnyVec, BytesVec, Database, GenericStoredVec, ImportableVec, IterableCloneableVec,
LazyVecFrom1, PcoVec, Stamp,
};
use crate::parallel_import;
#[derive(Clone, Traversable)] #[derive(Clone, Traversable)]
pub struct TxoutVecs { pub struct TxoutVecs {
pub height_to_first_txoutindex: PcoVec<Height, TxOutIndex>, pub height_to_first_txoutindex: PcoVec<Height, TxOutIndex>,
pub txoutindex_to_value: BytesVec<TxOutIndex, Sats>, pub txoutindex_to_txoutdata: BytesVec<TxOutIndex, TxOutData>,
pub txoutindex_to_outputtype: BytesVec<TxOutIndex, OutputType>,
pub txoutindex_to_typeindex: BytesVec<TxOutIndex, TypeIndex>,
pub txoutindex_to_txindex: PcoVec<TxOutIndex, TxIndex>, pub txoutindex_to_txindex: PcoVec<TxOutIndex, TxIndex>,
pub txoutindex_to_txinindex: BytesVec<TxOutIndex, TxInIndex>,
pub txoutindex_to_value: LazyVecFrom1<TxOutIndex, Sats, TxOutIndex, TxOutData>,
} }
impl TxoutVecs { impl TxoutVecs {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> { pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
let (
height_to_first_txoutindex,
txoutindex_to_txoutdata,
txoutindex_to_txindex,
txoutindex_to_txinindex,
) = parallel_import! {
height_to_first_txoutindex = PcoVec::forced_import(db, "first_txoutindex", version),
txoutindex_to_txoutdata = BytesVec::forced_import(db, "txoutdata", version),
txoutindex_to_txindex = PcoVec::forced_import(db, "txindex", version),
txoutindex_to_txinindex = BytesVec::forced_import(db, "txinindex", version),
};
let txoutindex_to_value = LazyVecFrom1::init(
"value",
txoutindex_to_txoutdata.version(),
txoutindex_to_txoutdata.boxed_clone(),
|index, iter| iter.get(index).map(|txoutdata: TxOutData| txoutdata.value),
);
Ok(Self { Ok(Self {
height_to_first_txoutindex: PcoVec::forced_import(db, "first_txoutindex", version)?, height_to_first_txoutindex,
txoutindex_to_value: BytesVec::forced_import(db, "value", version)?, txoutindex_to_txoutdata,
txoutindex_to_outputtype: BytesVec::forced_import(db, "outputtype", version)?, txoutindex_to_txindex,
txoutindex_to_typeindex: BytesVec::forced_import(db, "typeindex", version)?, txoutindex_to_txinindex,
txoutindex_to_txindex: PcoVec::forced_import(db, "txindex", version)?, txoutindex_to_value,
}) })
} }
pub fn truncate(&mut self, height: Height, txoutindex: TxOutIndex, stamp: Stamp) -> Result<()> { pub fn truncate(&mut self, height: Height, txoutindex: TxOutIndex, stamp: Stamp) -> Result<()> {
self.height_to_first_txoutindex self.height_to_first_txoutindex
.truncate_if_needed_with_stamp(height, stamp)?; .truncate_if_needed_with_stamp(height, stamp)?;
self.txoutindex_to_value self.txoutindex_to_txoutdata
.truncate_if_needed_with_stamp(txoutindex, stamp)?;
self.txoutindex_to_outputtype
.truncate_if_needed_with_stamp(txoutindex, stamp)?;
self.txoutindex_to_typeindex
.truncate_if_needed_with_stamp(txoutindex, stamp)?; .truncate_if_needed_with_stamp(txoutindex, stamp)?;
self.txoutindex_to_txindex self.txoutindex_to_txindex
.truncate_if_needed_with_stamp(txoutindex, stamp)?; .truncate_if_needed_with_stamp(txoutindex, stamp)?;
self.txoutindex_to_txinindex
.truncate_if_needed_with_stamp(txoutindex, stamp)?;
Ok(()) Ok(())
} }
pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> { pub fn par_iter_mut_any(&mut self) -> impl ParallelIterator<Item = &mut dyn AnyStoredVec> {
[ [
&mut self.height_to_first_txoutindex as &mut dyn AnyStoredVec, &mut self.height_to_first_txoutindex as &mut dyn AnyStoredVec,
&mut self.txoutindex_to_value, &mut self.txoutindex_to_txoutdata,
&mut self.txoutindex_to_outputtype,
&mut self.txoutindex_to_typeindex,
&mut self.txoutindex_to_txindex, &mut self.txoutindex_to_txindex,
&mut self.txoutindex_to_txinindex,
] ]
.into_par_iter() .into_par_iter()
} }

View File

@@ -9,14 +9,15 @@ repository.workspace = true
build = "build.rs" build = "build.rs"
[dependencies] [dependencies]
aide = { workspace = true } axum = { workspace = true }
brk_query = { workspace = true }
brk_rmcp = { version = "0.8.0", features = [ brk_rmcp = { version = "0.8.0", features = [
"transport-worker", "transport-worker",
"transport-streamable-http-server", "transport-streamable-http-server",
] } ] }
brk_types = { workspace = true }
log = { workspace = true } log = { workspace = true }
minreq = { workspace = true }
schemars = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
[package.metadata.cargo-machete] [package.metadata.cargo-machete]

View File

@@ -4,42 +4,31 @@ Model Context Protocol (MCP) server for Bitcoin on-chain data.
## What It Enables ## What It Enables
Expose BRK's query capabilities to AI assistants via the MCP standard. LLMs can browse metrics, fetch datasets, and analyze on-chain data through structured tool calls. Expose BRK's REST API to AI assistants via MCP. The LLM reads the OpenAPI spec and calls any endpoint through a generic fetch tool.
## Key Features
- **Tool-based API**: 8 tools for metric discovery and data retrieval
- **Pagination support**: Browse large metric catalogs in chunks
- **Self-documenting**: Built-in instructions explain available capabilities
- **Async**: Full tokio integration via `AsyncQuery`
## Available Tools ## Available Tools
| Tool | Description | | Tool | Description |
|------|-------------| |------|-------------|
| `get_metric_count` | Count of unique metrics | | `get_openapi` | Get the OpenAPI specification for all REST endpoints |
| `get_vec_count` | Total metric × index combinations | | `fetch` | Call any REST API endpoint by path and query |
| `get_indexes` | List all index types and variants |
| `get_vecids` | Paginated list of metric IDs | ## Workflow
| `get_index_to_vecids` | Metrics supporting a given index |
| `get_vecid_to_indexes` | Indexes supported by a metric | 1. LLM calls `get_openapi` to discover available endpoints
| `get_vecs` | Fetch metric data with range selection | 2. LLM calls `fetch` with the desired path and query parameters
| `get_version` | BRK version string |
## Usage ## Usage
```rust,ignore ```rust,ignore
let mcp = MCP::new(&async_query); let mcp = MCP::new("http://127.0.0.1:3110", openapi_json);
// The MCP server implements ServerHandler for use with rmcp
// Tools are auto-registered via the #[tool_router] macro
``` ```
## Integration ## Integration
The MCP server is integrated into `brk_server` and exposed at `/mcp` endpoint for MCP transport. The MCP server is integrated into `brk_server` and exposed at `/mcp` endpoint.
## Built On ## Built On
- `brk_query` for data access
- `brk_rmcp` for MCP protocol implementation - `brk_rmcp` for MCP protocol implementation
- `minreq` for HTTP requests

View File

@@ -1,6 +1,7 @@
#![doc = include_str!("../README.md")] #![doc = include_str!("../README.md")]
use brk_query::{AsyncQuery, MetricSelection, Pagination, PaginationIndex}; use std::sync::Arc;
use brk_rmcp::{ use brk_rmcp::{
ErrorData as McpError, RoleServer, ServerHandler, ErrorData as McpError, RoleServer, ServerHandler,
handler::server::{router::tool::ToolRouter, wrapper::Parameters}, handler::server::{router::tool::ToolRouter, wrapper::Parameters},
@@ -8,135 +9,68 @@ use brk_rmcp::{
service::RequestContext, service::RequestContext,
tool, tool_handler, tool_router, tool, tool_handler, tool_router,
}; };
use brk_types::Metric;
use log::info; use log::info;
use schemars::JsonSchema;
use serde::Deserialize;
pub mod route; pub mod route;
#[derive(Clone)] #[derive(Clone)]
pub struct MCP { pub struct MCP {
query: AsyncQuery, base_url: Arc<String>,
openapi_json: Arc<String>,
tool_router: ToolRouter<MCP>, tool_router: ToolRouter<MCP>,
} }
const VERSION: &str = env!("CARGO_PKG_VERSION"); /// Parameters for fetching from the REST API.
#[derive(Deserialize, JsonSchema)]
pub struct FetchParams {
/// API path (e.g., "/api/blocks" or "/api/metrics/list")
pub path: String,
/// Optional query string (e.g., "page=0" or "from=-1&to=-10")
pub query: Option<String>,
}
#[tool_router] #[tool_router]
impl MCP { impl MCP {
pub fn new(query: &AsyncQuery) -> Self { pub fn new(base_url: impl Into<String>, openapi_json: impl Into<String>) -> Self {
Self { Self {
query: query.clone(), base_url: Arc::new(base_url.into()),
openapi_json: Arc::new(openapi_json.into()),
tool_router: Self::tool_router(), tool_router: Self::tool_router(),
} }
} }
#[tool(description = " #[tool(description = "Get the OpenAPI specification describing all available REST API endpoints.")]
Get the count of unique metrics. async fn get_openapi(&self) -> Result<CallToolResult, McpError> {
")] info!("mcp: get_openapi");
async fn get_metric_count(&self) -> Result<CallToolResult, McpError> {
info!("mcp: distinct_metric_count");
Ok(CallToolResult::success(vec![
Content::json(self.query.sync(|q| q.distinct_metric_count())).unwrap(),
]))
}
#[tool(description = "
Get the count of all metrics. (distinct metrics multiplied by the number of indexes supported by each one)
")]
async fn get_vec_count(&self) -> Result<CallToolResult, McpError> {
info!("mcp: total_metric_count");
Ok(CallToolResult::success(vec![
Content::json(self.query.sync(|q| q.total_metric_count())).unwrap(),
]))
}
#[tool(description = "
Get the list of all existing indexes and their accepted variants.
")]
async fn get_indexes(&self) -> Result<CallToolResult, McpError> {
info!("mcp: get_indexes");
Ok(CallToolResult::success(vec![
Content::json(self.query.inner().indexes()).unwrap(),
]))
}
#[tool(description = "
Get a paginated list of all existing vec ids.
There are up to 1,000 values per page.
If the `page` param is omitted, it will default to the first page.
")]
async fn get_vecids(
&self,
Parameters(pagination): Parameters<Pagination>,
) -> Result<CallToolResult, McpError> {
info!("mcp: get_metrics");
Ok(CallToolResult::success(vec![
Content::json(self.query.sync(|q| q.metrics(pagination))).unwrap(),
]))
}
#[tool(description = "
Get a paginated list of all vec ids which support a given index.
There are up to 1,000 values per page.
If the `page` param is omitted, it will default to the first page.
")]
async fn get_index_to_vecids(
&self,
Parameters(paginated_index): Parameters<PaginationIndex>,
) -> Result<CallToolResult, McpError> {
info!("mcp: get_index_to_vecids");
let result = self
.query
.inner()
.index_to_vecids(paginated_index)
.unwrap_or_default();
Ok(CallToolResult::success(vec![
Content::json(result).unwrap(),
]))
}
#[tool(description = "
Get a list of all indexes supported by a given vec id.
The list will be empty if the vec id isn't correct.
")]
async fn get_vecid_to_indexes(
&self,
Parameters(metric): Parameters<Metric>,
) -> Result<CallToolResult, McpError> {
info!("mcp: get_vecid_to_indexes");
Ok(CallToolResult::success(vec![
Content::json(self.query.inner().metric_to_indexes(metric)).unwrap(),
]))
}
#[tool(description = "
Get one or multiple vecs depending on given parameters.
The response's format will depend on the given parameters, it will be:
- A value: If requested only one vec and the given range returns one value (for example: `from=-1`)
- A list: If requested only one vec and the given range returns multiple values (for example: `from=-1000&count=100` or `from=-444&to=-333`)
- A matrix: When multiple vecs are requested, even if they each return one value.
")]
async fn get_vecs(
&self,
Parameters(params): Parameters<MetricSelection>,
) -> Result<CallToolResult, McpError> {
info!("mcp: get_vecs");
Ok(CallToolResult::success(vec![Content::text( Ok(CallToolResult::success(vec![Content::text(
match self.query.run(move |q| q.search_and_format_legacy(params)).await { self.openapi_json.as_str(),
Ok(output) => output.to_string(),
Err(e) => format!("Error:\n{e}"),
},
)])) )]))
} }
#[tool(description = " #[tool(description = "Call a REST API endpoint. Use get_openapi first to discover available endpoints.")]
Get the running version of the Bitcoin Research Kit. async fn fetch(
")] &self,
async fn get_version(&self) -> Result<CallToolResult, McpError> { Parameters(params): Parameters<FetchParams>,
info!("mcp: get_version"); ) -> Result<CallToolResult, McpError> {
Ok(CallToolResult::success(vec![Content::text(format!( info!("mcp: fetch {}", params.path);
"v{VERSION}"
))])) let url = match &params.query {
Some(q) if !q.is_empty() => format!("{}{}?{}", self.base_url, params.path, q),
_ => format!("{}{}", self.base_url, params.path),
};
match minreq::get(&url).send() {
Ok(response) => {
let body = response.as_str().unwrap_or("").to_string();
Ok(CallToolResult::success(vec![Content::text(body)]))
}
Err(e) => Err(McpError::internal_error(
format!("HTTP request failed: {e}"),
None,
)),
}
} }
} }
@@ -149,17 +83,13 @@ impl ServerHandler for MCP {
server_info: Implementation::from_build_env(), server_info: Implementation::from_build_env(),
instructions: Some( instructions: Some(
" "
This server provides an interface to communicate with a running instance of the Bitcoin Research Kit (also called brk or BRK). Bitcoin Research Kit (BRK) - Bitcoin on-chain metrics and market data.
Multiple tools are at your disposal including ones to fetch all sorts of Bitcoin on-chain metrics and market prices. Workflow:
1. Call get_openapi to get the full API specification
2. Use fetch to call any endpoint described in the spec
If you're unsure which datasets are available, try out different tools before browsing the web. Each tool gives important information about BRK's capabilities. Example: fetch with path=\"/api/metrics/list\" to list metrics.
Vectors can also be called 'Vecs', 'Arrays' or 'Datasets', they can all be used interchangeably.
An 'Index' (or indexes) is the timeframe of a dataset.
'VecId' (or vecids) are the name of the dataset and what it represents.
" "
.to_string(), .to_string(),
), ),

View File

@@ -1,39 +1,26 @@
use aide::axum::ApiRouter; use std::sync::Arc;
use brk_query::AsyncQuery;
use axum::Router;
use brk_rmcp::transport::{ use brk_rmcp::transport::{
StreamableHttpServerConfig, StreamableHttpServerConfig,
streamable_http_server::{StreamableHttpService, session::local::LocalSessionManager}, streamable_http_server::{StreamableHttpService, session::local::LocalSessionManager},
}; };
use log::info; use log::info;
use crate::MCP; use crate::MCP;
pub trait MCPRoutes { /// Create an MCP service router.
fn add_mcp_routes(self, query: &AsyncQuery, mcp: bool) -> Self; pub fn mcp_router(base_url: String, openapi_json: Arc<String>) -> Router {
} info!("Setting up MCP...");
impl<T> MCPRoutes for ApiRouter<T> let service = StreamableHttpService::new(
where move || Ok(MCP::new(base_url.clone(), openapi_json.as_str())),
T: Clone + Send + Sync + 'static, LocalSessionManager::default().into(),
{ StreamableHttpServerConfig {
fn add_mcp_routes(self, query: &AsyncQuery, mcp: bool) -> Self { stateful_mode: false,
if !mcp { ..Default::default()
return self; },
} );
let query = query.clone(); Router::new().nest_service("/mcp", service)
let service = StreamableHttpService::new(
move || Ok(MCP::new(&query)),
LocalSessionManager::default().into(),
StreamableHttpServerConfig {
stateful_mode: false,
..Default::default()
},
);
info!("Setting MCP...");
self.nest_service("/mcp", service)
}
} }

View File

@@ -7,7 +7,7 @@ use brk_types::{
AddressIndexTxIndex, AddressStats, AnyAddressDataIndexEnum, OutputType, Sats, TxIndex, AddressIndexTxIndex, AddressStats, AnyAddressDataIndexEnum, OutputType, Sats, TxIndex,
TxStatus, Txid, TypeIndex, Unit, Utxo, Vout, TxStatus, Txid, TypeIndex, Unit, Utxo, Vout,
}; };
use vecdb::TypedVecIterator; use vecdb::{IterableVec, TypedVecIterator};
use crate::Query; use crate::Query;
@@ -169,7 +169,7 @@ impl Query {
let mut txindex_to_txid_iter = vecs.tx.txindex_to_txid.iter()?; let mut txindex_to_txid_iter = vecs.tx.txindex_to_txid.iter()?;
let mut txindex_to_height_iter = vecs.tx.txindex_to_height.iter()?; let mut txindex_to_height_iter = vecs.tx.txindex_to_height.iter()?;
let mut txindex_to_first_txoutindex_iter = vecs.tx.txindex_to_first_txoutindex.iter()?; let mut txindex_to_first_txoutindex_iter = vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut txoutindex_to_value_iter = vecs.txout.txoutindex_to_value.iter()?; let mut txoutindex_to_value_iter = vecs.txout.txoutindex_to_value.iter();
let mut height_to_blockhash_iter = vecs.block.height_to_blockhash.iter()?; let mut height_to_blockhash_iter = vecs.block.height_to_blockhash.iter()?;
let mut height_to_timestamp_iter = vecs.block.height_to_timestamp.iter()?; let mut height_to_timestamp_iter = vecs.block.height_to_timestamp.iter()?;

View File

@@ -6,7 +6,7 @@ use brk_types::{
Sats, Transaction, TxIn, TxInIndex, TxIndex, TxOut, TxOutspend, TxStatus, Txid, TxidParam, Sats, Transaction, TxIn, TxInIndex, TxIndex, TxOut, TxOutspend, TxStatus, Txid, TxidParam,
TxidPrefix, Vin, Vout, Weight, TxidPrefix, Vin, Vout, Weight,
}; };
use vecdb::{GenericStoredVec, TypedVecIterator}; use vecdb::{GenericStoredVec, IterableVec, TypedVecIterator};
use crate::Query; use crate::Query;
@@ -119,9 +119,10 @@ impl Query {
let txoutindex = first_txoutindex + vout; let txoutindex = first_txoutindex + vout;
// Look up spend status // Look up spend status
let computer = self.computer(); let indexer = self.indexer();
let txinindex = computer let txinindex = indexer
.stateful .vecs
.txout
.txoutindex_to_txinindex .txoutindex_to_txinindex
.read_once(txoutindex)?; .read_once(txoutindex)?;
@@ -167,8 +168,7 @@ impl Query {
let output_count = usize::from(next_first_txoutindex) - usize::from(first_txoutindex); let output_count = usize::from(next_first_txoutindex) - usize::from(first_txoutindex);
// Get spend status for each output // Get spend status for each output
let computer = self.computer(); let mut txoutindex_to_txinindex_iter = indexer.vecs.txout.txoutindex_to_txinindex.iter()?;
let mut txoutindex_to_txinindex_iter = computer.stateful.txoutindex_to_txinindex.iter()?;
let mut outspends = Vec::with_capacity(output_count); let mut outspends = Vec::with_capacity(output_count);
for i in 0..output_count { for i in 0..output_count {
@@ -220,7 +220,7 @@ impl Query {
let mut txindex_to_first_txoutindex_iter = let mut txindex_to_first_txoutindex_iter =
indexer.vecs.tx.txindex_to_first_txoutindex.iter()?; indexer.vecs.tx.txindex_to_first_txoutindex.iter()?;
let mut txinindex_to_outpoint_iter = indexer.vecs.txin.txinindex_to_outpoint.iter()?; let mut txinindex_to_outpoint_iter = indexer.vecs.txin.txinindex_to_outpoint.iter()?;
let mut txoutindex_to_value_iter = indexer.vecs.txout.txoutindex_to_value.iter()?; let mut txoutindex_to_value_iter = indexer.vecs.txout.txoutindex_to_value.iter();
// Build inputs with prevout information // Build inputs with prevout information
let input: Vec<TxIn> = tx let input: Vec<TxIn> = tx

View File

@@ -14,7 +14,7 @@ use axum::{
}; };
use brk_error::Result; use brk_error::Result;
use brk_logger::OwoColorize; use brk_logger::OwoColorize;
use brk_mcp::route::MCPRoutes; use brk_mcp::route::mcp_router;
use brk_query::AsyncQuery; use brk_query::AsyncQuery;
use log::{error, info}; use log::{error, info};
use quick_cache::sync::Cache; use quick_cache::sync::Cache;
@@ -92,7 +92,6 @@ impl Server {
let vecs = state.query.inner().vecs(); let vecs = state.query.inner().vecs();
let router = ApiRouter::new() let router = ApiRouter::new()
.add_api_routes() .add_api_routes()
.add_mcp_routes(&state.query, mcp)
.add_files_routes(state.path.as_ref()) .add_files_routes(state.path.as_ref())
.route( .route(
"/discord", "/discord",
@@ -136,24 +135,34 @@ impl Server {
let mut openapi = create_openapi(); let mut openapi = create_openapi();
let router = router.finish_api(&mut openapi); let router = router.finish_api(&mut openapi);
let clients_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) let workspace_root: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent() .parent()
.and_then(|p| p.parent())
.unwrap() .unwrap()
.join("brk_binder") .into();
.join("clients"); let output_paths = brk_binder::ClientOutputPaths::new()
if clients_path.exists() { .rust(workspace_root.join("crates/brk_client/src/lib.rs"))
let openapi_json = serde_json::to_string(&openapi).unwrap(); .javascript(workspace_root.join("modules/brk-client/index.js"))
let result = panic::catch_unwind(panic::AssertUnwindSafe(|| { .python(workspace_root.join("packages/brk_client/__init__.py"));
brk_binder::generate_clients(vecs, &openapi_json, &clients_path)
}));
match result { let openapi_json = Arc::new(serde_json::to_string(&openapi).unwrap());
Ok(Ok(())) => info!("Generated clients at {}", clients_path.display()), let result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
Ok(Err(e)) => error!("Failed to generate clients: {e}"), brk_binder::generate_clients(vecs, &openapi_json, &output_paths)
Err(_) => error!("Client generation panicked"), }));
}
match result {
Ok(Ok(())) => info!("Generated clients"),
Ok(Err(e)) => error!("Failed to generate clients: {e}"),
Err(_) => error!("Client generation panicked"),
} }
let router = if mcp {
let base_url = format!("http://127.0.0.1:{port}");
router.merge(mcp_router(base_url, openapi_json))
} else {
router
};
serve( serve(
listener, listener,
router router

View File

@@ -277,6 +277,7 @@ where
ingestion.write(ByteView::from(key), ByteView::from(value))?; ingestion.write(ByteView::from(key), ByteView::from(value))?;
} }
Item::Tomb(key) => { Item::Tomb(key) => {
// TODO: switch to write_weak_tombstone when lsm-tree ingestion API supports it
ingestion.write_tombstone(ByteView::from(key))?; ingestion.write_tombstone(ByteView::from(key))?;
} }
} }

View File

@@ -3,7 +3,7 @@ use std::{
path::{Path, PathBuf}, path::{Path, PathBuf},
}; };
use brk_error::Result; use brk_error::{Error, Result};
use brk_types::Version; use brk_types::Version;
use fjall::{Database, Keyspace}; use fjall::{Database, Keyspace};
@@ -30,16 +30,14 @@ impl StoreMeta {
let partition = open_partition_handle()?; let partition = open_partition_handle()?;
if Version::try_from(Self::path_version_(path).as_path()) if let Ok(prev_version) = Version::try_from(Self::path_version_(path).as_path())
.is_ok_and(|prev_version| version != prev_version) && version != prev_version
{ {
todo!(); return Err(Error::VersionMismatch {
// fs::remove_dir_all(path)?; path: path.to_path_buf(),
// // Doesn't exist expected: u64::from(version) as usize,
// // database.delete_partition(partition)?; found: u64::from(prev_version) as usize,
// fs::create_dir(path)?; });
// database.persist(PersistMode::SyncAll)?;
// partition = open_partition_handle()?;
} }
let slf = Self { let slf = Self {

View File

@@ -142,11 +142,10 @@ impl TryFrom<(&ScriptBuf, OutputType)> for AddressBytes {
let bytes = &script.as_bytes()[2..]; let bytes = &script.as_bytes()[2..];
Ok(Self::P2A(Box::new(P2ABytes::from(bytes)))) Ok(Self::P2A(Box::new(P2ABytes::from(bytes))))
} }
OutputType::P2MS => Err(Error::WrongAddressType), OutputType::P2MS
OutputType::Unknown => Err(Error::WrongAddressType), | OutputType::Unknown
OutputType::Empty => Err(Error::WrongAddressType), | OutputType::Empty
OutputType::OpReturn => Err(Error::WrongAddressType), | OutputType::OpReturn => Err(Error::WrongAddressType),
_ => unreachable!(),
} }
} }
} }

View File

@@ -138,6 +138,7 @@ mod txin;
mod txindex; mod txindex;
mod txinindex; mod txinindex;
mod txout; mod txout;
mod txoutdata;
mod txoutindex; mod txoutindex;
mod txoutspend; mod txoutspend;
mod txstatus; mod txstatus;
@@ -292,6 +293,7 @@ pub use txin::*;
pub use txindex::*; pub use txindex::*;
pub use txinindex::*; pub use txinindex::*;
pub use txout::*; pub use txout::*;
pub use txoutdata::*;
pub use txoutindex::*; pub use txoutindex::*;
pub use txoutspend::*; pub use txoutspend::*;
pub use txstatus::*; pub use txstatus::*;

View File

@@ -3,765 +3,35 @@ use brk_error::Error;
use schemars::JsonSchema; use schemars::JsonSchema;
use serde::Serialize; use serde::Serialize;
use strum::Display; use strum::Display;
use vecdb::{Bytes, Formattable}; use vecdb::{Bytes, Formattable, Pco, TransparentPco};
use crate::AddressBytes; use crate::AddressBytes;
#[derive( #[derive(Debug, Clone, Copy, Display, PartialEq, Eq, PartialOrd, Ord, Serialize, JsonSchema, Hash)]
Debug, Clone, Copy, Display, PartialEq, Eq, PartialOrd, Ord, Serialize, JsonSchema, Hash,
)]
#[serde(rename_all = "lowercase")] #[serde(rename_all = "lowercase")]
#[strum(serialize_all = "lowercase")] #[strum(serialize_all = "lowercase")]
#[repr(u8)] #[repr(u16)]
/// Type (P2PKH, P2WPKH, P2SH, P2TR, etc.) /// Type (P2PKH, P2WPKH, P2SH, P2TR, etc.)
pub enum OutputType { pub enum OutputType {
P2PK65, P2PK65 = 0,
P2PK33, P2PK33 = 1,
P2PKH, P2PKH = 2,
P2MS, P2MS = 3,
P2SH, P2SH = 4,
OpReturn, OpReturn = 5,
P2WPKH, P2WPKH = 6,
P2WSH, P2WSH = 7,
P2TR, P2TR = 8,
P2A, P2A = 9,
#[doc(hidden)] Empty = u16::MAX - 1,
#[schemars(skip)] Unknown = u16::MAX,
Dummy10,
#[doc(hidden)]
#[schemars(skip)]
Dummy11,
#[doc(hidden)]
#[schemars(skip)]
Dummy12,
#[doc(hidden)]
#[schemars(skip)]
Dummy13,
#[doc(hidden)]
#[schemars(skip)]
Dummy14,
#[doc(hidden)]
#[schemars(skip)]
Dummy15,
#[doc(hidden)]
#[schemars(skip)]
Dummy16,
#[doc(hidden)]
#[schemars(skip)]
Dummy17,
#[doc(hidden)]
#[schemars(skip)]
Dummy18,
#[doc(hidden)]
#[schemars(skip)]
Dummy19,
#[doc(hidden)]
#[schemars(skip)]
Dummy20,
#[doc(hidden)]
#[schemars(skip)]
Dummy21,
#[doc(hidden)]
#[schemars(skip)]
Dummy22,
#[doc(hidden)]
#[schemars(skip)]
Dummy23,
#[doc(hidden)]
#[schemars(skip)]
Dummy24,
#[doc(hidden)]
#[schemars(skip)]
Dummy25,
#[doc(hidden)]
#[schemars(skip)]
Dummy26,
#[doc(hidden)]
#[schemars(skip)]
Dummy27,
#[doc(hidden)]
#[schemars(skip)]
Dummy28,
#[doc(hidden)]
#[schemars(skip)]
Dummy29,
#[doc(hidden)]
#[schemars(skip)]
Dummy30,
#[doc(hidden)]
#[schemars(skip)]
Dummy31,
#[doc(hidden)]
#[schemars(skip)]
Dummy32,
#[doc(hidden)]
#[schemars(skip)]
Dummy33,
#[doc(hidden)]
#[schemars(skip)]
Dummy34,
#[doc(hidden)]
#[schemars(skip)]
Dummy35,
#[doc(hidden)]
#[schemars(skip)]
Dummy36,
#[doc(hidden)]
#[schemars(skip)]
Dummy37,
#[doc(hidden)]
#[schemars(skip)]
Dummy38,
#[doc(hidden)]
#[schemars(skip)]
Dummy39,
#[doc(hidden)]
#[schemars(skip)]
Dummy40,
#[doc(hidden)]
#[schemars(skip)]
Dummy41,
#[doc(hidden)]
#[schemars(skip)]
Dummy42,
#[doc(hidden)]
#[schemars(skip)]
Dummy43,
#[doc(hidden)]
#[schemars(skip)]
Dummy44,
#[doc(hidden)]
#[schemars(skip)]
Dummy45,
#[doc(hidden)]
#[schemars(skip)]
Dummy46,
#[doc(hidden)]
#[schemars(skip)]
Dummy47,
#[doc(hidden)]
#[schemars(skip)]
Dummy48,
#[doc(hidden)]
#[schemars(skip)]
Dummy49,
#[doc(hidden)]
#[schemars(skip)]
Dummy50,
#[doc(hidden)]
#[schemars(skip)]
Dummy51,
#[doc(hidden)]
#[schemars(skip)]
Dummy52,
#[doc(hidden)]
#[schemars(skip)]
Dummy53,
#[doc(hidden)]
#[schemars(skip)]
Dummy54,
#[doc(hidden)]
#[schemars(skip)]
Dummy55,
#[doc(hidden)]
#[schemars(skip)]
Dummy56,
#[doc(hidden)]
#[schemars(skip)]
Dummy57,
#[doc(hidden)]
#[schemars(skip)]
Dummy58,
#[doc(hidden)]
#[schemars(skip)]
Dummy59,
#[doc(hidden)]
#[schemars(skip)]
Dummy60,
#[doc(hidden)]
#[schemars(skip)]
Dummy61,
#[doc(hidden)]
#[schemars(skip)]
Dummy62,
#[doc(hidden)]
#[schemars(skip)]
Dummy63,
#[doc(hidden)]
#[schemars(skip)]
Dummy64,
#[doc(hidden)]
#[schemars(skip)]
Dummy65,
#[doc(hidden)]
#[schemars(skip)]
Dummy66,
#[doc(hidden)]
#[schemars(skip)]
Dummy67,
#[doc(hidden)]
#[schemars(skip)]
Dummy68,
#[doc(hidden)]
#[schemars(skip)]
Dummy69,
#[doc(hidden)]
#[schemars(skip)]
Dummy70,
#[doc(hidden)]
#[schemars(skip)]
Dummy71,
#[doc(hidden)]
#[schemars(skip)]
Dummy72,
#[doc(hidden)]
#[schemars(skip)]
Dummy73,
#[doc(hidden)]
#[schemars(skip)]
Dummy74,
#[doc(hidden)]
#[schemars(skip)]
Dummy75,
#[doc(hidden)]
#[schemars(skip)]
Dummy76,
#[doc(hidden)]
#[schemars(skip)]
Dummy77,
#[doc(hidden)]
#[schemars(skip)]
Dummy78,
#[doc(hidden)]
#[schemars(skip)]
Dummy79,
#[doc(hidden)]
#[schemars(skip)]
Dummy80,
#[doc(hidden)]
#[schemars(skip)]
Dummy81,
#[doc(hidden)]
#[schemars(skip)]
Dummy82,
#[doc(hidden)]
#[schemars(skip)]
Dummy83,
#[doc(hidden)]
#[schemars(skip)]
Dummy84,
#[doc(hidden)]
#[schemars(skip)]
Dummy85,
#[doc(hidden)]
#[schemars(skip)]
Dummy86,
#[doc(hidden)]
#[schemars(skip)]
Dummy87,
#[doc(hidden)]
#[schemars(skip)]
Dummy88,
#[doc(hidden)]
#[schemars(skip)]
Dummy89,
#[doc(hidden)]
#[schemars(skip)]
Dummy90,
#[doc(hidden)]
#[schemars(skip)]
Dummy91,
#[doc(hidden)]
#[schemars(skip)]
Dummy92,
#[doc(hidden)]
#[schemars(skip)]
Dummy93,
#[doc(hidden)]
#[schemars(skip)]
Dummy94,
#[doc(hidden)]
#[schemars(skip)]
Dummy95,
#[doc(hidden)]
#[schemars(skip)]
Dummy96,
#[doc(hidden)]
#[schemars(skip)]
Dummy97,
#[doc(hidden)]
#[schemars(skip)]
Dummy98,
#[doc(hidden)]
#[schemars(skip)]
Dummy99,
#[doc(hidden)]
#[schemars(skip)]
Dummy100,
#[doc(hidden)]
#[schemars(skip)]
Dummy101,
#[doc(hidden)]
#[schemars(skip)]
Dummy102,
#[doc(hidden)]
#[schemars(skip)]
Dummy103,
#[doc(hidden)]
#[schemars(skip)]
Dummy104,
#[doc(hidden)]
#[schemars(skip)]
Dummy105,
#[doc(hidden)]
#[schemars(skip)]
Dummy106,
#[doc(hidden)]
#[schemars(skip)]
Dummy107,
#[doc(hidden)]
#[schemars(skip)]
Dummy108,
#[doc(hidden)]
#[schemars(skip)]
Dummy109,
#[doc(hidden)]
#[schemars(skip)]
Dummy110,
#[doc(hidden)]
#[schemars(skip)]
Dummy111,
#[doc(hidden)]
#[schemars(skip)]
Dummy112,
#[doc(hidden)]
#[schemars(skip)]
Dummy113,
#[doc(hidden)]
#[schemars(skip)]
Dummy114,
#[doc(hidden)]
#[schemars(skip)]
Dummy115,
#[doc(hidden)]
#[schemars(skip)]
Dummy116,
#[doc(hidden)]
#[schemars(skip)]
Dummy117,
#[doc(hidden)]
#[schemars(skip)]
Dummy118,
#[doc(hidden)]
#[schemars(skip)]
Dummy119,
#[doc(hidden)]
#[schemars(skip)]
Dummy120,
#[doc(hidden)]
#[schemars(skip)]
Dummy121,
#[doc(hidden)]
#[schemars(skip)]
Dummy122,
#[doc(hidden)]
#[schemars(skip)]
Dummy123,
#[doc(hidden)]
#[schemars(skip)]
Dummy124,
#[doc(hidden)]
#[schemars(skip)]
Dummy125,
#[doc(hidden)]
#[schemars(skip)]
Dummy126,
#[doc(hidden)]
#[schemars(skip)]
Dummy127,
#[doc(hidden)]
#[schemars(skip)]
Dummy128,
#[doc(hidden)]
#[schemars(skip)]
Dummy129,
#[doc(hidden)]
#[schemars(skip)]
Dummy130,
#[doc(hidden)]
#[schemars(skip)]
Dummy131,
#[doc(hidden)]
#[schemars(skip)]
Dummy132,
#[doc(hidden)]
#[schemars(skip)]
Dummy133,
#[doc(hidden)]
#[schemars(skip)]
Dummy134,
#[doc(hidden)]
#[schemars(skip)]
Dummy135,
#[doc(hidden)]
#[schemars(skip)]
Dummy136,
#[doc(hidden)]
#[schemars(skip)]
Dummy137,
#[doc(hidden)]
#[schemars(skip)]
Dummy138,
#[doc(hidden)]
#[schemars(skip)]
Dummy139,
#[doc(hidden)]
#[schemars(skip)]
Dummy140,
#[doc(hidden)]
#[schemars(skip)]
Dummy141,
#[doc(hidden)]
#[schemars(skip)]
Dummy142,
#[doc(hidden)]
#[schemars(skip)]
Dummy143,
#[doc(hidden)]
#[schemars(skip)]
Dummy144,
#[doc(hidden)]
#[schemars(skip)]
Dummy145,
#[doc(hidden)]
#[schemars(skip)]
Dummy146,
#[doc(hidden)]
#[schemars(skip)]
Dummy147,
#[doc(hidden)]
#[schemars(skip)]
Dummy148,
#[doc(hidden)]
#[schemars(skip)]
Dummy149,
#[doc(hidden)]
#[schemars(skip)]
Dummy150,
#[doc(hidden)]
#[schemars(skip)]
Dummy151,
#[doc(hidden)]
#[schemars(skip)]
Dummy152,
#[doc(hidden)]
#[schemars(skip)]
Dummy153,
#[doc(hidden)]
#[schemars(skip)]
Dummy154,
#[doc(hidden)]
#[schemars(skip)]
Dummy155,
#[doc(hidden)]
#[schemars(skip)]
Dummy156,
#[doc(hidden)]
#[schemars(skip)]
Dummy157,
#[doc(hidden)]
#[schemars(skip)]
Dummy158,
#[doc(hidden)]
#[schemars(skip)]
Dummy159,
#[doc(hidden)]
#[schemars(skip)]
Dummy160,
#[doc(hidden)]
#[schemars(skip)]
Dummy161,
#[doc(hidden)]
#[schemars(skip)]
Dummy162,
#[doc(hidden)]
#[schemars(skip)]
Dummy163,
#[doc(hidden)]
#[schemars(skip)]
Dummy164,
#[doc(hidden)]
#[schemars(skip)]
Dummy165,
#[doc(hidden)]
#[schemars(skip)]
Dummy166,
#[doc(hidden)]
#[schemars(skip)]
Dummy167,
#[doc(hidden)]
#[schemars(skip)]
Dummy168,
#[doc(hidden)]
#[schemars(skip)]
Dummy169,
#[doc(hidden)]
#[schemars(skip)]
Dummy170,
#[doc(hidden)]
#[schemars(skip)]
Dummy171,
#[doc(hidden)]
#[schemars(skip)]
Dummy172,
#[doc(hidden)]
#[schemars(skip)]
Dummy173,
#[doc(hidden)]
#[schemars(skip)]
Dummy174,
#[doc(hidden)]
#[schemars(skip)]
Dummy175,
#[doc(hidden)]
#[schemars(skip)]
Dummy176,
#[doc(hidden)]
#[schemars(skip)]
Dummy177,
#[doc(hidden)]
#[schemars(skip)]
Dummy178,
#[doc(hidden)]
#[schemars(skip)]
Dummy179,
#[doc(hidden)]
#[schemars(skip)]
Dummy180,
#[doc(hidden)]
#[schemars(skip)]
Dummy181,
#[doc(hidden)]
#[schemars(skip)]
Dummy182,
#[doc(hidden)]
#[schemars(skip)]
Dummy183,
#[doc(hidden)]
#[schemars(skip)]
Dummy184,
#[doc(hidden)]
#[schemars(skip)]
Dummy185,
#[doc(hidden)]
#[schemars(skip)]
Dummy186,
#[doc(hidden)]
#[schemars(skip)]
Dummy187,
#[doc(hidden)]
#[schemars(skip)]
Dummy188,
#[doc(hidden)]
#[schemars(skip)]
Dummy189,
#[doc(hidden)]
#[schemars(skip)]
Dummy190,
#[doc(hidden)]
#[schemars(skip)]
Dummy191,
#[doc(hidden)]
#[schemars(skip)]
Dummy192,
#[doc(hidden)]
#[schemars(skip)]
Dummy193,
#[doc(hidden)]
#[schemars(skip)]
Dummy194,
#[doc(hidden)]
#[schemars(skip)]
Dummy195,
#[doc(hidden)]
#[schemars(skip)]
Dummy196,
#[doc(hidden)]
#[schemars(skip)]
Dummy197,
#[doc(hidden)]
#[schemars(skip)]
Dummy198,
#[doc(hidden)]
#[schemars(skip)]
Dummy199,
#[doc(hidden)]
#[schemars(skip)]
Dummy200,
#[doc(hidden)]
#[schemars(skip)]
Dummy201,
#[doc(hidden)]
#[schemars(skip)]
Dummy202,
#[doc(hidden)]
#[schemars(skip)]
Dummy203,
#[doc(hidden)]
#[schemars(skip)]
Dummy204,
#[doc(hidden)]
#[schemars(skip)]
Dummy205,
#[doc(hidden)]
#[schemars(skip)]
Dummy206,
#[doc(hidden)]
#[schemars(skip)]
Dummy207,
#[doc(hidden)]
#[schemars(skip)]
Dummy208,
#[doc(hidden)]
#[schemars(skip)]
Dummy209,
#[doc(hidden)]
#[schemars(skip)]
Dummy210,
#[doc(hidden)]
#[schemars(skip)]
Dummy211,
#[doc(hidden)]
#[schemars(skip)]
Dummy212,
#[doc(hidden)]
#[schemars(skip)]
Dummy213,
#[doc(hidden)]
#[schemars(skip)]
Dummy214,
#[doc(hidden)]
#[schemars(skip)]
Dummy215,
#[doc(hidden)]
#[schemars(skip)]
Dummy216,
#[doc(hidden)]
#[schemars(skip)]
Dummy217,
#[doc(hidden)]
#[schemars(skip)]
Dummy218,
#[doc(hidden)]
#[schemars(skip)]
Dummy219,
#[doc(hidden)]
#[schemars(skip)]
Dummy220,
#[doc(hidden)]
#[schemars(skip)]
Dummy221,
#[doc(hidden)]
#[schemars(skip)]
Dummy222,
#[doc(hidden)]
#[schemars(skip)]
Dummy223,
#[doc(hidden)]
#[schemars(skip)]
Dummy224,
#[doc(hidden)]
#[schemars(skip)]
Dummy225,
#[doc(hidden)]
#[schemars(skip)]
Dummy226,
#[doc(hidden)]
#[schemars(skip)]
Dummy227,
#[doc(hidden)]
#[schemars(skip)]
Dummy228,
#[doc(hidden)]
#[schemars(skip)]
Dummy229,
#[doc(hidden)]
#[schemars(skip)]
Dummy230,
#[doc(hidden)]
#[schemars(skip)]
Dummy231,
#[doc(hidden)]
#[schemars(skip)]
Dummy232,
#[doc(hidden)]
#[schemars(skip)]
Dummy233,
#[doc(hidden)]
#[schemars(skip)]
Dummy234,
#[doc(hidden)]
#[schemars(skip)]
Dummy235,
#[doc(hidden)]
#[schemars(skip)]
Dummy236,
#[doc(hidden)]
#[schemars(skip)]
Dummy237,
#[doc(hidden)]
#[schemars(skip)]
Dummy238,
#[doc(hidden)]
#[schemars(skip)]
Dummy239,
#[doc(hidden)]
#[schemars(skip)]
Dummy240,
#[doc(hidden)]
#[schemars(skip)]
Dummy241,
#[doc(hidden)]
#[schemars(skip)]
Dummy242,
#[doc(hidden)]
#[schemars(skip)]
Dummy243,
#[doc(hidden)]
#[schemars(skip)]
Dummy244,
#[doc(hidden)]
#[schemars(skip)]
Dummy245,
#[doc(hidden)]
#[schemars(skip)]
Dummy246,
#[doc(hidden)]
#[schemars(skip)]
Dummy247,
#[doc(hidden)]
#[schemars(skip)]
Dummy248,
#[doc(hidden)]
#[schemars(skip)]
Dummy249,
#[doc(hidden)]
#[schemars(skip)]
Dummy250,
#[doc(hidden)]
#[schemars(skip)]
Dummy251,
#[doc(hidden)]
#[schemars(skip)]
Dummy252,
#[doc(hidden)]
#[schemars(skip)]
Dummy253,
Empty = 254,
Unknown = 255,
} }
impl OutputType { impl OutputType {
fn is_valid(value: u16) -> bool {
value <= Self::P2A as u16 || value >= Self::Empty as u16
}
pub fn is_spendable(&self) -> bool { pub fn is_spendable(&self) -> bool {
match self { match self {
Self::P2PK65 => true, Self::P2PK65 => true,
@@ -776,7 +46,6 @@ impl OutputType {
Self::P2A => true, Self::P2A => true,
Self::Empty => true, Self::Empty => true,
Self::Unknown => true, Self::Unknown => true,
_ => unreachable!(),
} }
} }
@@ -794,7 +63,6 @@ impl OutputType {
Self::P2A => true, Self::P2A => true,
Self::Empty => false, Self::Empty => false,
Self::Unknown => false, Self::Unknown => false,
_ => unreachable!(),
} }
} }
@@ -924,7 +192,7 @@ impl Bytes for OutputType {
#[inline] #[inline]
fn to_bytes(&self) -> Self::Array { fn to_bytes(&self) -> Self::Array {
[*self as u8] (*self as u16).to_le_bytes()
} }
#[inline] #[inline]
@@ -935,9 +203,18 @@ impl Bytes for OutputType {
received: bytes.len(), received: bytes.len(),
}); });
}; };
// SAFETY: OutputType is repr(u8) and we're transmuting from u8 let value = u16::from_le_bytes([bytes[0], bytes[1]]);
// All values 0-255 are valid (includes dummy variants) if !Self::is_valid(value) {
let s: Self = unsafe { std::mem::transmute(bytes[0]) }; return Err(vecdb::Error::InvalidArgument("invalid OutputType"));
}
// SAFETY: We validated that value is a valid variant
let s: Self = unsafe { std::mem::transmute(value) };
Ok(s) Ok(s)
} }
} }
impl Pco for OutputType {
type NumberType = u16;
}
impl TransparentPco<u16> for OutputType {}

View File

@@ -50,6 +50,7 @@ impl Sats {
pub const _100K_BTC: Self = Self(100_000_00_000_000); pub const _100K_BTC: Self = Self(100_000_00_000_000);
pub const ONE_BTC: Self = Self(1_00_000_000); pub const ONE_BTC: Self = Self(1_00_000_000);
pub const MAX: Self = Self(u64::MAX); pub const MAX: Self = Self(u64::MAX);
pub const COINBASE: Self = Self(u64::MAX);
pub const FIFTY_BTC: Self = Self(50_00_000_000); pub const FIFTY_BTC: Self = Self(50_00_000_000);
pub const ONE_BTC_U128: u128 = 1_00_000_000; pub const ONE_BTC_U128: u128 = 1_00_000_000;

View File

@@ -0,0 +1,100 @@
//! Combined transaction output data for efficient access.
use std::fmt::{self, Display};
use std::mem::size_of;
use schemars::JsonSchema;
use serde::Serialize;
use vecdb::{Bytes, Formattable};
use crate::{OutputType, Sats, TypeIndex};
/// Core transaction output data: value, type, and type index.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)]
#[repr(C)]
pub struct TxOutData {
pub value: Sats,
pub typeindex: TypeIndex,
pub outputtype: OutputType,
_padding: u16,
}
impl TxOutData {
#[inline]
pub const fn new(value: Sats, outputtype: OutputType, typeindex: TypeIndex) -> Self {
Self {
value,
typeindex,
outputtype,
_padding: 0,
}
}
}
impl Bytes for TxOutData {
type Array = [u8; size_of::<Self>()];
#[inline]
fn to_bytes(&self) -> Self::Array {
let mut bytes = [0u8; 16];
bytes[0..8].copy_from_slice(&self.value.to_bytes());
bytes[8..12].copy_from_slice(&self.typeindex.to_bytes());
bytes[12..14].copy_from_slice(&self.outputtype.to_bytes());
// bytes[14..16] is padding, already zero
bytes
}
#[inline]
fn from_bytes(bytes: &[u8]) -> vecdb::Result<Self> {
if bytes.len() != size_of::<Self>() {
return Err(vecdb::Error::WrongLength {
expected: size_of::<Self>(),
received: bytes.len(),
});
}
Ok(Self {
value: Sats::from_bytes(&bytes[0..8])?,
typeindex: TypeIndex::from_bytes(&bytes[8..12])?,
outputtype: OutputType::from_bytes(&bytes[12..14])?,
_padding: 0,
})
}
}
impl Display for TxOutData {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"value: {}, outputtype: {}, typeindex: {}",
self.value, self.outputtype, self.typeindex
)
}
}
impl Formattable for TxOutData {
fn may_need_escaping() -> bool {
true
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_size() {
assert_eq!(size_of::<TxOutData>(), 16);
}
#[test]
fn test_roundtrip() {
let data = TxOutData::new(
Sats::from(123456789u64),
OutputType::P2TR,
TypeIndex::from(42u32),
);
let bytes = data.to_bytes();
let decoded = TxOutData::from_bytes(&bytes).unwrap();
assert_eq!(data, decoded);
}
}

View File

@@ -24,6 +24,8 @@ use vecdb::{CheckedSub, Formattable, Pco};
pub struct TypeIndex(u32); pub struct TypeIndex(u32);
impl TypeIndex { impl TypeIndex {
pub const COINBASE: Self = Self(u32::MAX);
pub fn new(i: u32) -> Self { pub fn new(i: u32) -> Self {
Self(i) Self(i)
} }

View File

@@ -1 +1,2 @@
generated generated
index.js

View File

@@ -1,11 +0,0 @@
/**
* @param {VoidFunction} callback
* @param {number} [timeout = 1]
*/
export function runWhenIdle(callback, timeout = 1) {
if ("requestIdleCallback" in window) {
requestIdleCallback(callback);
} else {
setTimeout(callback, timeout);
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,111 +0,0 @@
import {
INDEX_TO_WORD,
COMPRESSED_METRIC_TO_INDEXES,
} from "./generated/metrics";
/**
* @typedef {typeof import("./generated/metrics")["COMPRESSED_METRIC_TO_INDEXES"]} MetricToIndexes
* @typedef {string} Metric
*/
/** @type {Record<string, number>} */
const WORD_TO_INDEX = {};
INDEX_TO_WORD.forEach((word, index) => {
WORD_TO_INDEX[word] = index;
});
/**
* @param {Metric} metric
*/
export function getIndexesFromMetric(metric) {
return COMPRESSED_METRIC_TO_INDEXES[compressMetric(metric)];
}
/**
* @param {Metric} metric
*/
export function hasMetric(metric) {
return compressMetric(metric) in COMPRESSED_METRIC_TO_INDEXES;
}
/**
* @param {string} metric
*/
function compressMetric(metric) {
return metric
.split("_")
.map((word) => {
const index = WORD_TO_INDEX[word];
return index !== undefined ? indexToLetters(index) : word;
})
.join("_");
}
/**
* @param {string} compressedMetric
*/
function decompressMetric(compressedMetric) {
return compressedMetric
.split("_")
.map((code) => {
const index = lettersToIndex(code);
return INDEX_TO_WORD[index] || code; // Fallback to original if not found
})
.join("_");
}
/**
* @param {string} letters
*/
function lettersToIndex(letters) {
let result = 0;
for (let i = 0; i < letters.length; i++) {
const value = charToIndex(letters.charCodeAt(i));
result = result * 52 + value + 1;
}
return result - 1;
}
/**
* @param {number} byte
*/
function charToIndex(byte) {
if (byte >= 65 && byte <= 90) {
// 'A' to 'Z'
return byte - 65;
} else if (byte >= 97 && byte <= 122) {
// 'a' to 'z'
return byte - 97 + 26;
} else {
return 255; // Invalid
}
}
/**
* @param {number} index
*/
function indexToLetters(index) {
if (index < 52) {
return indexToChar(index);
}
let result = [];
while (true) {
result.push(indexToChar(index % 52));
index = Math.floor(index / 52);
if (index === 0) break;
index -= 1;
}
return result.reverse().join("");
}
/**
* @param {number} index
*/
function indexToChar(index) {
if (index <= 25) {
return String.fromCharCode(65 + index); // A-Z
} else {
return String.fromCharCode(97 + index - 26); // a-z
}
}