diff --git a/Cargo.lock b/Cargo.lock index 201f27c42..87db4edac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -666,6 +666,8 @@ dependencies = [ "brk_parser", "brk_store", "brk_structs", + "brk_vecs", + "brk_vecs_derive", "fjall", "log", "rayon", @@ -1197,6 +1199,23 @@ dependencies = [ "zerocopy-derive", ] +[[package]] +name = "brk_vecs" +version = "0.0.111" +dependencies = [ + "serde", + "vecdb", +] + +[[package]] +name = "brk_vecs_derive" +version = "0.0.111" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "brotli" version = "8.0.2" diff --git a/Cargo.toml b/Cargo.toml index 6c4d9ed37..2bb839dc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ package.version = "0.0.111" package.homepage = "https://bitcoinresearchkit.org" package.repository = "https://github.com/bitcoinresearchkit/brk" package.readme = "README.md" -package.rust-version = "1.89" +package.rust-version = "1.90" [profile.dev] lto = "thin" @@ -59,6 +59,8 @@ brk_parser = { version = "0.0.111", path = "crates/brk_parser" } brk_server = { version = "0.0.111", path = "crates/brk_server" } brk_store = { version = "0.0.111", path = "crates/brk_store" } brk_structs = { version = "0.0.111", path = "crates/brk_structs" } +brk_vecs = { version = "0.0.111", path = "crates/brk_vecs" } +brk_vecs_derive = { version = "0.0.111", path = "crates/brk_vecs_derive" } byteview = "=0.6.1" derive_deref = "1.1.1" fjall = "2.11.2" @@ -76,7 +78,7 @@ serde_json = { version = "1.0.145", features = ["float_roundtrip"] } sonic-rs = "0.5.5" tokio = { version = "1.47.1", features = ["rt-multi-thread"] } # vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]} -vecdb = { version = "0.2.16", features = ["derive"]} +vecdb = { version = "0.2.16", features = ["derive"] } zerocopy = "0.8.27" zerocopy-derive = "0.8.27" diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index c12102675..ac9f3af09 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -17,6 +17,8 @@ brk_error = { workspace = true } brk_logger = { workspace = true } brk_parser = { workspace = true } brk_store = { workspace = true } +brk_vecs = { workspace = true } +brk_vecs_derive = { workspace = true } vecdb = { workspace = true } fjall = { workspace = true } log = { workspace = true } diff --git a/crates/brk_indexer/src/vecs.rs b/crates/brk_indexer/src/vecs.rs index 20022df88..db100acc5 100644 --- a/crates/brk_indexer/src/vecs.rs +++ b/crates/brk_indexer/src/vecs.rs @@ -9,6 +9,8 @@ use brk_structs::{ RawLockTime, Sats, StoredBool, StoredF64, StoredU32, StoredU64, Timestamp, TxIndex, TxVersion, Txid, TypeIndex, UnknownOutputIndex, Version, Weight, }; +use brk_vecs::{IVecs, TreeNode}; +use brk_vecs_derive::IVecs; use rayon::prelude::*; use vecdb::{ AnyCollectableVec, AnyStoredVec, CompressedVec, Database, GenericStoredVec, PAGE_SIZE, RawVec, @@ -17,7 +19,7 @@ use vecdb::{ use crate::Indexes; -#[derive(Clone)] +#[derive(Clone, IVecs)] pub struct Vecs { db: Database, pub emptyoutputindex_to_txindex: CompressedVec, diff --git a/crates/brk_server/src/api/explorer/mod.rs b/crates/brk_server/src/api/chain/mod.rs similarity index 99% rename from crates/brk_server/src/api/explorer/mod.rs rename to crates/brk_server/src/api/chain/mod.rs index 947f2f351..0bbe274e2 100644 --- a/crates/brk_server/src/api/explorer/mod.rs +++ b/crates/brk_server/src/api/chain/mod.rs @@ -35,7 +35,7 @@ struct TxResponse { impl ApiExplorerRoutes for Router { fn add_api_explorer_routes(self) -> Self { self.route( - "/api/address/{address}", + "/api/chain/address/{address}", get( async |Path(address): Path, state: State| -> Response { let Ok(address) = Address::from_str(&address) else { @@ -148,7 +148,7 @@ impl ApiExplorerRoutes for Router { ), ) .route( - "/api/tx/{txid}", + "/api/chain/tx/{txid}", get( async |Path(txid): Path, state: State| -> Response { let Ok(txid) = bitcoin::Txid::from_str(&txid) else { diff --git a/crates/brk_server/src/api/explorer/reader.rs b/crates/brk_server/src/api/chain/reader.rs similarity index 100% rename from crates/brk_server/src/api/explorer/reader.rs rename to crates/brk_server/src/api/chain/reader.rs diff --git a/crates/brk_server/src/api/metrics/mod.rs b/crates/brk_server/src/api/metrics/mod.rs index 7b3d3700d..d060ad582 100644 --- a/crates/brk_server/src/api/metrics/mod.rs +++ b/crates/brk_server/src/api/metrics/mod.rs @@ -35,8 +35,19 @@ impl ApiMetricsRoutes for Router { Json(app_state.interface.get_indexes()).into_response() }), ) + .route( + "/api/metrics/list", + get( + async |State(app_state): State, + Query(pagination): Query| + -> Response { + Json(app_state.interface.get_metrics(pagination)).into_response() + }, + ), + ) + // TODO: // .route( - // "/api/vecs/metrics", + // "/api/metrics/search", // get( // async |State(app_state): State, // Query(pagination): Query| @@ -45,16 +56,6 @@ impl ApiMetricsRoutes for Router { // }, // ), // ) - // .route( - // "/api/vecs/index-to-metrics", - // get( - // async |State(app_state): State, - // Query(paginated_index): Query| - // -> Response { - // Json(app_state.interface.get_index_to_vecids(paginated_index)).into_response() - // }, - // ), - // ) .route( "/api/metrics/{metric}", get( @@ -85,7 +86,7 @@ impl ApiMetricsRoutes for Router { ), ) // !!! - // DEPRECATED + // DEPRECATED: Do not use // !!! .route( "/api/vecs/query", @@ -100,7 +101,7 @@ impl ApiMetricsRoutes for Router { ), ) // !!! - // DEPRECATED + // DEPRECATED: Do not use // !!! .route( "/api/vecs/{variant}", diff --git a/crates/brk_server/src/api/mod.rs b/crates/brk_server/src/api/mod.rs index dec94d49a..b943c720d 100644 --- a/crates/brk_server/src/api/mod.rs +++ b/crates/brk_server/src/api/mod.rs @@ -1,10 +1,10 @@ use axum::{Router, response::Redirect, routing::get}; -use crate::api::{explorer::ApiExplorerRoutes, metrics::ApiMetricsRoutes}; +use crate::api::{chain::ApiExplorerRoutes, metrics::ApiMetricsRoutes}; use super::AppState; -mod explorer; +mod chain; mod metrics; pub trait ApiRoutes { diff --git a/crates/brk_vecs/Cargo.toml b/crates/brk_vecs/Cargo.toml new file mode 100644 index 000000000..cba7726cd --- /dev/null +++ b/crates/brk_vecs/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "brk_vecs" +description = "Traits for Vecs structs throughout BRK" +version.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true +build = "build.rs" + +[dependencies] +serde = { workspace = true } +vecdb = { workspace = true } diff --git a/crates/brk_vecs/README.md b/crates/brk_vecs/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/crates/brk_vecs/build.rs b/crates/brk_vecs/build.rs new file mode 100644 index 000000000..a4055a31e --- /dev/null +++ b/crates/brk_vecs/build.rs @@ -0,0 +1,8 @@ +fn main() { + let profile = std::env::var("PROFILE").unwrap_or_default(); + + if profile == "release" { + println!("cargo:rustc-flag=-C"); + println!("cargo:rustc-flag=target-cpu=native"); + } +} diff --git a/crates/brk_vecs/src/lib.rs b/crates/brk_vecs/src/lib.rs new file mode 100644 index 000000000..af2a4a28e --- /dev/null +++ b/crates/brk_vecs/src/lib.rs @@ -0,0 +1,63 @@ +use std::collections::HashMap; + +use serde::Serialize; +use vecdb::AnyCollectableVec; + +pub trait IVecs { + fn to_tree_node(&self) -> TreeNode; + fn iter_any_collectable<'a>( + &'a self, + ) -> Box + 'a>; +} + +// Terminal implementation for any type that implements AnyCollectableVec +// impl IVecs for T { +// fn to_tree_node(&self) -> TreeNode { +// TreeNode::Leaf(self.name().to_string()) +// } + +// fn iter_any_collectable<'a>( +// &'a self, +// ) -> Box + 'a> { +// Box::new(std::iter::once(self as &dyn AnyCollectableVec)) +// } +// } + +// For Option types +// impl IVecs for Option { +// fn to_tree_node(&self) -> TreeNode { +// match self { +// Some(inner) => inner.to_tree_node(), +// None => TreeNode::Branch(HashMap::new()), +// } +// } + +// fn iter_any_collectable<'a>( +// &'a self, +// ) -> Box + 'a> { +// match self { +// Some(inner) => inner.iter_any_collectable(), +// None => Box::new(std::iter::empty()), +// } +// } +// } + +// For Box types +// impl IVecs for Box { +// fn to_tree_node(&self) -> TreeNode { +// (**self).to_tree_node() +// } + +// fn iter_any_collectable<'a>( +// &'a self, +// ) -> Box + 'a> { +// (**self).iter_any_collectable() +// } +// } + +#[derive(Debug, Clone, Serialize)] +#[serde(untagged)] +pub enum TreeNode { + Branch(HashMap), + Leaf(String), +} diff --git a/crates/brk_vecs_derive/Cargo.toml b/crates/brk_vecs_derive/Cargo.toml new file mode 100644 index 000000000..884eeefd7 --- /dev/null +++ b/crates/brk_vecs_derive/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "brk_vecs_derive" +description = "Derive for brk_vec's used in BRK" +version.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true +build = "build.rs" + +[lib] +proc-macro = true + +[dependencies] +syn = "2.0" +quote = "1.0" +proc-macro2 = "1.0.101" diff --git a/crates/brk_vecs_derive/README.md b/crates/brk_vecs_derive/README.md new file mode 100644 index 000000000..bd3aba254 --- /dev/null +++ b/crates/brk_vecs_derive/README.md @@ -0,0 +1 @@ +# brk_vecs_derive diff --git a/crates/brk_vecs_derive/build.rs b/crates/brk_vecs_derive/build.rs new file mode 100644 index 000000000..a4055a31e --- /dev/null +++ b/crates/brk_vecs_derive/build.rs @@ -0,0 +1,8 @@ +fn main() { + let profile = std::env::var("PROFILE").unwrap_or_default(); + + if profile == "release" { + println!("cargo:rustc-flag=-C"); + println!("cargo:rustc-flag=target-cpu=native"); + } +} diff --git a/crates/brk_vecs_derive/src/lib.rs b/crates/brk_vecs_derive/src/lib.rs new file mode 100644 index 000000000..ad3a37384 --- /dev/null +++ b/crates/brk_vecs_derive/src/lib.rs @@ -0,0 +1,166 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{Data, DeriveInput, Fields, Type, parse_macro_input}; + +#[proc_macro_derive(IVecs, attributes(vecs))] +pub fn derive_ivecs(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + let name = &input.ident; + + let traverse_impl = match &input.data { + Data::Struct(data) => { + let field_traversals = generate_field_traversals(&data.fields); + let iterator_impl = generate_iterator_impl(&data.fields); + + quote! { + impl IVecs for #name { + fn to_tree_node(&self) -> TreeNode { + let mut children = std::collections::HashMap::new(); + #field_traversals + TreeNode::Branch(children) + } + + #iterator_impl + } + } + } + _ => panic!("IVecs can only be derived for structs"), + }; + + TokenStream::from(traverse_impl) +} + +// This catches EagerVec, RawVec, CompressedVec, StoredVec, and any future *Vec types +fn is_vec_type(ty: &Type) -> bool { + if let Type::Path(type_path) = ty + && let Some(segment) = type_path.path.segments.last() + { + let ident = segment.ident.to_string(); + // Heuristic: if it ends with "Vec", it's likely a direct vec type + // This is more maintainable than hardcoding all vec types + return ident.ends_with("Vec"); + } + false +} + +fn generate_field_traversals(fields: &Fields) -> proc_macro2::TokenStream { + match fields { + Fields::Named(fields) => { + let traversals = fields.named.iter().filter_map(|f| { + let field_name = f.ident.as_ref()?; + let field_name_str = field_name.to_string(); + + if has_skip_attribute(f) { + return None; + } + + if !matches!(f.vis, syn::Visibility::Public(_)) { + return None; + } + + Some(quote! { + children.insert( + String::from(#field_name_str), + self.#field_name.to_tree_node() + ); + }) + }); + + quote! { #(#traversals)* } + } + _ => quote! {}, + } +} + +fn has_skip_attribute(field: &syn::Field) -> bool { + field.attrs.iter().any(|attr| { + attr.path().is_ident("vec_tree") + && attr + .parse_args::() + .map(|ident| ident == "skip") + .unwrap_or(false) + }) +} + +fn generate_iterator_impl(fields: &Fields) -> proc_macro2::TokenStream { + match fields { + Fields::Named(fields) => { + let mut direct_vecs = Vec::new(); + let mut option_vecs = Vec::new(); + let mut option_nested = Vec::new(); + let mut nested_fields = Vec::new(); + + for field in fields.named.iter() { + if let Some(field_name) = &field.ident { + if !matches!(field.vis, syn::Visibility::Public(_)) { + continue; + } + + if let Some(inner_ty) = get_option_inner_type(&field.ty) { + if is_vec_type(inner_ty) { + option_vecs.push(field_name); + } else { + option_nested.push(field_name); + } + } else if is_vec_type(&field.ty) { + direct_vecs.push(field_name); + } else { + nested_fields.push(field_name); + } + } + } + + let base = if !direct_vecs.is_empty() { + quote! { + let mut iter: Box + '_> = Box::new( + [#(&self.#direct_vecs as &dyn AnyCollectableVec,)*].into_iter() + ); + } + } else { + quote! { + let mut iter: Box + '_> = + Box::new(std::iter::empty()); + } + }; + + let option_vec_chains = option_vecs.iter().map(|f| { + quote! { iter = Box::new(iter.chain(self.#f.iter())); } + }); + + let option_nested_chains = option_nested.iter().map(|f| { + quote! { iter = Box::new(iter.chain(self.#f.iter().flat_map(|v| v.iter_any_collectable()))); } + }); + + let nested_chains = nested_fields.iter().map(|f| { + quote! { iter = Box::new(iter.chain(self.#f.iter_any_collectable())); } + }); + + quote! { + fn iter_any_collectable<'a>(&'a self) -> Box + 'a> { + #base + #(#option_vec_chains)* + #(#option_nested_chains)* + #(#nested_chains)* + iter + } + } + } + _ => quote! { + fn iter_any_collectable<'a>(&'a self) -> Box + 'a> { + Box::new(std::iter::empty()) + } + }, + } +} + +fn get_option_inner_type(ty: &Type) -> Option<&Type> { + if let Type::Path(type_path) = ty + && let Some(segment) = type_path.path.segments.last() + && segment.ident == "Option" + && let syn::PathArguments::AngleBracketed(args) = &segment.arguments + && let Some(syn::GenericArgument::Type(inner_ty)) = args.args.first() + { + return Some(inner_ty); + } + None +}