vecs: add trait + derive crates

Cargo.lock

@@ -666,6 +666,8 @@ dependencies = [
  "brk_parser",
  "brk_store",
  "brk_structs",
+ "brk_vecs",
+ "brk_vecs_derive",
  "fjall",
  "log",
  "rayon",
@@ -1197,6 +1199,23 @@ dependencies = [
  "zerocopy-derive",
 ]
 
+[[package]]
+name = "brk_vecs"
+version = "0.0.111"
+dependencies = [
+ "serde",
+ "vecdb",
+]
+
+[[package]]
+name = "brk_vecs_derive"
+version = "0.0.111"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.106",
+]
+
 [[package]]
 name = "brotli"
 version = "8.0.2"

Cargo.toml

@@ -8,7 +8,7 @@ package.version = "0.0.111"
 package.homepage = "https://bitcoinresearchkit.org"
 package.repository = "https://github.com/bitcoinresearchkit/brk"
 package.readme = "README.md"
-package.rust-version = "1.89"
+package.rust-version = "1.90"
 
 [profile.dev]
 lto = "thin"
@@ -59,6 +59,8 @@ brk_parser = { version = "0.0.111", path = "crates/brk_parser" }
 brk_server = { version = "0.0.111", path = "crates/brk_server" }
 brk_store = { version = "0.0.111", path = "crates/brk_store" }
 brk_structs = { version = "0.0.111", path = "crates/brk_structs" }
+brk_vecs = { version = "0.0.111", path = "crates/brk_vecs" }
+brk_vecs_derive = { version = "0.0.111", path = "crates/brk_vecs_derive" }
 byteview = "=0.6.1"
 derive_deref = "1.1.1"
 fjall = "2.11.2"
@@ -76,7 +78,7 @@ serde_json = { version = "1.0.145", features = ["float_roundtrip"] }
 sonic-rs = "0.5.5"
 tokio = { version = "1.47.1", features = ["rt-multi-thread"] }
 # vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]}
-vecdb = { version = "0.2.16", features = ["derive"]}
+vecdb = { version = "0.2.16", features = ["derive"] }
 zerocopy = "0.8.27"
 zerocopy-derive = "0.8.27"
 

crates/brk_indexer/Cargo.toml

@@ -17,6 +17,8 @@ brk_error = { workspace = true }
 brk_logger = { workspace = true }
 brk_parser = { workspace = true }
 brk_store = { workspace = true }
+brk_vecs = { workspace = true }
+brk_vecs_derive = { workspace = true }
 vecdb = { workspace = true }
 fjall = { workspace = true }
 log = { workspace = true }

crates/brk_indexer/src/vecs.rs

@@ -9,6 +9,8 @@ use brk_structs::{
     RawLockTime, Sats, StoredBool, StoredF64, StoredU32, StoredU64, Timestamp, TxIndex, TxVersion,
     Txid, TypeIndex, UnknownOutputIndex, Version, Weight,
 };
+use brk_vecs::{IVecs, TreeNode};
+use brk_vecs_derive::IVecs;
 use rayon::prelude::*;
 use vecdb::{
     AnyCollectableVec, AnyStoredVec, CompressedVec, Database, GenericStoredVec, PAGE_SIZE, RawVec,
@@ -17,7 +19,7 @@ use vecdb::{
 
 use crate::Indexes;
 
-#[derive(Clone)]
+#[derive(Clone, IVecs)]
 pub struct Vecs {
     db: Database,
     pub emptyoutputindex_to_txindex: CompressedVec<EmptyOutputIndex, TxIndex>,

crates/brk_server/src/api/chain/mod.rs

@@ -35,7 +35,7 @@ struct TxResponse {
 impl ApiExplorerRoutes for Router<AppState> {
     fn add_api_explorer_routes(self) -> Self {
         self.route(
-            "/api/address/{address}",
+            "/api/chain/address/{address}",
             get(
                 async |Path(address): Path<String>, state: State<AppState>| -> Response {
                     let Ok(address) = Address::from_str(&address) else {
@@ -148,7 +148,7 @@ impl ApiExplorerRoutes for Router<AppState> {
             ),
         )
         .route(
-            "/api/tx/{txid}",
+            "/api/chain/tx/{txid}",
             get(
                 async |Path(txid): Path<String>, state: State<AppState>| -> Response {
                     let Ok(txid) = bitcoin::Txid::from_str(&txid) else {

crates/brk_server/src/api/metrics/mod.rs

@@ -35,8 +35,19 @@ impl ApiMetricsRoutes for Router<AppState> {
                 Json(app_state.interface.get_indexes()).into_response()
             }),
         )
+        .route(
+            "/api/metrics/list",
+            get(
+                async |State(app_state): State<AppState>,
+                       Query(pagination): Query<PaginationParam>|
+                       -> Response {
+                    Json(app_state.interface.get_metrics(pagination)).into_response()
+                },
+            ),
+        )
+        // TODO:
         // .route(
-        //     "/api/vecs/metrics",
+        //     "/api/metrics/search",
         //     get(
         //         async |State(app_state): State<AppState>,
         //                Query(pagination): Query<PaginationParam>|
@@ -45,16 +56,6 @@ impl ApiMetricsRoutes for Router<AppState> {
         //         },
         //     ),
         // )
-        // .route(
-        //     "/api/vecs/index-to-metrics",
-        //     get(
-        //         async |State(app_state): State<AppState>,
-        //                Query(paginated_index): Query<PaginatedIndexParam>|
-        //                -> Response {
-        //             Json(app_state.interface.get_index_to_vecids(paginated_index)).into_response()
-        //         },
-        //     ),
-        // )
         .route(
             "/api/metrics/{metric}",
             get(
@@ -85,7 +86,7 @@ impl ApiMetricsRoutes for Router<AppState> {
             ),
         )
         // !!!
-        // DEPRECATED
+        // DEPRECATED: Do not use
         // !!!
         .route(
             "/api/vecs/query",
@@ -100,7 +101,7 @@ impl ApiMetricsRoutes for Router<AppState> {
             ),
         )
         // !!!
-        // DEPRECATED
+        // DEPRECATED: Do not use
         // !!!
         .route(
             "/api/vecs/{variant}",

crates/brk_server/src/api/mod.rs

@@ -1,10 +1,10 @@
 use axum::{Router, response::Redirect, routing::get};
 
-use crate::api::{explorer::ApiExplorerRoutes, metrics::ApiMetricsRoutes};
+use crate::api::{chain::ApiExplorerRoutes, metrics::ApiMetricsRoutes};
 
 use super::AppState;
 
-mod explorer;
+mod chain;
 mod metrics;
 
 pub trait ApiRoutes {

crates/brk_vecs/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "brk_vecs"
+description = "Traits for Vecs structs throughout BRK"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+build = "build.rs"
+
+[dependencies]
+serde = { workspace = true }
+vecdb = { workspace = true }

crates/brk_vecs/build.rs

@@ -0,0 +1,8 @@
+fn main() {
+    let profile = std::env::var("PROFILE").unwrap_or_default();
+
+    if profile == "release" {
+        println!("cargo:rustc-flag=-C");
+        println!("cargo:rustc-flag=target-cpu=native");
+    }
+}

crates/brk_vecs/src/lib.rs

@@ -0,0 +1,63 @@
+use std::collections::HashMap;
+
+use serde::Serialize;
+use vecdb::AnyCollectableVec;
+
+pub trait IVecs {
+    fn to_tree_node(&self) -> TreeNode;
+    fn iter_any_collectable<'a>(
+        &'a self,
+    ) -> Box<dyn Iterator<Item = &'a dyn AnyCollectableVec> + 'a>;
+}
+
+// Terminal implementation for any type that implements AnyCollectableVec
+// impl<T: AnyCollectableVec> IVecs for T {
+//     fn to_tree_node(&self) -> TreeNode {
+//         TreeNode::Leaf(self.name().to_string())
+//     }
+
+//     fn iter_any_collectable<'a>(
+//         &'a self,
+//     ) -> Box<dyn Iterator<Item = &'a dyn AnyCollectableVec> + 'a> {
+//         Box::new(std::iter::once(self as &dyn AnyCollectableVec))
+//     }
+// }
+
+// For Option types
+// impl<T: IVecs> IVecs for Option<T> {
+//     fn to_tree_node(&self) -> TreeNode {
+//         match self {
+//             Some(inner) => inner.to_tree_node(),
+//             None => TreeNode::Branch(HashMap::new()),
+//         }
+//     }
+
+//     fn iter_any_collectable<'a>(
+//         &'a self,
+//     ) -> Box<dyn Iterator<Item = &'a dyn AnyCollectableVec> + 'a> {
+//         match self {
+//             Some(inner) => inner.iter_any_collectable(),
+//             None => Box::new(std::iter::empty()),
+//         }
+//     }
+// }
+
+// For Box types
+// impl<T: IVecs> IVecs for Box<T> {
+//     fn to_tree_node(&self) -> TreeNode {
+//         (**self).to_tree_node()
+//     }
+
+//     fn iter_any_collectable<'a>(
+//         &'a self,
+//     ) -> Box<dyn Iterator<Item = &'a dyn AnyCollectableVec> + 'a> {
+//         (**self).iter_any_collectable()
+//     }
+// }
+
+#[derive(Debug, Clone, Serialize)]
+#[serde(untagged)]
+pub enum TreeNode {
+    Branch(HashMap<String, TreeNode>),
+    Leaf(String),
+}

crates/brk_vecs_derive/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "brk_vecs_derive"
+description = "Derive for brk_vec's used in BRK"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+build = "build.rs"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+syn = "2.0"
+quote = "1.0"
+proc-macro2 = "1.0.101"

crates/brk_vecs_derive/README.md

@@ -0,0 +1 @@
+# brk_vecs_derive

crates/brk_vecs_derive/build.rs

@@ -0,0 +1,8 @@
+fn main() {
+    let profile = std::env::var("PROFILE").unwrap_or_default();
+
+    if profile == "release" {
+        println!("cargo:rustc-flag=-C");
+        println!("cargo:rustc-flag=target-cpu=native");
+    }
+}

crates/brk_vecs_derive/src/lib.rs

@@ -0,0 +1,166 @@
+use proc_macro::TokenStream;
+use quote::quote;
+use syn::{Data, DeriveInput, Fields, Type, parse_macro_input};
+
+#[proc_macro_derive(IVecs, attributes(vecs))]
+pub fn derive_ivecs(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as DeriveInput);
+    let name = &input.ident;
+
+    let traverse_impl = match &input.data {
+        Data::Struct(data) => {
+            let field_traversals = generate_field_traversals(&data.fields);
+            let iterator_impl = generate_iterator_impl(&data.fields);
+
+            quote! {
+                impl IVecs for #name {
+                    fn to_tree_node(&self) -> TreeNode {
+                        let mut children = std::collections::HashMap::new();
+                        #field_traversals
+                        TreeNode::Branch(children)
+                    }
+
+                    #iterator_impl
+                }
+            }
+        }
+        _ => panic!("IVecs can only be derived for structs"),
+    };
+
+    TokenStream::from(traverse_impl)
+}
+
+// This catches EagerVec, RawVec, CompressedVec, StoredVec, and any future *Vec types
+fn is_vec_type(ty: &Type) -> bool {
+    if let Type::Path(type_path) = ty
+        && let Some(segment) = type_path.path.segments.last()
+    {
+        let ident = segment.ident.to_string();
+        // Heuristic: if it ends with "Vec", it's likely a direct vec type
+        // This is more maintainable than hardcoding all vec types
+        return ident.ends_with("Vec");
+    }
+    false
+}
+
+fn generate_field_traversals(fields: &Fields) -> proc_macro2::TokenStream {
+    match fields {
+        Fields::Named(fields) => {
+            let traversals = fields.named.iter().filter_map(|f| {
+                let field_name = f.ident.as_ref()?;
+                let field_name_str = field_name.to_string();
+
+                if has_skip_attribute(f) {
+                    return None;
+                }
+
+                if !matches!(f.vis, syn::Visibility::Public(_)) {
+                    return None;
+                }
+
+                Some(quote! {
+                    children.insert(
+                        String::from(#field_name_str),
+                        self.#field_name.to_tree_node()
+                    );
+                })
+            });
+
+            quote! { #(#traversals)* }
+        }
+        _ => quote! {},
+    }
+}
+
+fn has_skip_attribute(field: &syn::Field) -> bool {
+    field.attrs.iter().any(|attr| {
+        attr.path().is_ident("vec_tree")
+            && attr
+                .parse_args::<syn::Ident>()
+                .map(|ident| ident == "skip")
+                .unwrap_or(false)
+    })
+}
+
+fn generate_iterator_impl(fields: &Fields) -> proc_macro2::TokenStream {
+    match fields {
+        Fields::Named(fields) => {
+            let mut direct_vecs = Vec::new();
+            let mut option_vecs = Vec::new();
+            let mut option_nested = Vec::new();
+            let mut nested_fields = Vec::new();
+
+            for field in fields.named.iter() {
+                if let Some(field_name) = &field.ident {
+                    if !matches!(field.vis, syn::Visibility::Public(_)) {
+                        continue;
+                    }
+
+                    if let Some(inner_ty) = get_option_inner_type(&field.ty) {
+                        if is_vec_type(inner_ty) {
+                            option_vecs.push(field_name);
+                        } else {
+                            option_nested.push(field_name);
+                        }
+                    } else if is_vec_type(&field.ty) {
+                        direct_vecs.push(field_name);
+                    } else {
+                        nested_fields.push(field_name);
+                    }
+                }
+            }
+
+            let base = if !direct_vecs.is_empty() {
+                quote! {
+                    let mut iter: Box<dyn Iterator<Item = &dyn AnyCollectableVec> + '_> = Box::new(
+                        [#(&self.#direct_vecs as &dyn AnyCollectableVec,)*].into_iter()
+                    );
+                }
+            } else {
+                quote! {
+                    let mut iter: Box<dyn Iterator<Item = &dyn AnyCollectableVec> + '_> =
+                        Box::new(std::iter::empty());
+                }
+            };
+
+            let option_vec_chains = option_vecs.iter().map(|f| {
+                quote! { iter = Box::new(iter.chain(self.#f.iter())); }
+            });
+
+            let option_nested_chains = option_nested.iter().map(|f| {
+                quote! { iter = Box::new(iter.chain(self.#f.iter().flat_map(|v| v.iter_any_collectable()))); }
+            });
+
+            let nested_chains = nested_fields.iter().map(|f| {
+                quote! { iter = Box::new(iter.chain(self.#f.iter_any_collectable())); }
+            });
+
+            quote! {
+                fn iter_any_collectable<'a>(&'a self) -> Box<dyn Iterator<Item = &'a dyn AnyCollectableVec> + 'a> {
+                    #base
+                    #(#option_vec_chains)*
+                    #(#option_nested_chains)*
+                    #(#nested_chains)*
+                    iter
+                }
+            }
+        }
+        _ => quote! {
+            fn iter_any_collectable<'a>(&'a self) -> Box<dyn Iterator<Item = &'a dyn AnyCollectableVec> + 'a> {
+                Box::new(std::iter::empty())
+            }
+        },
+    }
+}
+
+fn get_option_inner_type(ty: &Type) -> Option<&Type> {
+    if let Type::Path(type_path) = ty
+        && let Some(segment) = type_path.path.segments.last()
+        && segment.ident == "Option"
+        && let syn::PathArguments::AngleBracketed(args) = &segment.arguments
+        && let Some(syn::GenericArgument::Type(inner_ty)) = args.args.first()
+    {
+        return Some(inner_ty);
+    }
+    None
+}