diff --git a/crates/brk_computer/examples/computer.rs b/crates/brk_computer/examples/computer.rs index 48325a32c..0e3886210 100644 --- a/crates/brk_computer/examples/computer.rs +++ b/crates/brk_computer/examples/computer.rs @@ -9,7 +9,6 @@ use brk_error::Result; use brk_fetcher::Fetcher; use brk_indexer::Indexer; use brk_parser::Parser; -use brk_traversable::Traversable; use vecdb::Exit; pub fn main() -> Result<()> { diff --git a/crates/brk_computer/src/grouped/from_dateindex.rs b/crates/brk_computer/src/grouped/from_dateindex.rs index 148eac949..e22253266 100644 --- a/crates/brk_computer/src/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/grouped/from_dateindex.rs @@ -11,7 +11,7 @@ use crate::{Indexes, grouped::LazyVecsBuilder, indexes}; use super::{ComputedType, EagerVecsBuilder, Source, VecBuilderOptions}; -#[derive(Clone, Traversable, Allocative)] +#[derive(Clone, Allocative)] pub struct ComputedVecsFromDateIndex where T: ComputedType + PartialOrd, @@ -144,3 +144,42 @@ where Ok(()) } } + +impl Traversable for ComputedVecsFromDateIndex +where + T: ComputedType, +{ + fn to_tree_node(&self) -> brk_traversable::TreeNode { + brk_traversable::TreeNode::List( + [ + self.dateindex.as_ref().map(|nested| nested.to_tree_node()), + Some(self.dateindex_extra.to_tree_node()), + Some(self.weekindex.to_tree_node()), + Some(self.monthindex.to_tree_node()), + Some(self.quarterindex.to_tree_node()), + Some(self.semesterindex.to_tree_node()), + Some(self.yearindex.to_tree_node()), + Some(self.decadeindex.to_tree_node()), + ] + .into_iter() + .flatten() + .collect(), + ) + .collect_unique_leaves() + } + + fn iter_any_collectable(&self) -> impl Iterator { + let mut regular_iter: Box> = + Box::new(self.dateindex_extra.iter_any_collectable()); + regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_collectable())); + if let Some(ref x) = self.dateindex { + regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable())); + } + regular_iter + } +} diff --git a/crates/brk_computer/src/grouped/from_height.rs b/crates/brk_computer/src/grouped/from_height.rs index 3dd3983c6..219cbbc96 100644 --- a/crates/brk_computer/src/grouped/from_height.rs +++ b/crates/brk_computer/src/grouped/from_height.rs @@ -16,7 +16,7 @@ use crate::{ use super::{ComputedType, EagerVecsBuilder, VecBuilderOptions}; -#[derive(Clone, Traversable, Allocative)] +#[derive(Clone, Allocative)] pub struct ComputedVecsFromHeight where T: ComputedType + PartialOrd, @@ -200,3 +200,46 @@ where Ok(()) } } + +impl Traversable for ComputedVecsFromHeight +where + T: ComputedType, +{ + fn to_tree_node(&self) -> brk_traversable::TreeNode { + brk_traversable::TreeNode::List( + [ + self.height.as_ref().map(|nested| nested.to_tree_node()), + Some(self.height_extra.to_tree_node()), + Some(self.dateindex.to_tree_node()), + Some(self.weekindex.to_tree_node()), + Some(self.difficultyepoch.to_tree_node()), + Some(self.monthindex.to_tree_node()), + Some(self.quarterindex.to_tree_node()), + Some(self.semesterindex.to_tree_node()), + Some(self.yearindex.to_tree_node()), + Some(self.decadeindex.to_tree_node()), + ] + .into_iter() + .flatten() + .collect(), + ) + .collect_unique_leaves() + } + + fn iter_any_collectable(&self) -> impl Iterator { + let mut regular_iter: Box> = + Box::new(self.height_extra.iter_any_collectable()); + regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_collectable())); + if let Some(ref x) = self.height { + regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable())); + } + regular_iter + } +} diff --git a/crates/brk_computer/src/grouped/from_height_strict.rs b/crates/brk_computer/src/grouped/from_height_strict.rs index 5c0078f28..67dc3ee2f 100644 --- a/crates/brk_computer/src/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/grouped/from_height_strict.rs @@ -8,7 +8,7 @@ use crate::{Indexes, indexes}; use super::{ComputedType, EagerVecsBuilder, VecBuilderOptions}; -#[derive(Clone, Traversable)] +#[derive(Clone)] pub struct ComputedVecsFromHeightStrict where T: ComputedType + PartialOrd, @@ -83,3 +83,29 @@ where Ok(()) } } + +impl Traversable for ComputedVecsFromHeightStrict +where + T: ComputedType, +{ + fn to_tree_node(&self) -> brk_traversable::TreeNode { + brk_traversable::TreeNode::List( + [ + Some(self.height.to_tree_node()), + Some(self.height_extra.to_tree_node()), + Some(self.difficultyepoch.to_tree_node()), + ] + .into_iter() + .flatten() + .collect(), + ) + .collect_unique_leaves() + } + fn iter_any_collectable(&self) -> impl Iterator { + let mut regular_iter: Box> = + Box::new(self.height.iter_any_collectable()); + regular_iter = Box::new(regular_iter.chain(self.height_extra.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_collectable())); + regular_iter + } +} diff --git a/crates/brk_computer/src/grouped/from_txindex.rs b/crates/brk_computer/src/grouped/from_txindex.rs index 3fa6ab673..36df68c4b 100644 --- a/crates/brk_computer/src/grouped/from_txindex.rs +++ b/crates/brk_computer/src/grouped/from_txindex.rs @@ -19,7 +19,7 @@ use crate::{ use super::{ComputedType, EagerVecsBuilder, VecBuilderOptions}; -#[derive(Clone, Traversable, Allocative)] +#[derive(Clone, Allocative)] pub struct ComputedVecsFromTxindex where T: ComputedType + PartialOrd, @@ -587,3 +587,46 @@ impl ComputedVecsFromTxindex { self.compute_after_height(indexes, starting_indexes, exit) } } + +impl Traversable for ComputedVecsFromTxindex +where + T: ComputedType, +{ + fn to_tree_node(&self) -> brk_traversable::TreeNode { + brk_traversable::TreeNode::List( + [ + self.txindex.as_ref().map(|nested| nested.to_tree_node()), + Some(self.height.to_tree_node()), + Some(self.dateindex.to_tree_node()), + Some(self.weekindex.to_tree_node()), + Some(self.difficultyepoch.to_tree_node()), + Some(self.monthindex.to_tree_node()), + Some(self.quarterindex.to_tree_node()), + Some(self.semesterindex.to_tree_node()), + Some(self.yearindex.to_tree_node()), + Some(self.decadeindex.to_tree_node()), + ] + .into_iter() + .flatten() + .collect(), + ) + .collect_unique_leaves() + } + + fn iter_any_collectable(&self) -> impl Iterator { + let mut regular_iter: Box> = + Box::new(self.height.iter_any_collectable()); + regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_collectable())); + regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_collectable())); + if let Some(ref x) = self.txindex { + regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable())); + } + regular_iter + } +} diff --git a/crates/brk_computer/src/stateful/address_cohorts.rs b/crates/brk_computer/src/stateful/address_cohorts.rs index 78e567bf8..bc226e8a0 100644 --- a/crates/brk_computer/src/stateful/address_cohorts.rs +++ b/crates/brk_computer/src/stateful/address_cohorts.rs @@ -470,7 +470,7 @@ impl Vecs { vecs, by_size_range .iter() - .filter(|other| other.includes(filter)) + .filter(|Filtered(other, _)| filter.includes(other)) .map(Filtered::t) .collect::>(), ) @@ -484,7 +484,7 @@ impl Vecs { vecs, by_size_range .iter() - .filter(|other| other.includes(filter)) + .filter(|Filtered(other, _)| filter.includes(other)) .map(Filtered::t) .collect::>(), ) diff --git a/crates/brk_computer/src/stateful/utxo_cohorts.rs b/crates/brk_computer/src/stateful/utxo_cohorts.rs index e55896f24..feb7a3bf9 100644 --- a/crates/brk_computer/src/stateful/utxo_cohorts.rs +++ b/crates/brk_computer/src/stateful/utxo_cohorts.rs @@ -1647,7 +1647,7 @@ impl Vecs { vecs, by_date_range .iter() - .filter(|other| other.includes(filter)) + .filter(|Filtered(other, _)| filter.includes(other)) .map(Filtered::t) .collect::>(), ) @@ -1657,7 +1657,7 @@ impl Vecs { vecs, by_date_range .iter() - .filter(|other| other.includes(filter)) + .filter(|Filtered(other, _)| filter.includes(other)) .map(Filtered::t) .collect::>(), ) @@ -1667,7 +1667,7 @@ impl Vecs { vecs, by_date_range .iter() - .filter(|other| other.includes(filter)) + .filter(|Filtered(other, _)| filter.includes(other)) .map(Filtered::t) .collect::>(), ) @@ -1677,7 +1677,7 @@ impl Vecs { vecs, by_size_range .iter() - .filter(|other| other.includes(filter)) + .filter(|Filtered(other, _)| filter.includes(other)) .map(Filtered::t) .collect::>(), ) @@ -1687,7 +1687,7 @@ impl Vecs { vecs, by_size_range .iter() - .filter(|other| other.includes(filter)) + .filter(|Filtered(other, _)| filter.includes(other)) .map(Filtered::t) .collect::>(), ) diff --git a/crates/brk_server/src/api/metrics/mod.rs b/crates/brk_server/src/api/metrics/mod.rs index 377ada6c4..d4dc195a7 100644 --- a/crates/brk_server/src/api/metrics/mod.rs +++ b/crates/brk_server/src/api/metrics/mod.rs @@ -7,6 +7,11 @@ use axum::{ }; use brk_interface::{Index, PaginationParam, Params, ParamsDeprec, ParamsOpt}; +use crate::{ + VERSION, + extended::{HeaderMapExtended, ResponseExtended}, +}; + use super::AppState; mod data; @@ -37,9 +42,27 @@ impl ApiMetricsRoutes for Router { ) .route( "/api/metrics/catalog", - get(async |State(app_state): State| -> Response { - Json(app_state.interface.get_metrics_catalog()).into_response() - }), + get( + async |headers: HeaderMap, State(app_state): State| -> Response { + let etag = VERSION; + + if headers + .get_if_none_match() + .is_some_and(|prev_etag| etag == prev_etag) + { + return Response::new_not_modified(); + } + + let mut response = + Json(app_state.interface.get_metrics_catalog()).into_response(); + + let headers = response.headers_mut(); + headers.insert_cors(); + headers.insert_etag(etag); + + response + }, + ), ) .route( "/api/metrics/list", diff --git a/crates/brk_structs/src/groups/address.rs b/crates/brk_structs/src/groups/address.rs index 258fda992..5b8145300 100644 --- a/crates/brk_structs/src/groups/address.rs +++ b/crates/brk_structs/src/groups/address.rs @@ -4,7 +4,7 @@ use crate::Filtered; use super::{ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount}; -#[derive(Default, Clone, Traversable)] +#[derive(Default, Clone)] pub struct AddressGroups { pub ge_amount: ByGreatEqualAmount, pub amount_range: ByAmountRange, @@ -46,3 +46,38 @@ impl From> for AddressGroups> { } } } + +impl Traversable for AddressGroups +where + ByGreatEqualAmount: brk_traversable::Traversable, + ByAmountRange: brk_traversable::Traversable, + ByLowerThanAmount: brk_traversable::Traversable, + T: Send + Sync, +{ + fn to_tree_node(&self) -> brk_traversable::TreeNode { + brk_traversable::TreeNode::Branch( + [ + (String::from("ge_amount"), self.ge_amount.to_tree_node()), + ( + String::from("amount_range"), + self.amount_range.to_tree_node(), + ), + (String::from("lt_amount"), self.lt_amount.to_tree_node()), + ] + .into(), + ) + } + + fn iter_any_collectable(&self) -> impl Iterator { + [ + Box::new(self.ge_amount.iter_any_collectable()) + as Box>, + Box::new(self.amount_range.iter_any_collectable()) + as Box>, + Box::new(self.lt_amount.iter_any_collectable()) + as Box>, + ] + .into_iter() + .flatten() + } +} diff --git a/crates/brk_traversable/src/lib.rs b/crates/brk_traversable/src/lib.rs index 2d24bbac6..203d40ec7 100644 --- a/crates/brk_traversable/src/lib.rs +++ b/crates/brk_traversable/src/lib.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, BTreeSet}, fmt::Debug, }; @@ -16,13 +16,42 @@ pub trait Traversable { fn iter_any_collectable(&self) -> impl Iterator; } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] #[serde(untagged)] pub enum TreeNode { - Branch(HashMap), + Branch(BTreeMap), + List(Vec), Leaf(String), } +impl TreeNode { + pub fn collect_unique_leaves(self) -> TreeNode { + let mut out = BTreeSet::new(); + + fn recurse(n: TreeNode, out: &mut BTreeSet) { + match n { + TreeNode::Leaf(s) => { + out.insert(s); + } + TreeNode::Branch(map) => { + map.into_values().for_each(|n| recurse(n, out)); + } + TreeNode::List(vec) => { + vec.into_iter().for_each(|n| recurse(n, out)); + } + } + } + + recurse(self, &mut out); + + match out.len() { + 0 => TreeNode::List(vec![]), + 1 => TreeNode::Leaf(out.into_iter().next().unwrap()), + _ => TreeNode::List(out.into_iter().map(TreeNode::Leaf).collect()), + } + } +} + impl Traversable for RawVec where I: StoredIndex, @@ -169,7 +198,7 @@ impl Traversable for Option { fn to_tree_node(&self) -> TreeNode { match self { Some(inner) => inner.to_tree_node(), - None => TreeNode::Branch(HashMap::new()), + None => TreeNode::Branch(BTreeMap::new()), } } diff --git a/crates/brk_traversable_derive/src/lib.rs b/crates/brk_traversable_derive/src/lib.rs index 5a64b6577..46f2a82df 100644 --- a/crates/brk_traversable_derive/src/lib.rs +++ b/crates/brk_traversable_derive/src/lib.rs @@ -14,12 +14,12 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream { match &data.fields { Fields::Unnamed(fields) if fields.unnamed.len() == 1 => { // Special case for single-field tuple structs - just delegate - let generic_params = generics.type_params().map(|p| &p.ident); + let generic_params: Vec<_> = generics.type_params().map(|p| &p.ident).collect(); let original_predicates = &generics.where_clause.as_ref().map(|w| &w.predicates); let where_clause = - if original_predicates.is_some() || generics.type_params().count() > 0 { + if original_predicates.is_some() || !generic_params.is_empty() { quote! { where #(#generic_params: Send + Sync,)* @@ -29,7 +29,7 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream { quote! {} }; - quote! { + return TokenStream::from(quote! { impl #impl_generics Traversable for #name #ty_generics #where_clause { @@ -41,37 +41,49 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream { self.0.iter_any_collectable() } } - } + }); } _ => { // Normal struct with named fields let field_traversals = generate_field_traversals(&data.fields); let iterator_impl = generate_iterator_impl(&data.fields); - // Collect field types that need to implement Traversable - let field_types = if let Fields::Named(named_fields) = &data.fields { - named_fields - .named - .iter() - .filter(|f| matches!(f.vis, syn::Visibility::Public(_))) - .filter(|f| !has_skip_attribute(f)) - .map(|f| &f.ty) - .collect::>() - } else { - Vec::new() - }; + let generic_params: Vec<_> = generics.type_params().map(|p| &p.ident).collect(); + + let generics_needing_traversable = + if let Fields::Named(named_fields) = &data.fields { + let mut used = std::collections::BTreeSet::new(); + + for field in named_fields.named.iter() { + if !should_process_field(field) { + continue; + } + + if let Type::Path(type_path) = &field.ty + && type_path.path.segments.len() == 1 + && let Some(seg) = type_path.path.segments.first() + && seg.arguments.is_empty() + && let Some(pos) = + generic_params.iter().position(|g| g == &&seg.ident) + { + used.insert(generic_params[pos]); + } + } + used.into_iter().collect::>() + } else { + Vec::new() + }; - let generic_params = generics.type_params().map(|p| &p.ident); let original_predicates = &generics.where_clause.as_ref().map(|w| &w.predicates); - let where_clause = if !field_types.is_empty() + let where_clause = if !generics_needing_traversable.is_empty() || original_predicates.is_some() - || generics.type_params().count() > 0 + || !generic_params.is_empty() { quote! { where - #(#field_types: brk_traversable::Traversable,)* + #(#generics_needing_traversable: brk_traversable::Traversable,)* #(#generic_params: Send + Sync,)* #original_predicates } @@ -93,12 +105,23 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream { } } } - _ => panic!("Traversable can only be derived for structs"), + _ => { + return syn::Error::new_spanned( + &input.ident, + "Traversable can only be derived for structs", + ) + .to_compile_error() + .into(); + } }; TokenStream::from(traverse_impl) } +fn should_process_field(field: &syn::Field) -> bool { + matches!(field.vis, syn::Visibility::Public(_)) && !has_skip_attribute(field) +} + fn generate_field_traversals(fields: &Fields) -> proc_macro2::TokenStream { match fields { Fields::Named(fields) => { @@ -106,7 +129,7 @@ fn generate_field_traversals(fields: &Fields) -> proc_macro2::TokenStream { let field_name = f.ident.as_ref()?; let field_name_str = field_name.to_string(); - if has_skip_attribute(f) || !matches!(f.vis, syn::Visibility::Public(_)) { + if !should_process_field(f) { return None; } @@ -122,12 +145,16 @@ fn generate_field_traversals(fields: &Fields) -> proc_macro2::TokenStream { }); quote! { - return brk_traversable::TreeNode::Branch( - [#(#entries,)*] - .into_iter() - .flatten() - .collect() - ); + let collected: std::collections::BTreeMap<_, _> = [#(#entries,)*] + .into_iter() + .flatten() + .collect(); + + return if collected.len() == 1 { + collected.into_values().next().unwrap() + } else { + brk_traversable::TreeNode::Branch(collected) + }; } } _ => quote! {}, @@ -152,11 +179,7 @@ fn generate_iterator_impl(fields: &Fields) -> proc_macro2::TokenStream { for field in fields.named.iter() { if let Some(field_name) = &field.ident { - if !matches!(field.vis, syn::Visibility::Public(_)) { - continue; - } - - if has_skip_attribute(field) { + if !should_process_field(field) { continue; } @@ -175,41 +198,47 @@ fn generate_iterator_impl(fields: &Fields) -> proc_macro2::TokenStream { } } } else { - let regular_part = if !regular_fields.is_empty() { + let (init_part, chain_part) = if !regular_fields.is_empty() { let first = regular_fields.first().unwrap(); let rest = ®ular_fields[1..]; - - quote! { - let mut regular_iter: Box> = - Box::new(self.#first.iter_any_collectable()); - #(regular_iter = Box::new(regular_iter.chain(self.#rest.iter_any_collectable()));)* - } + ( + quote! { + let mut regular_iter: Box> = + Box::new(self.#first.iter_any_collectable()); + }, + quote! { + #(regular_iter = Box::new(regular_iter.chain(self.#rest.iter_any_collectable()));)* + }, + ) } else { - quote! { - let regular_iter = std::iter::empty(); - } + ( + quote! { + let mut regular_iter: Box> = + Box::new(std::iter::empty()); + }, + quote! {}, + ) }; let option_part = if !option_fields.is_empty() { - quote! { - let option_iter = [ - #(self.#option_fields.as_ref().map(|x| Box::new(x.iter_any_collectable()) as Box>),)* - ] - .into_iter() - .flatten() - .flatten(); - } + let chains = option_fields.iter().map(|f| { + quote! { + if let Some(ref x) = self.#f { + regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable())); + } + } + }); + quote! { #(#chains)* } } else { - quote! { - let option_iter = std::iter::empty(); - } + quote! {} }; quote! { fn iter_any_collectable(&self) -> impl Iterator { - #regular_part + #init_part + #chain_part #option_part - regular_iter.chain(option_iter) + regular_iter } } } diff --git a/docs/TODO.md b/docs/TODO.md index 1f510754f..6f7e47f5e 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -92,6 +92,8 @@ - _STORE_ - FEAT: save height and version in one file - _STRUCTS_ + - _TRAVERSABLE_ + - UX: Improve `tree_node`, it's too verbose at the moment - _GLOBAL_ - PERF: https://davidlattimore.github.io/posts/2025/09/02/rustforge-wild-performance-tricks.html - __DOCS__