Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

server: catalog v1

Browse Source
This commit is contained in:
nym21
2025-10-06 12:52:18 +02:00
parent 1c6ece48a8
commit db344749b6
12 changed files with 344 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/brk_computer/examples/computer.rs
View File

@@ -9,7 +9,6 @@ use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_traversable::Traversable;
use vecdb::Exit;
pub fn main() -> Result<()> {

41
crates/brk_computer/src/grouped/from_dateindex.rs
View File

@@ -11,7 +11,7 @@ use crate::{Indexes, grouped::LazyVecsBuilder, indexes};
use super::{ComputedType, EagerVecsBuilder, Source, VecBuilderOptions};
#[derive(Clone, Traversable, Allocative)]
#[derive(Clone, Allocative)]
pub struct ComputedVecsFromDateIndex<T>
where
T: ComputedType + PartialOrd,
@@ -144,3 +144,42 @@ where
Ok(())
}
}
impl<T> Traversable for ComputedVecsFromDateIndex<T>
where
T: ComputedType,
{
fn to_tree_node(&self) -> brk_traversable::TreeNode {
brk_traversable::TreeNode::List(
[
self.dateindex.as_ref().map(|nested| nested.to_tree_node()),
Some(self.dateindex_extra.to_tree_node()),
Some(self.weekindex.to_tree_node()),
Some(self.monthindex.to_tree_node()),
Some(self.quarterindex.to_tree_node()),
Some(self.semesterindex.to_tree_node()),
Some(self.yearindex.to_tree_node()),
Some(self.decadeindex.to_tree_node()),
]
.into_iter()
.flatten()
.collect(),
)
.collect_unique_leaves()
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn vecdb::AnyCollectableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>> =
Box::new(self.dateindex_extra.iter_any_collectable());
regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_collectable()));
if let Some(ref x) = self.dateindex {
regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable()));
}
regular_iter
}
}

45
crates/brk_computer/src/grouped/from_height.rs
View File

@@ -16,7 +16,7 @@ use crate::{
use super::{ComputedType, EagerVecsBuilder, VecBuilderOptions};
#[derive(Clone, Traversable, Allocative)]
#[derive(Clone, Allocative)]
pub struct ComputedVecsFromHeight<T>
where
T: ComputedType + PartialOrd,
@@ -200,3 +200,46 @@ where
Ok(())
}
}
impl<T> Traversable for ComputedVecsFromHeight<T>
where
T: ComputedType,
{
fn to_tree_node(&self) -> brk_traversable::TreeNode {
brk_traversable::TreeNode::List(
[
self.height.as_ref().map(|nested| nested.to_tree_node()),
Some(self.height_extra.to_tree_node()),
Some(self.dateindex.to_tree_node()),
Some(self.weekindex.to_tree_node()),
Some(self.difficultyepoch.to_tree_node()),
Some(self.monthindex.to_tree_node()),
Some(self.quarterindex.to_tree_node()),
Some(self.semesterindex.to_tree_node()),
Some(self.yearindex.to_tree_node()),
Some(self.decadeindex.to_tree_node()),
]
.into_iter()
.flatten()
.collect(),
)
.collect_unique_leaves()
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn vecdb::AnyCollectableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>> =
Box::new(self.height_extra.iter_any_collectable());
regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_collectable()));
if let Some(ref x) = self.height {
regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable()));
}
regular_iter
}
}

28
crates/brk_computer/src/grouped/from_height_strict.rs
View File

@@ -8,7 +8,7 @@ use crate::{Indexes, indexes};
use super::{ComputedType, EagerVecsBuilder, VecBuilderOptions};
#[derive(Clone, Traversable)]
#[derive(Clone)]
pub struct ComputedVecsFromHeightStrict<T>
where
T: ComputedType + PartialOrd,
@@ -83,3 +83,29 @@ where
Ok(())
}
}
impl<T> Traversable for ComputedVecsFromHeightStrict<T>
where
T: ComputedType,
{
fn to_tree_node(&self) -> brk_traversable::TreeNode {
brk_traversable::TreeNode::List(
[
Some(self.height.to_tree_node()),
Some(self.height_extra.to_tree_node()),
Some(self.difficultyepoch.to_tree_node()),
]
.into_iter()
.flatten()
.collect(),
)
.collect_unique_leaves()
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn vecdb::AnyCollectableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>> =
Box::new(self.height.iter_any_collectable());
regular_iter = Box::new(regular_iter.chain(self.height_extra.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_collectable()));
regular_iter
}
}

45
crates/brk_computer/src/grouped/from_txindex.rs
View File

@@ -19,7 +19,7 @@ use crate::{
use super::{ComputedType, EagerVecsBuilder, VecBuilderOptions};
#[derive(Clone, Traversable, Allocative)]
#[derive(Clone, Allocative)]
pub struct ComputedVecsFromTxindex<T>
where
T: ComputedType + PartialOrd,
@@ -587,3 +587,46 @@ impl ComputedVecsFromTxindex<Dollars> {
self.compute_after_height(indexes, starting_indexes, exit)
}
}
impl<T> Traversable for ComputedVecsFromTxindex<T>
where
T: ComputedType,
{
fn to_tree_node(&self) -> brk_traversable::TreeNode {
brk_traversable::TreeNode::List(
[
self.txindex.as_ref().map(|nested| nested.to_tree_node()),
Some(self.height.to_tree_node()),
Some(self.dateindex.to_tree_node()),
Some(self.weekindex.to_tree_node()),
Some(self.difficultyepoch.to_tree_node()),
Some(self.monthindex.to_tree_node()),
Some(self.quarterindex.to_tree_node()),
Some(self.semesterindex.to_tree_node()),
Some(self.yearindex.to_tree_node()),
Some(self.decadeindex.to_tree_node()),
]
.into_iter()
.flatten()
.collect(),
)
.collect_unique_leaves()
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn vecdb::AnyCollectableVec> {
let mut regular_iter: Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>> =
Box::new(self.height.iter_any_collectable());
regular_iter = Box::new(regular_iter.chain(self.dateindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.weekindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.difficultyepoch.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.monthindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.quarterindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.semesterindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.yearindex.iter_any_collectable()));
regular_iter = Box::new(regular_iter.chain(self.decadeindex.iter_any_collectable()));
if let Some(ref x) = self.txindex {
regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable()));
}
regular_iter
}
}

4
crates/brk_computer/src/stateful/address_cohorts.rs
View File

@@ -470,7 +470,7 @@ impl Vecs {
vecs,
by_size_range
.iter()
.filter(|other| other.includes(filter))
.filter(|Filtered(other, _)| filter.includes(other))
.map(Filtered::t)
.collect::<Vec<_>>(),
)
@@ -484,7 +484,7 @@ impl Vecs {
vecs,
by_size_range
.iter()
.filter(|other| other.includes(filter))
.filter(|Filtered(other, _)| filter.includes(other))
.map(Filtered::t)
.collect::<Vec<_>>(),
)

10
crates/brk_computer/src/stateful/utxo_cohorts.rs
View File

@@ -1647,7 +1647,7 @@ impl Vecs {
vecs,
by_date_range
.iter()
.filter(|other| other.includes(filter))
.filter(|Filtered(other, _)| filter.includes(other))
.map(Filtered::t)
.collect::<Vec<_>>(),
)
@@ -1657,7 +1657,7 @@ impl Vecs {
vecs,
by_date_range
.iter()
.filter(|other| other.includes(filter))
.filter(|Filtered(other, _)| filter.includes(other))
.map(Filtered::t)
.collect::<Vec<_>>(),
)
@@ -1667,7 +1667,7 @@ impl Vecs {
vecs,
by_date_range
.iter()
.filter(|other| other.includes(filter))
.filter(|Filtered(other, _)| filter.includes(other))
.map(Filtered::t)
.collect::<Vec<_>>(),
)
@@ -1677,7 +1677,7 @@ impl Vecs {
vecs,
by_size_range
.iter()
.filter(|other| other.includes(filter))
.filter(|Filtered(other, _)| filter.includes(other))
.map(Filtered::t)
.collect::<Vec<_>>(),
)
@@ -1687,7 +1687,7 @@ impl Vecs {
vecs,
by_size_range
.iter()
.filter(|other| other.includes(filter))
.filter(|Filtered(other, _)| filter.includes(other))
.map(Filtered::t)
.collect::<Vec<_>>(),
)

29
crates/brk_server/src/api/metrics/mod.rs
View File

@@ -7,6 +7,11 @@ use axum::{
};
use brk_interface::{Index, PaginationParam, Params, ParamsDeprec, ParamsOpt};
use crate::{
VERSION,
extended::{HeaderMapExtended, ResponseExtended},
};
use super::AppState;
mod data;
@@ -37,9 +42,27 @@ impl ApiMetricsRoutes for Router<AppState> {
)
.route(
"/api/metrics/catalog",
get(async |State(app_state): State<AppState>| -> Response {
Json(app_state.interface.get_metrics_catalog()).into_response()
}),
get(
async |headers: HeaderMap, State(app_state): State<AppState>| -> Response {
let etag = VERSION;
if headers
.get_if_none_match()
.is_some_and(|prev_etag| etag == prev_etag)
{
return Response::new_not_modified();
}
let mut response =
Json(app_state.interface.get_metrics_catalog()).into_response();
let headers = response.headers_mut();
headers.insert_cors();
headers.insert_etag(etag);
response
},
),
)
.route(
"/api/metrics/list",

37
crates/brk_structs/src/groups/address.rs
View File

@@ -4,7 +4,7 @@ use crate::Filtered;
use super::{ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount};
#[derive(Default, Clone, Traversable)]
#[derive(Default, Clone)]
pub struct AddressGroups<T> {
pub ge_amount: ByGreatEqualAmount<T>,
pub amount_range: ByAmountRange<T>,
@@ -46,3 +46,38 @@ impl<T> From<AddressGroups<T>> for AddressGroups<Filtered<T>> {
}
}
}
impl<T> Traversable for AddressGroups<T>
where
ByGreatEqualAmount<T>: brk_traversable::Traversable,
ByAmountRange<T>: brk_traversable::Traversable,
ByLowerThanAmount<T>: brk_traversable::Traversable,
T: Send + Sync,
{
fn to_tree_node(&self) -> brk_traversable::TreeNode {
brk_traversable::TreeNode::Branch(
[
(String::from("ge_amount"), self.ge_amount.to_tree_node()),
(
String::from("amount_range"),
self.amount_range.to_tree_node(),
),
(String::from("lt_amount"), self.lt_amount.to_tree_node()),
]
.into(),
)
}
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn vecdb::AnyCollectableVec> {
[
Box::new(self.ge_amount.iter_any_collectable())
as Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>>,
Box::new(self.amount_range.iter_any_collectable())
as Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>>,
Box::new(self.lt_amount.iter_any_collectable())
as Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>>,
]
.into_iter()
.flatten()
}
}

37
crates/brk_traversable/src/lib.rs
View File

@@ -1,5 +1,5 @@
use std::{
collections::{BTreeMap, HashMap},
collections::{BTreeMap, BTreeSet},
fmt::Debug,
};
@@ -16,13 +16,42 @@ pub trait Traversable {
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn AnyCollectableVec>;
}
#[derive(Debug, Clone, Serialize)]
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
#[serde(untagged)]
pub enum TreeNode {
Branch(HashMap<String, TreeNode>),
Branch(BTreeMap<String, TreeNode>),
List(Vec<TreeNode>),
Leaf(String),
}
impl TreeNode {
pub fn collect_unique_leaves(self) -> TreeNode {
let mut out = BTreeSet::new();
fn recurse(n: TreeNode, out: &mut BTreeSet<String>) {
match n {
TreeNode::Leaf(s) => {
out.insert(s);
}
TreeNode::Branch(map) => {
map.into_values().for_each(|n| recurse(n, out));
}
TreeNode::List(vec) => {
vec.into_iter().for_each(|n| recurse(n, out));
}
}
}
recurse(self, &mut out);
match out.len() {
0 => TreeNode::List(vec![]),
1 => TreeNode::Leaf(out.into_iter().next().unwrap()),
_ => TreeNode::List(out.into_iter().map(TreeNode::Leaf).collect()),
}
}
}
impl<I, T> Traversable for RawVec<I, T>
where
I: StoredIndex,
@@ -169,7 +198,7 @@ impl<T: Traversable> Traversable for Option<T> {
fn to_tree_node(&self) -> TreeNode {
match self {
Some(inner) => inner.to_tree_node(),
None => TreeNode::Branch(HashMap::new()),
None => TreeNode::Branch(BTreeMap::new()),
}
}

141
crates/brk_traversable_derive/src/lib.rs
View File

@@ -14,12 +14,12 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream {
match &data.fields {
Fields::Unnamed(fields) if fields.unnamed.len() == 1 => {
// Special case for single-field tuple structs - just delegate
let generic_params = generics.type_params().map(|p| &p.ident);
let generic_params: Vec<_> = generics.type_params().map(|p| &p.ident).collect();
let original_predicates =
&generics.where_clause.as_ref().map(|w| &w.predicates);
let where_clause =
if original_predicates.is_some() || generics.type_params().count() > 0 {
if original_predicates.is_some() || !generic_params.is_empty() {
quote! {
where
#(#generic_params: Send + Sync,)*
@@ -29,7 +29,7 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream {
quote! {}
};
quote! {
return TokenStream::from(quote! {
impl #impl_generics Traversable for #name #ty_generics
#where_clause
{
@@ -41,37 +41,49 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream {
self.0.iter_any_collectable()
}
}
}
});
}
_ => {
// Normal struct with named fields
let field_traversals = generate_field_traversals(&data.fields);
let iterator_impl = generate_iterator_impl(&data.fields);
// Collect field types that need to implement Traversable
let field_types = if let Fields::Named(named_fields) = &data.fields {
named_fields
.named
.iter()
.filter(|f| matches!(f.vis, syn::Visibility::Public(_)))
.filter(|f| !has_skip_attribute(f))
.map(|f| &f.ty)
.collect::<Vec<_>>()
} else {
Vec::new()
};
let generic_params: Vec<_> = generics.type_params().map(|p| &p.ident).collect();
let generics_needing_traversable =
if let Fields::Named(named_fields) = &data.fields {
let mut used = std::collections::BTreeSet::new();
for field in named_fields.named.iter() {
if !should_process_field(field) {
continue;
}
if let Type::Path(type_path) = &field.ty
&& type_path.path.segments.len() == 1
&& let Some(seg) = type_path.path.segments.first()
&& seg.arguments.is_empty()
&& let Some(pos) =
generic_params.iter().position(|g| g == &&seg.ident)
{
used.insert(generic_params[pos]);
}
}
used.into_iter().collect::<Vec<_>>()
} else {
Vec::new()
};
let generic_params = generics.type_params().map(|p| &p.ident);
let original_predicates =
&generics.where_clause.as_ref().map(|w| &w.predicates);
let where_clause = if !field_types.is_empty()
let where_clause = if !generics_needing_traversable.is_empty()
|| original_predicates.is_some()
|| generics.type_params().count() > 0
|| !generic_params.is_empty()
{
quote! {
where
#(#field_types: brk_traversable::Traversable,)*
#(#generics_needing_traversable: brk_traversable::Traversable,)*
#(#generic_params: Send + Sync,)*
#original_predicates
}
@@ -93,12 +105,23 @@ pub fn derive_traversable(input: TokenStream) -> TokenStream {
}
}
}
_ => panic!("Traversable can only be derived for structs"),
_ => {
return syn::Error::new_spanned(
&input.ident,
"Traversable can only be derived for structs",
)
.to_compile_error()
.into();
}
};
TokenStream::from(traverse_impl)
}
fn should_process_field(field: &syn::Field) -> bool {
matches!(field.vis, syn::Visibility::Public(_)) && !has_skip_attribute(field)
}
fn generate_field_traversals(fields: &Fields) -> proc_macro2::TokenStream {
match fields {
Fields::Named(fields) => {
@@ -106,7 +129,7 @@ fn generate_field_traversals(fields: &Fields) -> proc_macro2::TokenStream {
let field_name = f.ident.as_ref()?;
let field_name_str = field_name.to_string();
if has_skip_attribute(f) || !matches!(f.vis, syn::Visibility::Public(_)) {
if !should_process_field(f) {
return None;
}
@@ -122,12 +145,16 @@ fn generate_field_traversals(fields: &Fields) -> proc_macro2::TokenStream {
});
quote! {
return brk_traversable::TreeNode::Branch(
[#(#entries,)*]
.into_iter()
.flatten()
.collect()
);
let collected: std::collections::BTreeMap<_, _> = [#(#entries,)*]
.into_iter()
.flatten()
.collect();
return if collected.len() == 1 {
collected.into_values().next().unwrap()
} else {
brk_traversable::TreeNode::Branch(collected)
};
}
}
_ => quote! {},
@@ -152,11 +179,7 @@ fn generate_iterator_impl(fields: &Fields) -> proc_macro2::TokenStream {
for field in fields.named.iter() {
if let Some(field_name) = &field.ident {
if !matches!(field.vis, syn::Visibility::Public(_)) {
continue;
}
if has_skip_attribute(field) {
if !should_process_field(field) {
continue;
}
@@ -175,41 +198,47 @@ fn generate_iterator_impl(fields: &Fields) -> proc_macro2::TokenStream {
}
}
} else {
let regular_part = if !regular_fields.is_empty() {
let (init_part, chain_part) = if !regular_fields.is_empty() {
let first = regular_fields.first().unwrap();
let rest = &regular_fields[1..];
quote! {
let mut regular_iter: Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>> =
Box::new(self.#first.iter_any_collectable());
#(regular_iter = Box::new(regular_iter.chain(self.#rest.iter_any_collectable()));)*
}
(
quote! {
let mut regular_iter: Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>> =
Box::new(self.#first.iter_any_collectable());
},
quote! {
#(regular_iter = Box::new(regular_iter.chain(self.#rest.iter_any_collectable()));)*
},
)
} else {
quote! {
let regular_iter = std::iter::empty();
}
(
quote! {
let mut regular_iter: Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>> =
Box::new(std::iter::empty());
},
quote! {},
)
};
let option_part = if !option_fields.is_empty() {
quote! {
let option_iter = [
#(self.#option_fields.as_ref().map(|x| Box::new(x.iter_any_collectable()) as Box<dyn Iterator<Item = &dyn vecdb::AnyCollectableVec>>),)*
]
.into_iter()
.flatten()
.flatten();
}
let chains = option_fields.iter().map(|f| {
quote! {
if let Some(ref x) = self.#f {
regular_iter = Box::new(regular_iter.chain(x.iter_any_collectable()));
}
}
});
quote! { #(#chains)* }
} else {
quote! {
let option_iter = std::iter::empty();
}
quote! {}
};
quote! {
fn iter_any_collectable(&self) -> impl Iterator<Item = &dyn vecdb::AnyCollectableVec> {
#regular_part
#init_part
#chain_part
#option_part
regular_iter.chain(option_iter)
regular_iter
}
}
}

2
docs/TODO.md
View File

@@ -92,6 +92,8 @@
- _STORE_
- FEAT: save height and version in one file
- _STRUCTS_
- _TRAVERSABLE_
- UX: Improve `tree_node`, it's too verbose at the moment
- _GLOBAL_
- PERF: https://davidlattimore.github.io/posts/2025/09/02/rustforge-wild-performance-tricks.html
- __DOCS__