Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-20 15:38:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: serialization optimizations for faster responses

Browse Source
This commit is contained in:
nym21
2025-09-20 18:42:15 +02:00
parent 23f6397a97
commit 2c5b502da9
90 changed files with 915 additions and 460 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_interface/Cargo.toml
-1
View File
@@ -23,5 +23,4 @@ schemars = "1.0.4"
serde = { workspace = true }
serde_json = { workspace = true }
serde_with = "3.14.0"
tabled = "0.20.0"
nucleo-matcher = "0.3.1"
crates/brk_interface/README.md
+5 -33
View File
@@ -7,13 +7,13 @@ Unified data query and formatting interface for Bitcoin datasets with intelligen
## Overview
This crate provides a high-level interface for querying and formatting data from BRK's indexer and computer components. It offers intelligent vector search with fuzzy matching, parameter validation, range queries, and multi-format output (JSON, CSV, TSV, Markdown) with efficient caching and pagination support.
This crate provides a high-level interface for querying and formatting data from BRK's indexer and computer components. It offers intelligent vector search with fuzzy matching, parameter validation, range queries, and multi-format output (JSON, CSV) with efficient caching and pagination support.
**Key Features:**
- Unified query interface across indexer and computer data sources
- Intelligent search with fuzzy matching and helpful error messages
- Multi-format output: JSON, CSV, TSV, Markdown with proper formatting
- Multi-format output: JSON, CSV with proper formatting
- Range-based data queries with flexible from/to parameters
- Comprehensive pagination support for large datasets
- Schema validation with JSON Schema generation for API documentation
@@ -66,7 +66,7 @@ println!("{}", output);
- **`Interface<'a>`**: Main query interface coordinating indexer and computer access
- **`Params`**: Query parameters including index, IDs, range, and formatting options
- **`Index`**: Enumeration of available data indexes (Height, Date, Address, etc.)
- **`Format`**: Output format specification (JSON, CSV, TSV, MD)
- **`Format`**: Output format specification (JSON, CSV)
- **`Output`**: Formatted query results with multiple value types
### Key Methods
@@ -146,31 +146,6 @@ match interface.search_and_format(params)? {
}
```
### Multi-Vector Query with Markdown Table
```rust
use brk_interface::{Interface, Params, Index, Format};
// Query multiple vectors and format as table
let params = Params {
index: Index::Height,
ids: vec![
"height-to-blockhash".to_string(),
"height-to-timestamp".to_string(),
"height-to-difficulty".to_string()
].into(),
from: Some(800000),
to: Some(800005),
format: Some(Format::MD),
..Default::default()
};
match interface.search_and_format(params)? {
Output::MD(table) => println!("{}", table),
_ => unreachable!(),
}
```
### Intelligent Error Handling
```rust
@@ -219,11 +194,10 @@ The interface acts as a bridge between:
- List: Array of values
- Matrix: Array of arrays for multi-vector queries
**Tabular Output (CSV/TSV/MD):**
**CSV Output:**
- Column headers from vector IDs
- Row-wise data iteration with proper escaping
- Markdown tables use `tabled` crate formatting
### Caching Strategy
@@ -247,15 +221,13 @@ Available indexes include:
- **JSON**: Structured data with nested objects/arrays
- **CSV**: Comma-separated values with proper escaping
- **TSV**: Tab-separated values for import compatibility
- **MD**: Markdown tables for documentation and reports
## Code Analysis Summary
**Main Structure**: `Interface` struct coordinating between `Indexer` and `Computer` data sources \
**Query System**: Parameter-driven search with `Params` struct supporting range queries and formatting options \
**Error Handling**: Intelligent fuzzy matching with cached error messages and helpful suggestions \
**Output Formats**: Multi-format support (JSON, CSV, TSV, Markdown) with proper data serialization \
**Output Formats**: Multi-format support (JSON, CSV) with proper data serialization \
**Caching**: `quick_cache` integration for error messages and expensive operations \
**Search Logic**: `nucleo-matcher` fuzzy search for user-friendly vector name resolution \
**Architecture**: Abstraction layer providing unified access to heterogeneous Bitcoin data sources
crates/brk_interface/src/format.rs
+7 -21
View File
@@ -1,40 +1,26 @@
use brk_error::Error;
use schemars::JsonSchema;
use serde::Deserialize;
#[allow(clippy::upper_case_acronyms)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, JsonSchema)]
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Deserialize, JsonSchema)]
#[serde(rename_all = "lowercase")]
pub enum Format {
#[default]
#[serde(alias = "json")]
JSON,
#[serde(alias = "csv")]
CSV,
#[serde(alias = "tsv")]
TSV,
#[serde(alias = "md", alias = "markdown")]
MD,
}
impl TryFrom<Option<String>> for Format {
type Error = Error;
fn try_from(value: Option<String>) -> Result<Self, Self::Error> {
impl From<Option<String>> for Format {
fn from(value: Option<String>) -> Self {
if let Some(value) = value {
let value = value.to_lowercase();
let value = value.as_str();
if value == "md" || value == "markdown" {
Ok(Self::MD)
} else if value == "csv" {
Ok(Self::CSV)
} else if value == "tsv" {
Ok(Self::TSV)
} else if value == "json" {
Ok(Self::JSON)
} else {
Err(Error::Str("Fail"))
if value == "csv" {
return Self::CSV;
}
} else {
Err(Error::Str("Fail"))
}
Self::JSON
}
}
crates/brk_interface/src/index.rs
+1 -1
View File
@@ -5,7 +5,7 @@ use brk_structs::{
DateIndex, DecadeIndex, DifficultyEpoch, EmptyAddressIndex, EmptyOutputIndex, HalvingEpoch,
Height, InputIndex, LoadedAddressIndex, MonthIndex, OpReturnIndex, OutputIndex,
P2AAddressIndex, P2MSOutputIndex, P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex,
P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex, Printable,
P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex, PrintableIndex,
QuarterIndex, SemesterIndex, TxIndex, UnknownOutputIndex, WeekIndex, YearIndex,
};
use schemars::JsonSchema;
crates/brk_interface/src/lib.rs
+51 -63
View File
@@ -12,7 +12,6 @@ use nucleo_matcher::{
pattern::{AtomKind, CaseMatching, Normalization, Pattern},
};
use quick_cache::sync::Cache;
use tabled::settings::Style;
use vecdb::{AnyCollectableVec, AnyStoredVec};
mod deser;
@@ -22,7 +21,6 @@ mod index;
mod output;
mod pagination;
mod params;
mod table;
mod vecs;
pub use format::Format;
@@ -30,7 +28,6 @@ pub use index::Index;
pub use output::{Output, Value};
pub use pagination::{PaginatedIndexParam, PaginationParam};
pub use params::{IdParam, Params, ParamsOpt};
pub use table::Tabled;
use vecs::Vecs;
use crate::vecs::{IdToVec, IndexToVec};
@@ -146,75 +143,66 @@ impl<'a> Interface<'a> {
.unwrap_or_default()
});
let mut values = vecs
.iter()
.map(|(_, vec)| -> Result<Vec<serde_json::Value>> {
Ok(vec.collect_range_serde_json(from, to)?)
})
.collect::<Result<Vec<_>>>()?;
let format = params.format();
if values.is_empty() {
return Ok(Output::default(format));
}
let ids_last_i = vecs.len() - 1;
Ok(match format {
Some(Format::CSV) | Some(Format::TSV) => {
let delimiter = if format == Some(Format::CSV) {
','
} else {
'\t'
};
let mut text = vecs
Format::CSV => {
let headers = vecs.iter().map(|(id, _)| id.as_str()).collect::<Vec<_>>();
let mut values = vecs
.iter()
.map(|(id, _)| id.to_owned())
.collect::<Vec<_>>()
.join(&delimiter.to_string());
.map(|(_, vec)| Ok(vec.collect_range_string(from, to)?))
.collect::<Result<Vec<_>>>()?;
text.push('\n');
let values_len = values.first().unwrap().len();
(0..values_len).for_each(|i| {
let mut line = "".to_string();
values.iter().enumerate().for_each(|(id_i, v)| {
line += &v.get(i).unwrap().to_string();
if id_i == ids_last_i {
line.push('\n');
} else {
line.push(delimiter);
}
});
text += &line;
});
if format == Some(Format::CSV) {
Output::CSV(text)
} else {
Output::TSV(text)
if values.is_empty() {
return Ok(Output::CSV(headers.join(",")));
}
}
Some(Format::MD) => {
let mut table =
values.to_table(vecs.iter().map(|(s, _)| s.to_owned()).collect::<Vec<_>>());
table.with(Style::markdown());
let first_len = values[0].len();
let estimated_size = (headers.len() + values.len() * first_len) * 15;
let mut csv = String::with_capacity(estimated_size);
Output::MD(table.to_string())
}
Some(Format::JSON) | None => {
if values.len() == 1 {
let mut values = values.pop().unwrap();
if values.len() == 1 {
let value = values.pop().unwrap();
Output::Json(Value::Single(value))
} else {
Output::Json(Value::List(values))
csv.push_str(&headers.join(","));
csv.push('\n');
for col_index in 0..first_len {
let mut first = true;
for vec in &mut values {
if col_index < vec.len() {
if !first {
csv.push(',');
}
first = false;
let field = std::mem::take(&mut vec[col_index]);
if field.contains(',') {
csv.push('"');
csv.push_str(&field);
csv.push('"');
} else {
csv.push_str(&field);
}
}
}
csv.push('\n');
}
Output::CSV(csv)
}
Format::JSON => {
let mut values = vecs
.iter()
.map(|(_, vec)| -> Result<Vec<u8>> {
Ok(vec.collect_range_json_bytes(from, to)?)
})
.collect::<Result<Vec<_>>>()?;
if values.is_empty() {
return Ok(Output::default(format));
}
if values.len() == 1 {
Output::Json(Value::List(values.pop().unwrap()))
} else {
Output::Json(Value::Matrix(values))
}
crates/brk_interface/src/output.rs
+38 -28
View File
@@ -1,44 +1,54 @@
use std::fmt;
use serde::Serialize;
use tabled::Tabled as TabledTabled;
use crate::Format;
#[derive(Debug, Serialize)]
#[serde(untagged, rename_all = "lowercase")]
#[derive(Debug)]
pub enum Output {
Json(Value),
CSV(String),
TSV(String),
MD(String),
}
#[derive(Debug, Serialize, TabledTabled)]
#[serde(untagged)]
pub enum Value {
Matrix(Vec<Vec<serde_json::Value>>),
List(Vec<serde_json::Value>),
Single(serde_json::Value),
}
impl Output {
pub fn default(format: Option<Format>) -> Self {
match format {
Some(Format::CSV) => Output::CSV("".to_string()),
Some(Format::TSV) => Output::TSV("".to_string()),
_ => Output::Json(Value::Single(serde_json::Value::Null)),
#[allow(clippy::inherent_to_string)]
pub fn to_string(self) -> String {
match self {
Output::CSV(s) => s,
Output::Json(v) => unsafe { String::from_utf8_unchecked(v.to_vec()) },
}
}
}
impl fmt::Display for Output {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
#[derive(Debug)]
pub enum Value {
Matrix(Vec<Vec<u8>>),
List(Vec<u8>),
}
impl Value {
pub fn to_vec(self) -> Vec<u8> {
match self {
Self::Json(value) => write!(f, "{}", serde_json::to_string_pretty(value).unwrap()),
Self::CSV(string) => write!(f, "{string}"),
Self::TSV(string) => write!(f, "{string}"),
Self::MD(string) => write!(f, "{string}"),
Value::List(v) => v,
Self::Matrix(m) => {
let total_size = m.iter().map(|v| v.len()).sum::<usize>() + m.len() - 1 + 2;
let mut matrix = Vec::with_capacity(total_size);
matrix.push(b'[');
for (i, vec) in m.into_iter().enumerate() {
if i > 0 {
matrix.push(b',');
}
matrix.extend(vec);
}
matrix.push(b']');
matrix
}
}
}
}
impl Output {
pub fn default(format: Format) -> Self {
match format {
Format::CSV => Output::CSV("".to_string()),
Format::JSON => Output::Json(Value::List(b"[]".to_vec())),
}
}
}
crates/brk_interface/src/params.rs
+2 -7
View File
@@ -63,7 +63,7 @@ pub struct ParamsOpt {
#[serde(default)]
/// Format of the output
#[schemars(description = "Format of the output")]
format: Option<Format>,
format: Format,
}
impl ParamsOpt {
@@ -82,11 +82,6 @@ impl ParamsOpt {
self
}
pub fn set_format(mut self, format: Format) -> Self {
self.format.replace(format);
self
}
pub fn from(&self) -> Option<i64> {
self.from
}
@@ -107,7 +102,7 @@ impl ParamsOpt {
self.to
}
pub fn format(&self) -> Option<Format> {
pub fn format(&self) -> Format {
self.format
}
}
crates/brk_interface/src/table.rs
-26
View File
@@ -1,26 +0,0 @@
use tabled::{Table, builder::Builder};
pub trait Tabled {
fn to_table(&self, ids: Vec<String>) -> Table;
}
impl Tabled for Vec<Vec<serde_json::Value>> {
fn to_table(&self, ids: Vec<String>) -> Table {
let mut builder = Builder::default();
builder.push_record(ids);
if let Some(first) = self.first() {
let len = first.len();
(0..len).for_each(|index| {
builder.push_record(
self.iter()
.map(|vec| vec.get(index).unwrap().to_string().replace("\"", "")),
);
});
}
builder.build()
}
}