global: snapshot

This commit is contained in:
nym21
2025-12-20 10:16:06 +01:00
parent e134ed11a9
commit 4a0ce6337f
21 changed files with 2164 additions and 468 deletions

29
Cargo.lock generated
View File

@@ -568,6 +568,7 @@ version = "0.1.0-alpha.0"
dependencies = [ dependencies = [
"brk_query", "brk_query",
"brk_types", "brk_types",
"oas3",
"schemars", "schemars",
"serde_json", "serde_json",
"vecdb", "vecdb",
@@ -1203,6 +1204,7 @@ version = "0.1.0-alpha.0"
dependencies = [ dependencies = [
"aide", "aide",
"axum", "axum",
"brk_binder",
"brk_computer", "brk_computer",
"brk_error", "brk_error",
"brk_fetcher", "brk_fetcher",
@@ -1364,9 +1366,9 @@ dependencies = [
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.2.49" version = "1.2.50"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" checksum = "9f50d563227a1c37cc0a263f64eca3334388c01c5e4c4861a9def205c614383c"
dependencies = [ dependencies = [
"find-msvc-tools", "find-msvc-tools",
"jobserver", "jobserver",
@@ -3219,6 +3221,23 @@ dependencies = [
"syn 2.0.111", "syn 2.0.111",
] ]
[[package]]
name = "oas3"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16f67c885c7b19aaf652e84102035258ecb4e0425a4f71037e187798e367bd87"
dependencies = [
"derive_more",
"http",
"log",
"once_cell",
"regex",
"semver",
"serde",
"serde_json",
"url",
]
[[package]] [[package]]
name = "objc2" name = "objc2"
version = "0.6.3" version = "0.6.3"
@@ -4198,7 +4217,7 @@ dependencies = [
[[package]] [[package]]
name = "rawdb" name = "rawdb"
version = "0.4.3" version = "0.4.4"
dependencies = [ dependencies = [
"libc", "libc",
"log", "log",
@@ -5388,7 +5407,7 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]] [[package]]
name = "vecdb" name = "vecdb"
version = "0.4.3" version = "0.4.4"
dependencies = [ dependencies = [
"ctrlc", "ctrlc",
"log", "log",
@@ -5407,7 +5426,7 @@ dependencies = [
[[package]] [[package]]
name = "vecdb_derive" name = "vecdb_derive"
version = "0.4.3" version = "0.4.4"
dependencies = [ dependencies = [
"quote", "quote",
"syn 2.0.111", "syn 2.0.111",

1
crates/brk_binder/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
clients/

View File

@@ -11,6 +11,7 @@ build = "build.rs"
[dependencies] [dependencies]
brk_query = { workspace = true } brk_query = { workspace = true }
brk_types = { workspace = true } brk_types = { workspace = true }
oas3 = "0.20"
schemars = { workspace = true } schemars = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
vecdb = { workspace = true } vecdb = { workspace = true }

View File

@@ -8,27 +8,29 @@ Generate typed API clients for **Rust, JavaScript, and Python** with:
## Current State ## Current State
### What Exists ### What's Working ✅
1. **`js.rs`**: Generates compressed metric catalogs for JS (constants only, no HTTP client) 1. **JS + JSDoc generator**: Generates `client.js` with full JSDoc type annotations
2. **`tree.rs`**: (kept for reference, not compiled) Brainstorming output for pattern extraction 2. **Python generator**: Generates `client.py` with type hints and httpx
3. **`generator/`**: Module structure for client generation 3. **Rust generator**: Generates `client.rs` with strong typing and reqwest
- `types.rs`: Intermediate representation (`ClientMetadata`, `MetricInfo`, `IndexPattern`, `schema_to_jsdoc`) 4. **schemars integration**: JSON schemas embedded in `MetricLeafWithSchema` for type info
- `rust.rs`: Rust client generation (stub) 5. **Tree navigation**: `client.tree.blocks.difficulty.fetch()` pattern
- `javascript.rs`: JavaScript + JSDoc client generation ✅ IMPLEMENTED 6. **OpenAPI integration**: All GET endpoints generate typed methods
- `python.rs`: Python client generation (stub) 7. **Server integration**: brk_server calls brk_binder on startup (when clients/ dir exists)
### What's Working ### Generated Output
- **JS + JSDoc generator**: Generates `client.js` with full JSDoc type annotations When `crates/brk_binder/clients/` directory exists, running the server generates:
- **schemars integration**: JSON schemas embedded in `MetricLeafWithSchema` for type info
- **Tree navigation**: `client.tree.blocks.difficulty.fetch()` pattern
### What's Missing ```
crates/brk_binder/clients/
- OpenAPI integration for non-metric endpoints ├── javascript/
- Python client implementation │ └── client.js # JS + JSDoc with tree + API methods
- Rust client implementation ├── python/
│ └── client.py # Python with type hints + httpx
└── rust/
└── client.rs # Rust with reqwest + strong typing
```
## Target Architecture ## Target Architecture
@@ -56,13 +58,13 @@ const data = await client.tree.supply.active.by_date.fetch();
```python ```python
# Python # Python
client = BrkClient("http://localhost:3000") client = BrkClient("http://localhost:3000")
data = await client.tree.supply.active.by_date.fetch() data = client.tree.supply.active.by_date.fetch()
``` ```
```rust ```rust
// Rust // Rust
let client = BrkClient::new("http://localhost:3000"); let client = BrkClient::new("http://localhost:3000")?;
let data = client.tree.supply.active.by_date.fetch().await?; let data = client.tree().supply.active.by_date.fetch()?;
``` ```
## Implementation Details ## Implementation Details
@@ -78,19 +80,11 @@ Each tree leaf becomes a "smart node" holding a client reference:
* @template T * @template T
*/ */
class MetricNode { class MetricNode {
/**
* @param {BrkClientBase} client
* @param {string} path
*/
constructor(client, path) { constructor(client, path) {
this._client = client; this._client = client;
this._path = path; this._path = path;
} }
/**
* Fetch the metric value
* @returns {Promise<T>}
*/
async fetch() { async fetch() {
return this._client.get(this._path); return this._client.get(this._path);
} }
@@ -100,30 +94,30 @@ class MetricNode {
```python ```python
# Python # Python
class MetricNode(Generic[T]): class MetricNode(Generic[T]):
def __init__(self, client: BrkClient, path: str): def __init__(self, client: BrkClientBase, path: str):
self._client = client self._client = client
self._path = path self._path = path
async def fetch(self) -> T: def fetch(self) -> T:
return await self._client.get(self._path) return self._client.get(self._path)
``` ```
```rust ```rust
// Rust // Rust
pub struct MetricNode<T> { pub struct MetricNode<'a, T> {
client: Arc<BrkClient>, client: &'a BrkClientBase,
path: &'static str, path: &'static str,
_phantom: PhantomData<T>, _marker: PhantomData<T>,
} }
impl<T: DeserializeOwned> MetricNode<T> { impl<'a, T: DeserializeOwned> MetricNode<'a, T> {
pub async fn fetch(&self) -> Result<T, BrkError> { pub fn fetch(&self) -> Result<T> {
self.client.get(&self.path).await self.client.get(self.path)
} }
} }
``` ```
### Pattern Reuse (from tree.rs) ### Pattern Reuse
To avoid 20k+ individual types, reuse structural patterns: To avoid 20k+ individual types, reuse structural patterns:
@@ -142,17 +136,6 @@ struct Supply {
} }
``` ```
### Rust Client: Using brk_types
The Rust client should import `brk_types` rather than generating duplicate types:
```rust
use brk_types::{Height, Sats, DateIndex, ...};
// Response types come from brk_types
pub struct MetricNode<T: brk_types::Metric> { ... }
```
## Type Discovery Solution ✅ IMPLEMENTED ## Type Discovery Solution ✅ IMPLEMENTED
### The Problem ### The Problem
@@ -163,45 +146,9 @@ Type information was erased at runtime because metrics are stored as `&dyn AnyEx
Use `std::any::type_name::<T>()` with caching to extract short type names. Use `std::any::type_name::<T>()` with caching to extract short type names.
> **Note**: Unlike `PrintableIndex` which needs `to_possible_strings()` for parsing from
> multiple string representations, for values we only need output, so `type_name` suffices.
#### Implementation (vecdb) #### Implementation (vecdb)
Added `short_type_name<T>()` helper in `traits/printable.rs`: Added `short_type_name<T>()` helper and `value_type_to_string()` to `AnyVec` trait.
```rust
pub fn short_type_name<T: 'static>() -> &'static str {
static CACHE: OnceLock<Mutex<HashMap<&'static str, &'static str>>> = OnceLock::new();
let full: &'static str = std::any::type_name::<T>();
// ... caching logic, extracts "Sats" from "brk_types::sats::Sats"
}
```
Added `value_type_to_string()` to `AnyVec` trait in `traits/any.rs`:
```rust
pub trait AnyVec: Send + Sync {
// ... existing methods
fn value_type_to_string(&self) -> &'static str;
}
```
Implemented in all vec variants:
- `variants/eager/mod.rs`
- `variants/lazy/from1/mod.rs`, `from2/mod.rs`, `from3/mod.rs`
- `variants/raw/inner/mod.rs`
- `variants/compressed/inner/mod.rs`
- `variants/macros.rs` (for wrapper types)
```rust
fn value_type_to_string(&self) -> &'static str {
short_type_name::<V::T>()
}
```
**No changes needed to brk_types** - works automatically for all types.
### Result ### Result
@@ -219,26 +166,11 @@ for (metric_name, index_to_vec) in &vecs.metric_to_index_to_vec {
} }
``` ```
This enables fully typed client generation.
## TreeNode Enhancement ✅ IMPLEMENTED ## TreeNode Enhancement ✅ IMPLEMENTED
### The Problem
`TreeNode::Leaf` originally held just a `String` (the metric name), losing type and index information.
### The Solution
Changed `TreeNode::Leaf(String)` to `TreeNode::Leaf(MetricLeafWithSchema)` where: Changed `TreeNode::Leaf(String)` to `TreeNode::Leaf(MetricLeafWithSchema)` where:
```rust ```rust
#[derive(Debug, Clone, Serialize, PartialEq, Eq, JsonSchema)]
pub struct MetricLeaf {
pub name: String,
pub value_type: String,
pub indexes: BTreeSet<Index>,
}
#[derive(Debug, Clone, Serialize, JsonSchema)] #[derive(Debug, Clone, Serialize, JsonSchema)]
pub struct MetricLeafWithSchema { pub struct MetricLeafWithSchema {
#[serde(flatten)] #[serde(flatten)]
@@ -248,113 +180,68 @@ pub struct MetricLeafWithSchema {
} }
``` ```
#### Implementation ## OpenAPI Integration ✅ IMPLEMENTED
**brk_types/src/treenode.rs**: ### Flow
- Added `MetricLeaf` struct with `name`, `value_type`, and `indexes`
- Added `MetricLeafWithSchema` wrapper with JSON schema
- Helper methods: `name()`, `value_type()`, `indexes()`, `is_same_metric()`, `merge_indexes()`
- Updated `TreeNode` enum to use `Leaf(MetricLeafWithSchema)`
**brk_traversable/src/lib.rs**: 1. brk_server creates OpenAPI spec via aide
- Added `make_leaf<I, T, V>()` helper that creates `MetricLeafWithSchema` with schema from schemars 2. On startup, serializes spec to JSON string
- Updated all `Traversable::to_tree_node()` implementations with `JsonSchema` bounds 3. Passes JSON to `brk_binder::generate_clients()`
- Schema generated via `schemars::SchemaGenerator::default().into_root_schema_for::<T>()` 4. brk_binder parses with `oas3` crate (supports OpenAPI 3.1)
5. Generates typed methods for all GET endpoints
**vecdb** (schemars feature): ### Why oas3?
- Added optional `schemars` dependency
- Added `AnySchemaVec` trait with blanket impl for `TypedVec where T: JsonSchema`
### Result aide generates OpenAPI 3.1 specs. The `openapiv3` crate only supports 3.0.x.
The `oas3` crate supports OpenAPI 3.1.x parsing.
The catalog tree now includes full type information and JSON schema at each leaf:
```rust
TreeNode::Leaf(MetricLeafWithSchema {
leaf: MetricLeaf {
name: "difficulty".to_string(),
value_type: "StoredF64".to_string(),
indexes: btreeset![Index::Height, Index::Date],
},
schema: json!({ "type": "number" }), // schemars-generated
})
```
When trees are merged/simplified, indexes are unioned together.
### 2. Async Runtime
- TypeScript: Native `Promise`
- Python: `asyncio` or sync variant?
- Rust: `tokio` assumed, or feature-flag for other runtimes?
### 3. Error Handling
- HTTP errors (4xx, 5xx)
- Deserialization errors
- Network errors
- Should errors be typed per language?
### 4. Additional Client Features
- Request timeout configuration
- Retry logic
- Rate limiting
- Caching
- Batch requests (fetch multiple metrics at once)
## Tasks ## Tasks
### Phase 0: Type Infrastructure ✅ COMPLETE ### Phase 0: Type Infrastructure ✅ COMPLETE
- [x] **vecdb**: Add `short_type_name<T>()` helper in `traits/printable.rs` - [x] vecdb: Add `short_type_name<T>()` and `value_type_to_string()`
- [x] **vecdb**: Add `value_type_to_string()` to `AnyVec` trait - [x] vecdb: Add optional `schemars` feature with `AnySchemaVec` trait
- [x] **vecdb**: Implement in all vec variants (eager, lazy, raw, compressed, macros) - [x] brk_types: Enhance `TreeNode::Leaf` to include `MetricLeafWithSchema`
- [x] **vecdb**: Add optional `schemars` feature with `AnySchemaVec` trait - [x] brk_traversable: Update all `to_tree_node()` with schemars integration
- [x] **brk_types**: Enhance `TreeNode::Leaf` to include `MetricLeafWithSchema` - [x] brk_binder: Set up generator module structure
- [x] **brk_types**: Add `JsonSchema` derives to all value types
- [x] **brk_traversable**: Update all `to_tree_node()` implementations with schemars integration
- [x] **brk_query**: Export `Vecs` publicly for client generation
- [x] **brk_binder**: Set up generator module structure
- [x] **brk**: Verify compilation
### Phase 1: JavaScript Client ✅ COMPLETE ### Phase 1: JavaScript Client ✅ COMPLETE
- [x] Define `MetricNode` class with JSDoc generics - [x] Define `MetricNode` class with JSDoc generics
- [x] Define `BrkClient` with base HTTP functionality - [x] Define `BrkClient` with base HTTP functionality
- [x] Implement `ClientMetadata::from_vecs()` to extract metadata
- [x] Generate `client.js` with full JSDoc type annotations - [x] Generate `client.js` with full JSDoc type annotations
- [x] Use `schema_to_jsdoc()` to convert JSON schemas to JSDoc types
- [x] Tree navigation: `client.tree.category.metric.fetch()` - [x] Tree navigation: `client.tree.category.metric.fetch()`
- [x] API methods from OpenAPI endpoints
### Phase 2: OpenAPI Integration (NEXT) ### Phase 2: OpenAPI Integration ✅ COMPLETE
- [ ] Add `openapiv3` crate dependency - [x] Add `oas3` crate dependency (OpenAPI 3.1 support)
- [ ] Parse OpenAPI spec from aide (brk_server generates this) - [x] brk_server passes OpenAPI JSON to brk_binder on startup
- [ ] Extract non-metric endpoint definitions (health, info, catalog, etc.) - [x] Parse OpenAPI spec and extract endpoint definitions
- [ ] Generate methods for each endpoint with proper types - [x] Generate typed methods for each GET endpoint
- [ ] Merge with tree-based metric access
### Phase 3: Python Client ### Phase 3: Python Client ✅ COMPLETE
- [ ] Define `MetricNode` class with type hints - [x] Define `MetricNode` class with type hints
- [ ] Define `BrkClient` with httpx/aiohttp - [x] Define `BrkClient` with httpx
- [ ] Generate typed methods from OpenAPI - [x] Generate typed methods from OpenAPI
- [ ] Generate tree navigation - [x] Generate tree navigation
### Phase 4: Rust Client ### Phase 4: Rust Client ✅ COMPLETE
- [ ] Define `MetricNode<T>` struct using `brk_types` - [x] Define `MetricNode<T>` struct with lifetimes
- [ ] Define `BrkClient` with reqwest - [x] Define `BrkClient` with reqwest (blocking)
- [ ] Import types from `brk_types` instead of generating - [x] Generate tree navigation with proper lifetimes
- [ ] Generate tree navigation with proper lifetimes - [x] Generate typed methods from OpenAPI
### Phase 5: Polish ### Phase 5: Polish
- [x] Switch from `openapiv3` to `oas3` crate
- [ ] Error types per language - [ ] Error types per language
- [ ] Documentation generation - [ ] Documentation generation
- [ ] Tests - [ ] Tests
- [ ] Example usage in each language - [ ] Example usage in each language
- [ ] Async Rust client variant
## File Structure ## File Structure
@@ -363,16 +250,26 @@ crates/brk_binder/
├── src/ ├── src/
│ ├── lib.rs │ ├── lib.rs
│ ├── js.rs # JS constants generation (existing) │ ├── js.rs # JS constants generation (existing)
│ ├── tree.rs # Pattern extraction (reference only, not compiled)
│ └── generator/ │ └── generator/
│ ├── mod.rs │ ├── mod.rs # generate_clients() entry point
│ ├── types.rs # ClientMetadata, MetricInfo, IndexPattern, schema_to_jsdoc │ ├── types.rs # ClientMetadata, MetricInfo, IndexPattern
│ ├── javascript.rs # JavaScript + JSDoc client generation ✅ │ ├── openapi.rs # OpenAPI 3.1 spec parsing (oas3)
│ ├── python.rs # Python client generation (stub) │ ├── javascript.rs # JavaScript + JSDoc client ✅
── rust.rs # Rust client generation (stub) ── python.rs # Python client ✅
│ └── rust.rs # Rust client ✅
├── clients/ # Generated output (gitignored)
│ ├── javascript/
│ ├── python/
│ └── rust/
├── Cargo.toml ├── Cargo.toml
├── README.md ├── README.md
└── DESIGN.md # This file └── DESIGN.md
crates/brk_server/
└── src/
├── lib.rs # Calls brk_binder::generate_clients() on startup
└── api/
└── openapi.rs # create_openapi() for aide
``` ```
## Dependencies ## Dependencies
@@ -381,11 +278,19 @@ crates/brk_binder/
[dependencies] [dependencies]
brk_query = { workspace = true } brk_query = { workspace = true }
brk_types = { workspace = true } brk_types = { workspace = true }
oas3 = "0.20" # OpenAPI 3.1 spec parsing
schemars = { workspace = true } schemars = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
vecdb = { workspace = true } ```
# For OpenAPI integration (Phase 2): ## Usage
# openapiv3 = "2" # OpenAPI parsing
# serde_yaml = "0.9" # If parsing YAML specs To generate clients:
```bash
# Create the output directory
mkdir -p crates/brk_binder/clients
# Run the server (generates clients on startup)
cargo run -p brk_server
``` ```

View File

@@ -1,69 +1,105 @@
use std::collections::HashSet;
use std::fmt::Write as FmtWrite; use std::fmt::Write as FmtWrite;
use std::fs; use std::fs;
use std::io; use std::io;
use std::path::Path; use std::path::Path;
use brk_types::TreeNode; use brk_types::{Index, TreeNode};
use super::{schema_to_jsdoc, to_camel_case, ClientMetadata, IndexPattern}; use super::{ClientMetadata, Endpoint, IndexSetPattern, PatternField, StructuralPattern, get_node_fields, to_camel_case, to_pascal_case, to_snake_case};
/// Generate JavaScript + JSDoc client from metadata /// Generate JavaScript + JSDoc client from metadata and OpenAPI endpoints
pub fn generate_javascript_client(metadata: &ClientMetadata, output_dir: &Path) -> io::Result<()> { pub fn generate_javascript_client(
metadata: &ClientMetadata,
endpoints: &[Endpoint],
output_dir: &Path,
) -> io::Result<()> {
let mut output = String::new(); let mut output = String::new();
// Header // Header
writeln!(output, "// Auto-generated BRK JavaScript client").unwrap(); writeln!(output, "// Auto-generated BRK JavaScript client").unwrap();
writeln!(output, "// Do not edit manually\n").unwrap(); writeln!(output, "// Do not edit manually\n").unwrap();
// Generate pattern JSDoc typedefs for index groupings
generate_pattern_typedefs(&mut output, &metadata.patterns);
// Generate the base client class // Generate the base client class
generate_base_client(&mut output); generate_base_client(&mut output);
// Generate tree JSDoc typedefs from catalog // Generate index accessor factory functions
generate_tree_typedefs(&mut output, &metadata.catalog); generate_index_accessors(&mut output, &metadata.index_set_patterns);
// Generate the main client class with tree // Generate structural pattern factory functions
generate_main_client(&mut output, &metadata.catalog); generate_structural_patterns(&mut output, &metadata.structural_patterns, metadata);
// Generate tree JSDoc typedefs
generate_tree_typedefs(&mut output, &metadata.catalog, metadata);
// Generate the main client class with tree and API methods
generate_main_client(&mut output, &metadata.catalog, metadata, endpoints);
fs::write(output_dir.join("client.js"), output)?; fs::write(output_dir.join("client.js"), output)?;
Ok(()) Ok(())
} }
/// Generate JSDoc typedefs for common index patterns
fn generate_pattern_typedefs(output: &mut String, patterns: &[IndexPattern]) {
writeln!(output, "// Index pattern typedefs").unwrap();
writeln!(output, "// Reusable patterns for metrics with the same index groupings\n").unwrap();
for pattern in patterns {
let pattern_name = pattern_to_name(pattern);
writeln!(output, "/**").unwrap();
writeln!(output, " * @template T").unwrap();
writeln!(output, " * @typedef {{Object}} {}", pattern_name).unwrap();
for index in &pattern.indexes {
let field_name = to_camel_case(&index.serialize_long());
writeln!(output, " * @property {{MetricNode<T>}} {}", field_name).unwrap();
}
writeln!(output, " */\n").unwrap();
}
}
/// Generate the base BrkClient class with HTTP functionality /// Generate the base BrkClient class with HTTP functionality
fn generate_base_client(output: &mut String) { fn generate_base_client(output: &mut String) {
writeln!( writeln!(
output, output,
r#"/** r#"/**
* @typedef {{Object}} BrkClientOptions * @typedef {{Object}} BrkClientOptions
* @property {{string}} baseUrl - The base URL for the API * @property {{string}} baseUrl - Base URL for the API
* @property {{number}} [timeout] - Request timeout in ms (default: 30000) * @property {{number}} [timeout] - Request timeout in milliseconds
*/ */
/** /**
* Base HTTP client * Custom error class for BRK client errors
*/
class BrkError extends Error {{
/**
* @param {{string}} message
* @param {{number}} [status]
*/
constructor(message, status) {{
super(message);
this.name = 'BrkError';
this.status = status;
}}
}}
/**
* A metric node that can fetch data for different indexes.
* @template T
*/
class MetricNode {{
/**
* @param {{BrkClientBase}} client
* @param {{string}} path
*/
constructor(client, path) {{
this.client = client;
this.path = path;
}}
/**
* Fetch all data points for this metric.
* @returns {{Promise<T[]>}}
*/
async get() {{
return this.client.get(this.path);
}}
/**
* Fetch data points within a date range.
* @param {{string}} from
* @param {{string}} to
* @returns {{Promise<T[]>}}
*/
async getRange(from, to) {{
return this.client.get(`${{this.path}}?from=${{from}}&to=${{to}}`);
}}
}}
/**
* Base HTTP client for making requests
*/ */
class BrkClientBase {{ class BrkClientBase {{
/** /**
@@ -71,15 +107,16 @@ class BrkClientBase {{
*/ */
constructor(options) {{ constructor(options) {{
if (typeof options === 'string') {{ if (typeof options === 'string') {{
this.baseUrl = options.replace(/\/$/, ''); this.baseUrl = options;
this.timeout = 30000; this.timeout = 30000;
}} else {{ }} else {{
this.baseUrl = options.baseUrl.replace(/\/$/, ''); this.baseUrl = options.baseUrl;
this.timeout = options.timeout ?? 30000; this.timeout = options.timeout || 30000;
}} }}
}} }}
/** /**
* Make a GET request
* @template T * @template T
* @param {{string}} path * @param {{string}} path
* @returns {{Promise<T>}} * @returns {{Promise<T>}}
@@ -91,127 +128,241 @@ class BrkClientBase {{
try {{ try {{
const response = await fetch(`${{this.baseUrl}}${{path}}`, {{ const response = await fetch(`${{this.baseUrl}}${{path}}`, {{
signal: controller.signal, signal: controller.signal,
headers: {{ 'Accept': 'application/json' }},
}}); }});
if (!response.ok) {{ if (!response.ok) {{
throw new BrkError(`HTTP ${{response.status}}: ${{response.statusText}}`, response.status); throw new BrkError(`HTTP error: ${{response.status}}`, response.status);
}} }}
return await response.json(); return response.json();
}} catch (error) {{
if (error.name === 'AbortError') {{
throw new BrkError('Request timeout');
}}
throw error;
}} finally {{ }} finally {{
clearTimeout(timeoutId); clearTimeout(timeoutId);
}} }}
}} }}
}} }}
/**
* Error class for BRK API errors
*/
class BrkError extends Error {{
/**
* @param {{string}} message
* @param {{number}} [statusCode]
*/
constructor(message, statusCode) {{
super(message);
this.name = 'BrkError';
this.statusCode = statusCode;
}}
}}
/**
* Metric node with fetch capability
* @template T
*/
class MetricNode {{
/**
* @param {{BrkClientBase}} client
* @param {{string}} path
*/
constructor(client, path) {{
this._client = client;
this._path = path;
}}
/**
* Fetch the metric value
* @returns {{Promise<T>}}
*/
async fetch() {{
return this._client.get(this._path);
}}
toString() {{
return this._path;
}}
}}
"# "#
) )
.unwrap(); .unwrap();
} }
/// Generate JSDoc typedefs for the catalog tree /// Generate index accessor factory functions
fn generate_tree_typedefs(output: &mut String, catalog: &TreeNode) { fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) {
writeln!(output, "// Catalog tree typedefs\n").unwrap(); if patterns.is_empty() {
generate_node_typedef(output, "CatalogTree", catalog, ""); return;
}
writeln!(output, "// Index accessor factory functions\n").unwrap();
for pattern in patterns {
// Generate JSDoc typedef for the accessor
writeln!(output, "/**").unwrap();
writeln!(output, " * @template T").unwrap();
writeln!(output, " * @typedef {{Object}} {}", pattern.name).unwrap();
for index in &pattern.indexes {
let field_name = index_to_camel_case(index);
writeln!(output, " * @property {{MetricNode<T>}} {}", field_name).unwrap();
}
writeln!(output, " */\n").unwrap();
// Generate factory function
writeln!(output, "/**").unwrap();
writeln!(output, " * Create a {} accessor", pattern.name).unwrap();
writeln!(output, " * @template T").unwrap();
writeln!(output, " * @param {{BrkClientBase}} client").unwrap();
writeln!(output, " * @param {{string}} basePath").unwrap();
writeln!(output, " * @returns {{{}<T>}}", pattern.name).unwrap();
writeln!(output, " */").unwrap();
writeln!(output, "function create{}(client, basePath) {{", pattern.name).unwrap();
writeln!(output, " return {{").unwrap();
for (i, index) in pattern.indexes.iter().enumerate() {
let field_name = index_to_camel_case(index);
let path_segment = index.serialize_short();
let comma = if i < pattern.indexes.len() - 1 { "," } else { "" };
writeln!(
output,
" {}: new MetricNode(client, `${{basePath}}/{}`){}",
field_name, path_segment, comma
).unwrap();
}
writeln!(output, " }};").unwrap();
writeln!(output, "}}\n").unwrap();
}
} }
/// Recursively generate typedef for a tree node /// Convert an Index to a camelCase field name (e.g., DateIndex -> byDate)
fn generate_node_typedef(output: &mut String, name: &str, node: &TreeNode, path: &str) { fn index_to_camel_case(index: &Index) -> String {
match node { let short = index.serialize_short();
TreeNode::Leaf(_leaf) => { format!("by{}", to_pascal_case(&to_snake_case(short)))
// Leaf nodes are MetricNode<ValueType> }
// No separate typedef needed, handled inline
/// Generate structural pattern factory functions
fn generate_structural_patterns(output: &mut String, patterns: &[StructuralPattern], metadata: &ClientMetadata) {
if patterns.is_empty() {
return;
} }
TreeNode::Branch(children) => {
writeln!(output, "// Reusable structural pattern factories\n").unwrap();
for pattern in patterns {
// Generate JSDoc typedef
writeln!(output, "/**").unwrap();
writeln!(output, " * @typedef {{Object}} {}", pattern.name).unwrap();
for field in &pattern.fields {
let js_type = field_to_js_type(field, metadata);
writeln!(output, " * @property {{{}}} {}", js_type, to_camel_case(&field.name)).unwrap();
}
writeln!(output, " */\n").unwrap();
// Generate factory function
writeln!(output, "/**").unwrap();
writeln!(output, " * Create a {} pattern node", pattern.name).unwrap();
writeln!(output, " * @param {{BrkClientBase}} client").unwrap();
writeln!(output, " * @param {{string}} basePath").unwrap();
writeln!(output, " * @returns {{{}}}", pattern.name).unwrap();
writeln!(output, " */").unwrap();
writeln!(output, "function create{}(client, basePath) {{", pattern.name).unwrap();
writeln!(output, " return {{").unwrap();
for (i, field) in pattern.fields.iter().enumerate() {
let comma = if i < pattern.fields.len() - 1 { "," } else { "" };
if metadata.is_pattern_type(&field.rust_type) {
writeln!(
output,
" {}: create{}(client, `${{basePath}}/{}`){}",
to_camel_case(&field.name), field.rust_type, field.name, comma
).unwrap();
} else if field_uses_accessor(field, metadata) {
let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap();
writeln!(
output,
" {}: create{}(client, `${{basePath}}/{}`){}",
to_camel_case(&field.name), accessor.name, field.name, comma
).unwrap();
} else {
writeln!(
output,
" {}: new MetricNode(client, `${{basePath}}/{}`){}",
to_camel_case(&field.name), field.name, comma
).unwrap();
}
}
writeln!(output, " }};").unwrap();
writeln!(output, "}}\n").unwrap();
}
}
/// Convert pattern field to JavaScript/JSDoc type
fn field_to_js_type(field: &PatternField, metadata: &ClientMetadata) -> String {
if metadata.is_pattern_type(&field.rust_type) {
// Pattern type - use pattern name directly
field.rust_type.clone()
} else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) {
// Leaf with a reusable accessor pattern
let js_type = json_type_to_js(&field.json_type);
format!("{}<{}>", accessor.name, js_type)
} else {
// Leaf with unique index set - use MetricNode directly
let js_type = json_type_to_js(&field.json_type);
format!("MetricNode<{}>", js_type)
}
}
/// Check if a field should use an index accessor
fn field_uses_accessor(field: &PatternField, metadata: &ClientMetadata) -> bool {
metadata.find_index_set_pattern(&field.indexes).is_some()
}
/// Convert JSON Schema type to JSDoc type
fn json_type_to_js(json_type: &str) -> &str {
match json_type {
"integer" | "number" => "number",
"boolean" => "boolean",
"string" => "string",
"array" => "Array",
"object" => "Object",
_ => "*",
}
}
/// Generate tree typedefs
fn generate_tree_typedefs(
output: &mut String,
catalog: &TreeNode,
metadata: &ClientMetadata,
) {
writeln!(output, "// Catalog tree typedefs\n").unwrap();
let pattern_lookup = metadata.pattern_lookup();
let mut generated = HashSet::new();
generate_tree_typedef(output, "CatalogTree", catalog, &pattern_lookup, metadata, &mut generated);
}
/// Recursively generate tree typedefs
fn generate_tree_typedef(
output: &mut String,
name: &str,
node: &TreeNode,
pattern_lookup: &std::collections::HashMap<Vec<PatternField>, String>,
metadata: &ClientMetadata,
generated: &mut HashSet<String>,
) {
if let TreeNode::Branch(children) = node {
// Build signature
let fields = get_node_fields(children, pattern_lookup);
// Skip if this matches a pattern (already generated)
if pattern_lookup.contains_key(&fields) && pattern_lookup.get(&fields) != Some(&name.to_string()) {
return;
}
if generated.contains(name) {
return;
}
generated.insert(name.to_string());
writeln!(output, "/**").unwrap(); writeln!(output, "/**").unwrap();
writeln!(output, " * @typedef {{Object}} {}", name).unwrap(); writeln!(output, " * @typedef {{Object}} {}", name).unwrap();
for (child_name, child_node) in children { for field in &fields {
let field_name = to_camel_case(child_name); let js_type = field_to_js_type(field, metadata);
writeln!(output, " * @property {{{}}} {}", js_type, to_camel_case(&field.name)).unwrap();
match child_node {
TreeNode::Leaf(leaf) => {
let js_type = schema_to_jsdoc(&leaf.schema);
writeln!(
output,
" * @property {{MetricNode<{}>}} {}",
js_type, field_name
)
.unwrap();
}
TreeNode::Branch(_) => {
let child_type_name = format!("{}_{}", name, to_pascal_case(child_name));
writeln!(output, " * @property {{{}}} {}", child_type_name, field_name)
.unwrap();
}
}
} }
writeln!(output, " */\n").unwrap(); writeln!(output, " */\n").unwrap();
// Generate child typedefs // Generate child typedefs
for (child_name, child_node) in children { for (child_name, child_node) in children {
if let TreeNode::Branch(_) = child_node { if let TreeNode::Branch(grandchildren) = child_node {
let child_fields = get_node_fields(grandchildren, pattern_lookup);
if !pattern_lookup.contains_key(&child_fields) {
let child_type_name = format!("{}_{}", name, to_pascal_case(child_name)); let child_type_name = format!("{}_{}", name, to_pascal_case(child_name));
let child_path = if path.is_empty() { generate_tree_typedef(output, &child_type_name, child_node, pattern_lookup, metadata, generated);
format!("/{}", child_name)
} else {
format!("{}/{}", path, child_name)
};
generate_node_typedef(output, &child_type_name, child_node, &child_path);
} }
} }
} }
} }
} }
/// Generate the main client class with initialized tree /// Generate main client
fn generate_main_client(output: &mut String, catalog: &TreeNode) { fn generate_main_client(
output: &mut String,
catalog: &TreeNode,
metadata: &ClientMetadata,
endpoints: &[Endpoint],
) {
let pattern_lookup = metadata.pattern_lookup();
writeln!(output, "/**").unwrap(); writeln!(output, "/**").unwrap();
writeln!(output, " * Main BRK client with catalog tree").unwrap(); writeln!(output, " * Main BRK client with catalog tree and API methods").unwrap();
writeln!(output, " * @extends BrkClientBase").unwrap(); writeln!(output, " * @extends BrkClientBase").unwrap();
writeln!(output, " */").unwrap(); writeln!(output, " */").unwrap();
writeln!(output, "class BrkClient extends BrkClientBase {{").unwrap(); writeln!(output, "class BrkClient extends BrkClientBase {{").unwrap();
@@ -221,27 +372,39 @@ fn generate_main_client(output: &mut String, catalog: &TreeNode) {
writeln!(output, " constructor(options) {{").unwrap(); writeln!(output, " constructor(options) {{").unwrap();
writeln!(output, " super(options);").unwrap(); writeln!(output, " super(options);").unwrap();
writeln!(output, " /** @type {{CatalogTree}} */").unwrap(); writeln!(output, " /** @type {{CatalogTree}} */").unwrap();
writeln!(output, " this.tree = this._buildTree();").unwrap(); writeln!(output, " this.tree = this._buildTree('');").unwrap();
writeln!(output, " }}\n").unwrap(); writeln!(output, " }}\n").unwrap();
// Generate _buildTree method // Generate _buildTree method
writeln!(output, " /**").unwrap(); writeln!(output, " /**").unwrap();
writeln!(output, " * @private").unwrap(); writeln!(output, " * @private").unwrap();
writeln!(output, " * @param {{string}} basePath").unwrap();
writeln!(output, " * @returns {{CatalogTree}}").unwrap(); writeln!(output, " * @returns {{CatalogTree}}").unwrap();
writeln!(output, " */").unwrap(); writeln!(output, " */").unwrap();
writeln!(output, " _buildTree() {{").unwrap(); writeln!(output, " _buildTree(basePath) {{").unwrap();
writeln!(output, " return {{").unwrap(); writeln!(output, " return {{").unwrap();
generate_tree_initializer(output, catalog, "", 3); generate_tree_initializer(output, catalog, "", 3, &pattern_lookup, metadata);
writeln!(output, " }};").unwrap(); writeln!(output, " }};").unwrap();
writeln!(output, " }}").unwrap(); writeln!(output, " }}\n").unwrap();
// Generate API methods
generate_api_methods(output, endpoints);
writeln!(output, "}}\n").unwrap(); writeln!(output, "}}\n").unwrap();
// Export for ES modules // Export
writeln!(output, "export {{ BrkClient, BrkClientBase, BrkError, MetricNode }};").unwrap(); writeln!(output, "export {{ BrkClient, BrkClientBase, BrkError, MetricNode }};").unwrap();
} }
/// Generate the tree initializer code /// Generate tree initializer
fn generate_tree_initializer(output: &mut String, node: &TreeNode, path: &str, indent: usize) { fn generate_tree_initializer(
output: &mut String,
node: &TreeNode,
path: &str,
indent: usize,
pattern_lookup: &std::collections::HashMap<Vec<PatternField>, String>,
metadata: &ClientMetadata,
) {
let indent_str = " ".repeat(indent); let indent_str = " ".repeat(indent);
if let TreeNode::Branch(children) = node { if let TreeNode::Branch(children) = node {
@@ -256,43 +419,118 @@ fn generate_tree_initializer(output: &mut String, node: &TreeNode, path: &str, i
let comma = if i < children.len() - 1 { "," } else { "" }; let comma = if i < children.len() - 1 { "," } else { "" };
match child_node { match child_node {
TreeNode::Leaf(_) => { TreeNode::Leaf(leaf) => {
if let Some(accessor) = metadata.find_index_set_pattern(leaf.indexes()) {
writeln!(
output,
"{}{}: create{}(this, '{}'){}",
indent_str, field_name, accessor.name, child_path, comma
).unwrap();
} else {
writeln!( writeln!(
output, output,
"{}{}: new MetricNode(this, '{}'){}", "{}{}: new MetricNode(this, '{}'){}",
indent_str, field_name, child_path, comma indent_str, field_name, child_path, comma
) ).unwrap();
.unwrap();
} }
TreeNode::Branch(_) => { }
TreeNode::Branch(grandchildren) => {
let child_fields = get_node_fields(grandchildren, pattern_lookup);
if let Some(pattern_name) = pattern_lookup.get(&child_fields) {
writeln!(
output,
"{}{}: create{}(this, '{}'){}",
indent_str, field_name, pattern_name, child_path, comma
).unwrap();
} else {
writeln!(output, "{}{}: {{", indent_str, field_name).unwrap(); writeln!(output, "{}{}: {{", indent_str, field_name).unwrap();
generate_tree_initializer(output, child_node, &child_path, indent + 1); generate_tree_initializer(output, child_node, &child_path, indent + 1, pattern_lookup, metadata);
writeln!(output, "{}}}{}", indent_str, comma).unwrap(); writeln!(output, "{}}}{}", indent_str, comma).unwrap();
} }
} }
} }
} }
}
/// Convert pattern to a JSDoc typedef name
fn pattern_to_name(pattern: &IndexPattern) -> String {
let index_names: Vec<String> = pattern
.indexes
.iter()
.map(|i| to_pascal_case(&i.serialize_long()))
.collect();
format!("Pattern_{}", index_names.join("_"))
}
/// Convert string to PascalCase
fn to_pascal_case(s: &str) -> String {
s.split('_')
.map(|word| {
let mut chars = word.chars();
match chars.next() {
None => String::new(),
Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),
} }
})
.collect()
} }
/// Generate API methods
fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
for endpoint in endpoints {
if endpoint.method != "GET" {
continue;
}
let method_name = endpoint_to_method_name(endpoint);
let return_type = endpoint.response_type.as_deref().unwrap_or("*");
writeln!(output, " /**").unwrap();
if let Some(summary) = &endpoint.summary {
writeln!(output, " * {}", summary).unwrap();
}
for param in &endpoint.path_params {
let desc = param.description.as_deref().unwrap_or("");
writeln!(output, " * @param {{{}}} {} {}", param.param_type, param.name, desc).unwrap();
}
for param in &endpoint.query_params {
let optional = if param.required { "" } else { "=" };
let desc = param.description.as_deref().unwrap_or("");
writeln!(output, " * @param {{{}{}}} [{}] {}", param.param_type, optional, param.name, desc).unwrap();
}
writeln!(output, " * @returns {{Promise<{}>}}", return_type).unwrap();
writeln!(output, " */").unwrap();
let params = build_method_params(endpoint);
writeln!(output, " async {}({}) {{", method_name, params).unwrap();
let path = build_path_template(&endpoint.path, &endpoint.path_params);
if endpoint.query_params.is_empty() {
writeln!(output, " return this.get(`{}`);", path).unwrap();
} else {
writeln!(output, " const params = new URLSearchParams();").unwrap();
for param in &endpoint.query_params {
if param.required {
writeln!(output, " params.set('{}', String({}));", param.name, param.name).unwrap();
} else {
writeln!(output, " if ({} !== undefined) params.set('{}', String({}));", param.name, param.name, param.name).unwrap();
}
}
writeln!(output, " const query = params.toString();").unwrap();
writeln!(output, " return this.get(`{}${{query ? '?' + query : ''}}`);", path).unwrap();
}
writeln!(output, " }}\n").unwrap();
}
}
fn endpoint_to_method_name(endpoint: &Endpoint) -> String {
if let Some(op_id) = &endpoint.operation_id {
return to_camel_case(op_id);
}
let parts: Vec<&str> = endpoint.path.split('/').filter(|s| !s.is_empty() && !s.starts_with('{')).collect();
format!("get{}", to_pascal_case(&parts.join("_")))
}
fn build_method_params(endpoint: &Endpoint) -> String {
let mut params = Vec::new();
for param in &endpoint.path_params {
params.push(param.name.clone());
}
for param in &endpoint.query_params {
params.push(param.name.clone());
}
params.join(", ")
}
fn build_path_template(path: &str, path_params: &[super::Parameter]) -> String {
let mut result = path.to_string();
for param in path_params {
let placeholder = format!("{{{}}}", param.name);
let interpolation = format!("${{{}}}", param.name);
result = result.replace(&placeholder, &interpolation);
}
result
}

View File

@@ -1,35 +1,41 @@
mod javascript; mod javascript;
mod openapi;
mod python; mod python;
mod rust; mod rust;
mod types; mod types;
pub use javascript::generate_javascript_client; pub use javascript::*;
pub use python::generate_python_client; pub use openapi::*;
pub use rust::generate_rust_client; pub use python::*;
pub use rust::*;
pub use types::*; pub use types::*;
use brk_query::Vecs; use brk_query::Vecs;
use std::io; use std::io;
use std::path::Path; use std::path::Path;
/// Generate all client libraries from the query vecs /// Generate all client libraries from the query vecs and OpenAPI JSON
pub fn generate_clients(vecs: &Vecs, output_dir: &Path) -> io::Result<()> { pub fn generate_clients(vecs: &Vecs, openapi_json: &str, output_dir: &Path) -> io::Result<()> {
let metadata = ClientMetadata::from_vecs(vecs); let metadata = ClientMetadata::from_vecs(vecs);
// Parse OpenAPI spec from JSON
let spec = parse_openapi_json(openapi_json)?;
let endpoints = extract_endpoints(&spec);
// Generate Rust client // Generate Rust client
let rust_path = output_dir.join("rust"); let rust_path = output_dir.join("rust");
std::fs::create_dir_all(&rust_path)?; std::fs::create_dir_all(&rust_path)?;
generate_rust_client(&metadata, &rust_path)?; generate_rust_client(&metadata, &endpoints, &rust_path)?;
// Generate JavaScript client // Generate JavaScript client
let js_path = output_dir.join("javascript"); let js_path = output_dir.join("javascript");
std::fs::create_dir_all(&js_path)?; std::fs::create_dir_all(&js_path)?;
generate_javascript_client(&metadata, &js_path)?; generate_javascript_client(&metadata, &endpoints, &js_path)?;
// Generate Python client // Generate Python client
let python_path = output_dir.join("python"); let python_path = output_dir.join("python");
std::fs::create_dir_all(&python_path)?; std::fs::create_dir_all(&python_path)?;
generate_python_client(&metadata, &python_path)?; generate_python_client(&metadata, &endpoints, &python_path)?;
Ok(()) Ok(())
} }

View File

@@ -0,0 +1,214 @@
use std::io;
use oas3::spec::{ObjectOrReference, Operation, ParameterIn, PathItem, Schema, SchemaTypeSet};
use oas3::Spec;
use serde_json::Value;
/// Endpoint information extracted from OpenAPI spec
#[derive(Debug, Clone)]
pub struct Endpoint {
/// HTTP method (GET, POST, etc.)
pub method: String,
/// Path template (e.g., "/blocks/{hash}")
pub path: String,
/// Operation ID (e.g., "getBlockByHash")
pub operation_id: Option<String>,
/// Summary/description
pub summary: Option<String>,
/// Tags for grouping
pub tags: Vec<String>,
/// Path parameters
pub path_params: Vec<Parameter>,
/// Query parameters
pub query_params: Vec<Parameter>,
/// Response type (simplified)
pub response_type: Option<String>,
}
/// Parameter information
#[derive(Debug, Clone)]
pub struct Parameter {
pub name: String,
pub required: bool,
pub param_type: String,
pub description: Option<String>,
}
/// Parse OpenAPI spec from JSON string
///
/// Pre-processes the JSON to handle oas3 limitations:
/// - Removes unsupported siblings from `$ref` objects (oas3 only supports `summary` and `description`)
pub fn parse_openapi_json(json: &str) -> io::Result<Spec> {
let mut value: Value =
serde_json::from_str(json).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
// Clean up for oas3 compatibility
clean_for_oas3(&mut value);
let cleaned_json = serde_json::to_string(&value)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
oas3::from_json(&cleaned_json).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
}
/// Clean up OpenAPI spec for oas3 compatibility.
/// - Removes unsupported siblings from $ref objects (oas3 only supports summary and description)
/// - Converts boolean schemas to object schemas (oas3 doesn't handle `"schema": true`)
fn clean_for_oas3(value: &mut Value) {
match value {
Value::Object(map) => {
// Handle $ref with unsupported siblings
if map.contains_key("$ref") {
map.retain(|k, _| k == "$ref" || k == "summary" || k == "description");
} else {
// Convert boolean schemas to empty object schemas
if let Some(schema) = map.get_mut("schema") {
if schema.is_boolean() {
*schema = Value::Object(serde_json::Map::new());
}
}
for v in map.values_mut() {
clean_for_oas3(v);
}
}
}
Value::Array(arr) => {
for v in arr {
clean_for_oas3(v);
}
}
_ => {}
}
}
/// Extract all endpoints from OpenAPI spec
pub fn extract_endpoints(spec: &Spec) -> Vec<Endpoint> {
let mut endpoints = Vec::new();
let Some(paths) = &spec.paths else {
return endpoints;
};
for (path, path_item) in paths {
for (method, operation) in get_operations(path_item) {
if let Some(endpoint) = extract_endpoint(path, &method, operation) {
endpoints.push(endpoint);
}
}
}
endpoints
}
fn get_operations(path_item: &PathItem) -> Vec<(String, &Operation)> {
let mut ops = Vec::new();
if let Some(op) = &path_item.get {
ops.push(("GET".to_string(), op));
}
if let Some(op) = &path_item.post {
ops.push(("POST".to_string(), op));
}
if let Some(op) = &path_item.put {
ops.push(("PUT".to_string(), op));
}
if let Some(op) = &path_item.delete {
ops.push(("DELETE".to_string(), op));
}
if let Some(op) = &path_item.patch {
ops.push(("PATCH".to_string(), op));
}
ops
}
fn extract_endpoint(path: &str, method: &str, operation: &Operation) -> Option<Endpoint> {
let path_params = extract_parameters(operation, ParameterIn::Path);
let query_params = extract_parameters(operation, ParameterIn::Query);
let response_type = extract_response_type(operation);
Some(Endpoint {
method: method.to_string(),
path: path.to_string(),
operation_id: operation.operation_id.clone(),
summary: operation.summary.clone().or_else(|| operation.description.clone()),
tags: operation.tags.clone(),
path_params,
query_params,
response_type,
})
}
fn extract_parameters(operation: &Operation, location: ParameterIn) -> Vec<Parameter> {
operation
.parameters
.iter()
.filter_map(|p| match p {
ObjectOrReference::Object(param) if param.location == location => Some(Parameter {
name: param.name.clone(),
required: param.required.unwrap_or(false),
param_type: "string".to_string(), // Simplified
description: param.description.clone(),
}),
_ => None,
})
.collect()
}
fn extract_response_type(operation: &Operation) -> Option<String> {
let responses = operation.responses.as_ref()?;
// Look for 200 OK response
let response = responses.get("200")?;
match response {
ObjectOrReference::Object(response) => {
// Look for JSON content
let content = response.content.get("application/json")?;
match &content.schema {
Some(ObjectOrReference::Ref { ref_path, .. }) => {
// Extract type name from reference like "#/components/schemas/Block"
Some(ref_path.rsplit('/').next()?.to_string())
}
Some(ObjectOrReference::Object(schema)) => schema_to_type_name(schema),
None => None,
}
}
ObjectOrReference::Ref { .. } => None,
}
}
fn schema_type_from_schema(schema: &Schema) -> Option<String> {
match schema {
Schema::Boolean(_) => Some("boolean".to_string()),
Schema::Object(obj_or_ref) => match obj_or_ref.as_ref() {
ObjectOrReference::Object(obj_schema) => schema_to_type_name(obj_schema),
ObjectOrReference::Ref { ref_path, .. } => {
ref_path.rsplit('/').next().map(|s| s.to_string())
}
},
}
}
fn schema_to_type_name(schema: &oas3::spec::ObjectSchema) -> Option<String> {
let schema_type = schema.schema_type.as_ref()?;
match schema_type {
SchemaTypeSet::Single(t) => match t {
oas3::spec::SchemaType::String => Some("string".to_string()),
oas3::spec::SchemaType::Number => Some("number".to_string()),
oas3::spec::SchemaType::Integer => Some("number".to_string()),
oas3::spec::SchemaType::Boolean => Some("boolean".to_string()),
oas3::spec::SchemaType::Array => {
let inner = match &schema.items {
Some(boxed_schema) => schema_type_from_schema(boxed_schema),
None => Some("*".to_string()),
};
inner.map(|t| format!("{}[]", t))
}
oas3::spec::SchemaType::Object => Some("Object".to_string()),
oas3::spec::SchemaType::Null => Some("null".to_string()),
},
SchemaTypeSet::Multiple(_) => Some("*".to_string()),
}
}

View File

@@ -1,10 +1,428 @@
use std::collections::HashSet;
use std::fmt::Write as FmtWrite;
use std::fs;
use std::io; use std::io;
use std::path::Path; use std::path::Path;
use super::ClientMetadata; use brk_types::{Index, TreeNode};
use super::{ClientMetadata, Endpoint, IndexSetPattern, PatternField, StructuralPattern, get_node_fields, to_pascal_case, to_snake_case};
/// Generate Python client from metadata and OpenAPI endpoints
pub fn generate_python_client(
metadata: &ClientMetadata,
endpoints: &[Endpoint],
output_dir: &Path,
) -> io::Result<()> {
let mut output = String::new();
// Header
writeln!(output, "# Auto-generated BRK Python client").unwrap();
writeln!(output, "# Do not edit manually\n").unwrap();
writeln!(output, "from __future__ import annotations").unwrap();
writeln!(output, "from typing import TypeVar, Generic, Any, Optional, List").unwrap();
writeln!(output, "from dataclasses import dataclass").unwrap();
writeln!(output, "import httpx\n").unwrap();
// Type variable for generic MetricNode
writeln!(output, "T = TypeVar('T')\n").unwrap();
// Generate base client class
generate_base_client(&mut output);
// Generate MetricNode class
generate_metric_node(&mut output);
// Generate index accessor classes
generate_index_accessors(&mut output, &metadata.index_set_patterns);
// Generate structural pattern classes
generate_structural_patterns(&mut output, &metadata.structural_patterns, metadata);
// Generate tree classes
generate_tree_classes(&mut output, &metadata.catalog, metadata);
// Generate main client with tree and API methods
generate_main_client(&mut output, endpoints);
fs::write(output_dir.join("client.py"), output)?;
/// Generate Python client from metadata
pub fn generate_python_client(_metadata: &ClientMetadata, _output_dir: &Path) -> io::Result<()> {
// TODO: Implement Python client generation
Ok(()) Ok(())
} }
/// Generate the base BrkClient class with HTTP functionality
fn generate_base_client(output: &mut String) {
writeln!(
output,
r#"class BrkError(Exception):
"""Custom error class for BRK client errors."""
def __init__(self, message: str, status: Optional[int] = None):
super().__init__(message)
self.status = status
class BrkClientBase:
"""Base HTTP client for making requests."""
def __init__(self, base_url: str, timeout: float = 30.0):
self.base_url = base_url
self.timeout = timeout
self._client = httpx.Client(timeout=timeout)
def get(self, path: str) -> Any:
"""Make a GET request."""
try:
response = self._client.get(f"{{self.base_url}}{{path}}")
response.raise_for_status()
return response.json()
except httpx.HTTPStatusError as e:
raise BrkError(f"HTTP error: {{e.response.status_code}}", e.response.status_code)
except httpx.RequestError as e:
raise BrkError(str(e))
def close(self):
"""Close the HTTP client."""
self._client.close()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
"#
)
.unwrap();
}
/// Generate the MetricNode class
fn generate_metric_node(output: &mut String) {
writeln!(
output,
r#"class MetricNode(Generic[T]):
"""A metric node that can fetch data for different indexes."""
def __init__(self, client: BrkClientBase, path: str):
self._client = client
self._path = path
def get(self) -> List[T]:
"""Fetch all data points for this metric."""
return self._client.get(self._path)
def get_range(self, from_date: str, to_date: str) -> List[T]:
"""Fetch data points within a date range."""
return self._client.get(f"{{self._path}}?from={{from_date}}&to={{to_date}}")
"#
)
.unwrap();
}
/// Generate index accessor classes
fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) {
if patterns.is_empty() {
return;
}
writeln!(output, "# Index accessor classes\n").unwrap();
for pattern in patterns {
writeln!(output, "class {}(Generic[T]):", pattern.name).unwrap();
writeln!(output, " \"\"\"Index accessor for metrics with {} indexes.\"\"\"", pattern.indexes.len()).unwrap();
writeln!(output, " ").unwrap();
writeln!(output, " def __init__(self, client: BrkClientBase, base_path: str):").unwrap();
for index in &pattern.indexes {
let field_name = index_to_snake_case(index);
let path_segment = index.serialize_short();
writeln!(
output,
" self.{}: MetricNode[T] = MetricNode(client, f'{{base_path}}/{}')",
field_name, path_segment
).unwrap();
}
writeln!(output).unwrap();
}
}
/// Convert an Index to a snake_case field name (e.g., DateIndex -> by_date)
fn index_to_snake_case(index: &Index) -> String {
let short = index.serialize_short();
format!("by_{}", to_snake_case(short))
}
/// Generate structural pattern classes
fn generate_structural_patterns(output: &mut String, patterns: &[StructuralPattern], metadata: &ClientMetadata) {
if patterns.is_empty() {
return;
}
writeln!(output, "# Reusable structural pattern classes\n").unwrap();
for pattern in patterns {
writeln!(output, "class {}:", pattern.name).unwrap();
writeln!(output, " \"\"\"Pattern struct for repeated tree structure.\"\"\"").unwrap();
writeln!(output, " ").unwrap();
writeln!(output, " def __init__(self, client: BrkClientBase, base_path: str):").unwrap();
for field in &pattern.fields {
let py_type = field_to_python_type(field, metadata);
if metadata.is_pattern_type(&field.rust_type) {
writeln!(
output,
" self.{}: {} = {}(client, f'{{base_path}}/{}')",
to_snake_case(&field.name), py_type, field.rust_type, field.name
).unwrap();
} else if field_uses_accessor(field, metadata) {
let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap();
writeln!(
output,
" self.{}: {} = {}(client, f'{{base_path}}/{}')",
to_snake_case(&field.name), py_type, accessor.name, field.name
).unwrap();
} else {
writeln!(
output,
" self.{}: {} = MetricNode(client, f'{{base_path}}/{}')",
to_snake_case(&field.name), py_type, field.name
).unwrap();
}
}
writeln!(output).unwrap();
}
}
/// Convert pattern field to Python type annotation
fn field_to_python_type(field: &PatternField, metadata: &ClientMetadata) -> String {
if metadata.is_pattern_type(&field.rust_type) {
// Pattern type - use pattern name directly
field.rust_type.clone()
} else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) {
// Leaf with a reusable accessor pattern
let py_type = json_type_to_python(&field.json_type);
format!("{}[{}]", accessor.name, py_type)
} else {
// Leaf with unique index set - use MetricNode directly
let py_type = json_type_to_python(&field.json_type);
format!("MetricNode[{}]", py_type)
}
}
/// Check if a field should use an index accessor
fn field_uses_accessor(field: &PatternField, metadata: &ClientMetadata) -> bool {
metadata.find_index_set_pattern(&field.indexes).is_some()
}
/// Convert JSON Schema type to Python type
fn json_type_to_python(json_type: &str) -> &str {
match json_type {
"integer" => "int",
"number" => "float",
"boolean" => "bool",
"string" => "str",
"array" => "List",
"object" => "dict",
_ => "Any",
}
}
/// Generate tree classes
fn generate_tree_classes(
output: &mut String,
catalog: &TreeNode,
metadata: &ClientMetadata,
) {
writeln!(output, "# Catalog tree classes\n").unwrap();
let pattern_lookup = metadata.pattern_lookup();
let mut generated = HashSet::new();
generate_tree_class(output, "CatalogTree", catalog, &pattern_lookup, metadata, &mut generated);
}
/// Recursively generate tree classes
fn generate_tree_class(
output: &mut String,
name: &str,
node: &TreeNode,
pattern_lookup: &std::collections::HashMap<Vec<PatternField>, String>,
metadata: &ClientMetadata,
generated: &mut HashSet<String>,
) {
if let TreeNode::Branch(children) = node {
// Build signature
let fields = get_node_fields(children, pattern_lookup);
// Skip if this matches a pattern (already generated)
if pattern_lookup.contains_key(&fields) && pattern_lookup.get(&fields) != Some(&name.to_string()) {
return;
}
if generated.contains(name) {
return;
}
generated.insert(name.to_string());
writeln!(output, "class {}:", name).unwrap();
writeln!(output, " \"\"\"Catalog tree node.\"\"\"").unwrap();
writeln!(output, " ").unwrap();
writeln!(output, " def __init__(self, client: BrkClientBase, base_path: str = ''):").unwrap();
for field in &fields {
let py_type = field_to_python_type(field, metadata);
if metadata.is_pattern_type(&field.rust_type) {
writeln!(
output,
" self.{}: {} = {}(client, f'{{base_path}}/{}')",
to_snake_case(&field.name), py_type, field.rust_type, field.name
).unwrap();
} else if field_uses_accessor(field, metadata) {
let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap();
writeln!(
output,
" self.{}: {} = {}(client, f'{{base_path}}/{}')",
to_snake_case(&field.name), py_type, accessor.name, field.name
).unwrap();
} else {
writeln!(
output,
" self.{}: {} = MetricNode(client, f'{{base_path}}/{}')",
to_snake_case(&field.name), py_type, field.name
).unwrap();
}
}
writeln!(output).unwrap();
// Generate child classes
for (child_name, child_node) in children {
if let TreeNode::Branch(grandchildren) = child_node {
let child_fields = get_node_fields(grandchildren, pattern_lookup);
if !pattern_lookup.contains_key(&child_fields) {
let child_class_name = format!("{}_{}", name, to_pascal_case(child_name));
generate_tree_class(output, &child_class_name, child_node, pattern_lookup, metadata, generated);
}
}
}
}
}
/// Generate the main client class
fn generate_main_client(output: &mut String, endpoints: &[Endpoint]) {
writeln!(output, "class BrkClient(BrkClientBase):").unwrap();
writeln!(output, " \"\"\"Main BRK client with catalog tree and API methods.\"\"\"").unwrap();
writeln!(output, " ").unwrap();
writeln!(output, " def __init__(self, base_url: str = 'http://localhost:3000', timeout: float = 30.0):").unwrap();
writeln!(output, " super().__init__(base_url, timeout)").unwrap();
writeln!(output, " self.tree = CatalogTree(self)").unwrap();
writeln!(output).unwrap();
// Generate API methods
generate_api_methods(output, endpoints);
}
/// Generate API methods from OpenAPI endpoints
fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
for endpoint in endpoints {
if endpoint.method != "GET" {
continue;
}
let method_name = endpoint_to_method_name(endpoint);
let return_type = endpoint.response_type.as_deref().unwrap_or("Any");
// Build method signature
let params = build_method_params(endpoint);
writeln!(output, " def {}(self{}) -> {}:", method_name, params, return_type).unwrap();
// Docstring
if let Some(summary) = &endpoint.summary {
writeln!(output, " \"\"\"{}\"\"\"", summary).unwrap();
}
// Build path
let path = build_path_template(&endpoint.path, &endpoint.path_params);
if endpoint.query_params.is_empty() {
writeln!(output, " return self.get(f'{}')", path).unwrap();
} else {
writeln!(output, " params = []").unwrap();
for param in &endpoint.query_params {
if param.required {
writeln!(output, " params.append(f'{}={{{}}}')", param.name, param.name).unwrap();
} else {
writeln!(output, " if {} is not None: params.append(f'{}={{{}}}')", param.name, param.name, param.name).unwrap();
}
}
writeln!(output, " query = '&'.join(params)").unwrap();
writeln!(output, " return self.get(f'{}{{\"?\" + query if query else \"\"}}')", path).unwrap();
}
writeln!(output).unwrap();
}
}
fn endpoint_to_method_name(endpoint: &Endpoint) -> String {
if let Some(op_id) = &endpoint.operation_id {
return to_snake_case(op_id);
}
let parts: Vec<&str> = endpoint.path.split('/').filter(|s| !s.is_empty() && !s.starts_with('{')).collect();
format!("get_{}", parts.join("_"))
}
fn build_method_params(endpoint: &Endpoint) -> String {
let mut params = Vec::new();
for param in &endpoint.path_params {
params.push(format!(", {}: str", param.name));
}
for param in &endpoint.query_params {
if param.required {
params.push(format!(", {}: str", param.name));
} else {
params.push(format!(", {}: Optional[str] = None", param.name));
}
}
params.join("")
}
fn build_path_template(path: &str, path_params: &[super::Parameter]) -> String {
let mut result = path.to_string();
for param in path_params {
let placeholder = format!("{{{}}}", param.name);
let interpolation = format!("{{{{{}}}}}", param.name);
result = result.replace(&placeholder, &interpolation);
}
result
}
/// Convert JSON Schema to Python type hint
pub fn schema_to_python_type(schema: &serde_json::Value) -> String {
if let Some(ty) = schema.get("type").and_then(|v| v.as_str()) {
match ty {
"null" => "None".to_string(),
"boolean" => "bool".to_string(),
"integer" => "int".to_string(),
"number" => "float".to_string(),
"string" => "str".to_string(),
"array" => {
if let Some(items) = schema.get("items") {
format!("List[{}]", schema_to_python_type(items))
} else {
"List[Any]".to_string()
}
}
"object" => "dict[str, Any]".to_string(),
_ => "Any".to_string(),
}
} else if schema.get("anyOf").is_some() || schema.get("oneOf").is_some() {
"Any".to_string()
} else if let Some(reference) = schema.get("$ref").and_then(|v| v.as_str()) {
reference.rsplit('/').next().unwrap_or("Any").to_string()
} else {
"Any".to_string()
}
}

View File

@@ -1,10 +1,533 @@
use std::collections::HashSet;
use std::fmt::Write as FmtWrite;
use std::fs;
use std::io; use std::io;
use std::path::Path; use std::path::Path;
use super::ClientMetadata; use brk_types::{Index, TreeNode};
use super::{ClientMetadata, Endpoint, IndexSetPattern, PatternField, StructuralPattern, get_node_fields, to_pascal_case, to_snake_case};
/// Generate Rust client from metadata and OpenAPI endpoints
pub fn generate_rust_client(
metadata: &ClientMetadata,
endpoints: &[Endpoint],
output_dir: &Path,
) -> io::Result<()> {
let mut output = String::new();
// Header
writeln!(output, "// Auto-generated BRK Rust client").unwrap();
writeln!(output, "// Do not edit manually\n").unwrap();
writeln!(output, "#![allow(non_camel_case_types)]").unwrap();
writeln!(output, "#![allow(dead_code)]\n").unwrap();
// Imports
generate_imports(&mut output);
// Generate base client
generate_base_client(&mut output);
// Generate MetricNode
generate_metric_node(&mut output);
// Generate index accessor structs (for each unique set of indexes)
generate_index_accessors(&mut output, &metadata.index_set_patterns);
// Generate pattern structs (reusable, appearing 2+ times)
generate_pattern_structs(&mut output, &metadata.structural_patterns, metadata);
// Generate tree - each node uses its pattern or is generated inline
generate_tree(&mut output, &metadata.catalog, metadata);
// Generate main client with API methods
generate_main_client(&mut output, endpoints);
fs::write(output_dir.join("client.rs"), output)?;
/// Generate Rust client from metadata
pub fn generate_rust_client(_metadata: &ClientMetadata, _output_dir: &Path) -> io::Result<()> {
// TODO: Implement Rust client generation
Ok(()) Ok(())
} }
fn generate_imports(output: &mut String) {
writeln!(
output,
r#"use std::marker::PhantomData;
use serde::de::DeserializeOwned;
use brk_types::*;
"#
)
.unwrap();
}
fn generate_base_client(output: &mut String) {
writeln!(
output,
r#"/// Error type for BRK client operations.
#[derive(Debug)]
pub struct BrkError {{
pub message: String,
}}
impl std::fmt::Display for BrkError {{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{
write!(f, "{{}}", self.message)
}}
}}
impl std::error::Error for BrkError {{}}
/// Result type for BRK client operations.
pub type Result<T> = std::result::Result<T, BrkError>;
/// Options for configuring the BRK client.
#[derive(Debug, Clone)]
pub struct BrkClientOptions {{
pub base_url: String,
pub timeout_ms: u64,
}}
impl Default for BrkClientOptions {{
fn default() -> Self {{
Self {{
base_url: "http://localhost:3000".to_string(),
timeout_ms: 30000,
}}
}}
}}
/// Base HTTP client for making requests.
#[derive(Debug, Clone)]
pub struct BrkClientBase {{
base_url: String,
client: reqwest::blocking::Client,
}}
impl BrkClientBase {{
/// Create a new client with the given base URL.
pub fn new(base_url: impl Into<String>) -> Result<Self> {{
let base_url = base_url.into();
let client = reqwest::blocking::Client::new();
Ok(Self {{ base_url, client }})
}}
/// Create a new client with options.
pub fn with_options(options: BrkClientOptions) -> Result<Self> {{
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_millis(options.timeout_ms))
.build()
.map_err(|e| BrkError {{ message: e.to_string() }})?;
Ok(Self {{
base_url: options.base_url,
client,
}})
}}
/// Make a GET request.
pub fn get<T: DeserializeOwned>(&self, path: &str) -> Result<T> {{
let url = format!("{{}}{{}}", self.base_url, path);
self.client
.get(&url)
.send()
.map_err(|e| BrkError {{ message: e.to_string() }})?
.json()
.map_err(|e| BrkError {{ message: e.to_string() }})
}}
}}
"#
)
.unwrap();
}
fn generate_metric_node(output: &mut String) {
writeln!(
output,
r#"/// A metric node that can fetch data for different indexes.
pub struct MetricNode<'a, T> {{
client: &'a BrkClientBase,
path: String,
_marker: PhantomData<T>,
}}
impl<'a, T: DeserializeOwned> MetricNode<'a, T> {{
pub fn new(client: &'a BrkClientBase, path: String) -> Self {{
Self {{
client,
path,
_marker: PhantomData,
}}
}}
/// Fetch all data points for this metric.
pub fn get(&self) -> Result<Vec<T>> {{
self.client.get(&self.path)
}}
/// Fetch data points within a date range.
pub fn get_range(&self, from: &str, to: &str) -> Result<Vec<T>> {{
let path = format!("{{}}?from={{}}&to={{}}", self.path, from, to);
self.client.get(&path)
}}
}}
"#
)
.unwrap();
}
/// Generate index accessor structs for each unique set of indexes
fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) {
if patterns.is_empty() {
return;
}
writeln!(output, "// Index accessor structs\n").unwrap();
for pattern in patterns {
writeln!(output, "/// Index accessor for metrics with {} indexes.", pattern.indexes.len()).unwrap();
writeln!(output, "pub struct {}<'a, T> {{", pattern.name).unwrap();
for index in &pattern.indexes {
let field_name = index_to_field_name(index);
writeln!(output, " pub {}: MetricNode<'a, T>,", field_name).unwrap();
}
writeln!(output, " _marker: PhantomData<T>,").unwrap();
writeln!(output, "}}\n").unwrap();
// Generate impl block with constructor
writeln!(output, "impl<'a, T: DeserializeOwned> {}<'a, T> {{", pattern.name).unwrap();
writeln!(output, " pub fn new(client: &'a BrkClientBase, base_path: &str) -> Self {{").unwrap();
writeln!(output, " Self {{").unwrap();
for index in &pattern.indexes {
let field_name = index_to_field_name(index);
let path_segment = index.serialize_short();
writeln!(
output,
" {}: MetricNode::new(client, format!(\"{{base_path}}/{}\")),",
field_name, path_segment
).unwrap();
}
writeln!(output, " _marker: PhantomData,").unwrap();
writeln!(output, " }}").unwrap();
writeln!(output, " }}").unwrap();
writeln!(output, "}}\n").unwrap();
}
}
/// Convert an Index to a snake_case field name (e.g., DateIndex -> by_date)
fn index_to_field_name(index: &Index) -> String {
let short = index.serialize_short();
format!("by_{}", to_snake_case(short))
}
/// Generate pattern structs (those appearing 2+ times)
fn generate_pattern_structs(output: &mut String, patterns: &[StructuralPattern], metadata: &ClientMetadata) {
if patterns.is_empty() {
return;
}
writeln!(output, "// Reusable pattern structs\n").unwrap();
for pattern in patterns {
writeln!(output, "/// Pattern struct for repeated tree structure.").unwrap();
writeln!(output, "pub struct {}<'a> {{", pattern.name).unwrap();
for field in &pattern.fields {
let field_name = to_snake_case(&field.name);
let type_annotation = field_to_type_annotation(field, metadata);
writeln!(output, " pub {}: {},", field_name, type_annotation).unwrap();
}
writeln!(output, "}}\n").unwrap();
// Generate impl block with constructor
writeln!(output, "impl<'a> {}<'a> {{", pattern.name).unwrap();
writeln!(output, " pub fn new(client: &'a BrkClientBase, base_path: &str) -> Self {{").unwrap();
writeln!(output, " Self {{").unwrap();
for field in &pattern.fields {
let field_name = to_snake_case(&field.name);
if metadata.is_pattern_type(&field.rust_type) {
writeln!(
output,
" {}: {}::new(client, &format!(\"{{base_path}}/{}\"))," ,
field_name, field.rust_type, field.name
).unwrap();
} else if field_uses_accessor(field, metadata) {
let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap();
writeln!(
output,
" {}: {}::new(client, &format!(\"{{base_path}}/{}\"))," ,
field_name, accessor.name, field.name
).unwrap();
} else {
writeln!(
output,
" {}: MetricNode::new(client, format!(\"{{base_path}}/{}\"))," ,
field_name, field.name
).unwrap();
}
}
writeln!(output, " }}").unwrap();
writeln!(output, " }}").unwrap();
writeln!(output, "}}\n").unwrap();
}
}
/// Convert a PatternField to the full type annotation
fn field_to_type_annotation(field: &PatternField, metadata: &ClientMetadata) -> String {
if metadata.is_pattern_type(&field.rust_type) {
format!("{}<'a>", field.rust_type)
} else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) {
// Leaf with a reusable accessor pattern
format!("{}<'a, {}>", accessor.name, field.rust_type)
} else {
// Leaf with unique index set - use MetricNode directly
format!("MetricNode<'a, {}>", field.rust_type)
}
}
/// Check if a field should use an index accessor
fn field_uses_accessor(field: &PatternField, metadata: &ClientMetadata) -> bool {
metadata.find_index_set_pattern(&field.indexes).is_some()
}
/// Generate the catalog tree structure
fn generate_tree(
output: &mut String,
catalog: &TreeNode,
metadata: &ClientMetadata,
) {
writeln!(output, "// Catalog tree\n").unwrap();
let pattern_lookup = metadata.pattern_lookup();
let mut generated = HashSet::new();
generate_tree_node(output, "CatalogTree", catalog, &pattern_lookup, metadata, &mut generated);
}
/// Recursively generate tree nodes
fn generate_tree_node(
output: &mut String,
name: &str,
node: &TreeNode,
pattern_lookup: &std::collections::HashMap<Vec<PatternField>, String>,
metadata: &ClientMetadata,
generated: &mut HashSet<String>,
) {
if let TreeNode::Branch(children) = node {
// Build the signature for this node
let mut fields: Vec<PatternField> = children
.iter()
.map(|(child_name, child_node)| {
let (rust_type, json_type, indexes) = match child_node {
TreeNode::Leaf(leaf) => (
leaf.value_type().to_string(),
leaf.schema.get("type").and_then(|v| v.as_str()).unwrap_or("object").to_string(),
leaf.indexes().clone(),
),
TreeNode::Branch(grandchildren) => {
// Get pattern name for this child
let child_fields = get_node_fields(grandchildren, pattern_lookup);
let pattern_name = pattern_lookup
.get(&child_fields)
.cloned()
.unwrap_or_else(|| format!("{}_{}", name, to_pascal_case(child_name)));
(pattern_name.clone(), pattern_name, std::collections::BTreeSet::new())
}
};
PatternField {
name: child_name.clone(),
rust_type,
json_type,
indexes,
}
})
.collect();
fields.sort_by(|a, b| a.name.cmp(&b.name));
// Check if this matches a reusable pattern
if let Some(pattern_name) = pattern_lookup.get(&fields) {
// This node matches a pattern that will be generated separately
// Don't generate it here, it's already in pattern_structs
if pattern_name != name {
return;
}
}
// Generate this struct if not already generated
if generated.contains(name) {
return;
}
generated.insert(name.to_string());
writeln!(output, "/// Catalog tree node.").unwrap();
writeln!(output, "pub struct {}<'a> {{", name).unwrap();
for field in &fields {
let field_name = to_snake_case(&field.name);
let type_annotation = field_to_type_annotation(field, metadata);
writeln!(output, " pub {}: {},", field_name, type_annotation).unwrap();
}
writeln!(output, "}}\n").unwrap();
// Generate impl block
writeln!(output, "impl<'a> {}<'a> {{", name).unwrap();
writeln!(output, " pub fn new(client: &'a BrkClientBase, base_path: &str) -> Self {{").unwrap();
writeln!(output, " Self {{").unwrap();
for field in &fields {
let field_name = to_snake_case(&field.name);
if metadata.is_pattern_type(&field.rust_type) {
writeln!(
output,
" {}: {}::new(client, &format!(\"{{base_path}}/{}\"))," ,
field_name, field.rust_type, field.name
).unwrap();
} else if field_uses_accessor(field, metadata) {
let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap();
writeln!(
output,
" {}: {}::new(client, &format!(\"{{base_path}}/{}\"))," ,
field_name, accessor.name, field.name
).unwrap();
} else {
writeln!(
output,
" {}: MetricNode::new(client, format!(\"{{base_path}}/{}\"))," ,
field_name, field.name
).unwrap();
}
}
writeln!(output, " }}").unwrap();
writeln!(output, " }}").unwrap();
writeln!(output, "}}\n").unwrap();
// Recursively generate child nodes that aren't patterns
for (child_name, child_node) in children {
if let TreeNode::Branch(grandchildren) = child_node {
let child_fields = get_node_fields(grandchildren, pattern_lookup);
if !pattern_lookup.contains_key(&child_fields) {
let child_struct_name = format!("{}_{}", name, to_pascal_case(child_name));
generate_tree_node(output, &child_struct_name, child_node, pattern_lookup, metadata, generated);
}
}
}
}
}
/// Generate the main client struct
fn generate_main_client(output: &mut String, endpoints: &[Endpoint]) {
writeln!(
output,
r#"/// Main BRK client with catalog tree and API methods.
pub struct BrkClient {{
base: BrkClientBase,
}}
impl BrkClient {{
/// Create a new client with the given base URL.
pub fn new(base_url: impl Into<String>) -> Result<Self> {{
Ok(Self {{
base: BrkClientBase::new(base_url)?,
}})
}}
/// Create a new client with options.
pub fn with_options(options: BrkClientOptions) -> Result<Self> {{
Ok(Self {{
base: BrkClientBase::with_options(options)?,
}})
}}
/// Get the catalog tree for navigating metrics.
pub fn tree(&self) -> CatalogTree<'_> {{
CatalogTree::new(&self.base, "")
}}
"#
)
.unwrap();
// Generate API methods
generate_api_methods(output, endpoints);
writeln!(output, "}}").unwrap();
}
/// Generate API methods from OpenAPI endpoints
fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
for endpoint in endpoints {
if endpoint.method != "GET" {
continue;
}
let method_name = endpoint_to_method_name(endpoint);
let return_type = endpoint.response_type.as_deref().unwrap_or("serde_json::Value");
// Build doc comment
writeln!(output, " /// {}", endpoint.summary.as_deref().unwrap_or(&method_name)).unwrap();
// Build method signature
let params = build_method_params(endpoint);
writeln!(output, " pub fn {}(&self{}) -> Result<{}> {{", method_name, params, return_type).unwrap();
// Build path
let path = build_path_template(&endpoint.path, &endpoint.path_params);
if endpoint.query_params.is_empty() {
writeln!(output, " self.base.get(&format!(\"{}\"))", path).unwrap();
} else {
writeln!(output, " let mut query = Vec::new();").unwrap();
for param in &endpoint.query_params {
if param.required {
writeln!(output, " query.push(format!(\"{}={{}}\", {}));", param.name, param.name).unwrap();
} else {
writeln!(output, " if let Some(v) = {} {{ query.push(format!(\"{}={{}}\", v)); }}", param.name, param.name).unwrap();
}
}
writeln!(output, " let query_str = if query.is_empty() {{ String::new() }} else {{ format!(\"?{{}}\", query.join(\"&\")) }};").unwrap();
writeln!(output, " self.base.get(&format!(\"{}{{}}\", query_str))", path).unwrap();
}
writeln!(output, " }}\n").unwrap();
}
}
fn endpoint_to_method_name(endpoint: &Endpoint) -> String {
if let Some(op_id) = &endpoint.operation_id {
return to_snake_case(op_id);
}
let parts: Vec<&str> = endpoint.path.split('/').filter(|s| !s.is_empty() && !s.starts_with('{')).collect();
format!("get_{}", parts.join("_"))
}
fn build_method_params(endpoint: &Endpoint) -> String {
let mut params = Vec::new();
for param in &endpoint.path_params {
params.push(format!(", {}: &str", param.name));
}
for param in &endpoint.query_params {
if param.required {
params.push(format!(", {}: &str", param.name));
} else {
params.push(format!(", {}: Option<&str>", param.name));
}
}
params.join("")
}
fn build_path_template(path: &str, path_params: &[super::Parameter]) -> String {
let mut result = path.to_string();
for param in path_params {
let placeholder = format!("{{{}}}", param.name);
let interpolation = format!("{{{}}}", param.name);
result = result.replace(&placeholder, &interpolation);
}
result
}

View File

@@ -1,4 +1,5 @@
use std::collections::{BTreeMap, BTreeSet}; use std::collections::{BTreeSet, HashMap};
use std::hash::{Hash, Hasher};
use brk_query::Vecs; use brk_query::Vecs;
use brk_types::{Index, TreeNode}; use brk_types::{Index, TreeNode};
@@ -6,94 +7,236 @@ use brk_types::{Index, TreeNode};
/// Metadata extracted from brk_query for client generation /// Metadata extracted from brk_query for client generation
#[derive(Debug)] #[derive(Debug)]
pub struct ClientMetadata { pub struct ClientMetadata {
/// All metrics with their available indexes and value type
pub metrics: BTreeMap<String, MetricInfo>,
/// The catalog tree structure (with schemas in leaves) /// The catalog tree structure (with schemas in leaves)
pub catalog: TreeNode, pub catalog: TreeNode,
/// Discovered patterns (sets of indexes that appear together frequently) /// Structural patterns - tree node shapes that repeat
pub patterns: Vec<IndexPattern>, pub structural_patterns: Vec<StructuralPattern>,
/// All indexes used across the catalog
pub used_indexes: BTreeSet<Index>,
/// Index set patterns - sets of indexes that appear together on metrics
pub index_set_patterns: Vec<IndexSetPattern>,
} }
/// Information about a single metric /// A pattern of indexes that appear together on multiple metrics
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct MetricInfo { pub struct IndexSetPattern {
/// Metric name (e.g., "difficulty", "supply_total") /// Pattern name (e.g., "DateHeightIndexes")
pub name: String, pub name: String,
/// Available indexes for this metric /// The set of indexes
pub indexes: BTreeSet<Index>, pub indexes: BTreeSet<Index>,
/// Value type name (e.g., "Sats", "StoredF64")
pub value_type: String,
} }
/// A pattern of indexes that appears multiple times across metrics /// A structural pattern - a branch structure that appears multiple times in the tree
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct IndexPattern { pub struct StructuralPattern {
/// Unique identifier for this pattern /// Pattern name - sanitized for all languages (e.g., "BaseCumulativeSum")
pub id: usize, pub name: String,
/// The set of indexes in this pattern /// Ordered list of child fields (sorted by field name)
pub indexes: BTreeSet<Index>, pub fields: Vec<PatternField>,
/// How many metrics use this exact pattern
pub usage_count: usize,
} }
/// A field in a structural pattern
#[derive(Debug, Clone, PartialOrd, Ord)]
pub struct PatternField {
/// Field name
pub name: String,
/// Rust type: brk_types type for leaves ("Sats", "StoredF64") or pattern name for branches
pub rust_type: String,
/// JSON type from schema: "integer", "number", "string", "boolean", or pattern name for branches
pub json_type: String,
/// For leaves: the set of supported indexes. Empty for branches.
pub indexes: BTreeSet<Index>,
}
// Manual implementations of Hash/Eq/PartialEq that exclude `indexes`
// since indexes aren't part of the structural pattern identity
impl Hash for PatternField {
fn hash<H: Hasher>(&self, state: &mut H) {
self.name.hash(state);
self.rust_type.hash(state);
self.json_type.hash(state);
// indexes excluded from hash
}
}
impl PartialEq for PatternField {
fn eq(&self, other: &Self) -> bool {
self.name == other.name
&& self.rust_type == other.rust_type
&& self.json_type == other.json_type
// indexes excluded from equality
}
}
impl Eq for PatternField {}
impl ClientMetadata { impl ClientMetadata {
/// Extract metadata from brk_query::Vecs /// Extract metadata from brk_query::Vecs
pub fn from_vecs(vecs: &Vecs) -> Self { pub fn from_vecs(vecs: &Vecs) -> Self {
let mut metrics = BTreeMap::new(); let catalog = vecs.catalog().clone();
let mut pattern_counts: BTreeMap<BTreeSet<Index>, usize> = BTreeMap::new(); let structural_patterns = detect_structural_patterns(&catalog);
let (used_indexes, index_set_patterns) = detect_index_patterns(&catalog);
// Extract metric information
for (name, index_to_vec) in &vecs.metric_to_index_to_vec {
let indexes: BTreeSet<Index> = index_to_vec.keys().copied().collect();
// Get value type from the first available vec
let value_type = index_to_vec
.values()
.next()
.map(|v| v.value_type_to_string().to_string())
.unwrap_or_else(|| "unknown".to_string());
// Count pattern usage
*pattern_counts.entry(indexes.clone()).or_insert(0) += 1;
metrics.insert(
name.to_string(),
MetricInfo {
name: name.to_string(),
indexes,
value_type,
},
);
}
// Extract patterns that are used by multiple metrics
let mut patterns: Vec<IndexPattern> = pattern_counts
.into_iter()
.filter(|(_, count)| *count >= 2) // Only patterns used by 2+ metrics
.enumerate()
.map(|(id, (indexes, usage_count))| IndexPattern {
id,
indexes,
usage_count,
})
.collect();
// Sort by usage count descending
patterns.sort_by(|a, b| b.usage_count.cmp(&a.usage_count));
ClientMetadata { ClientMetadata {
metrics, catalog,
catalog: vecs.catalog().clone(), structural_patterns,
patterns, used_indexes,
index_set_patterns,
} }
} }
/// Find the pattern that matches a metric's indexes, if any /// Check if an index set matches a pattern
pub fn find_pattern_for_metric(&self, metric: &MetricInfo) -> Option<&IndexPattern> { pub fn find_index_set_pattern(&self, indexes: &BTreeSet<Index>) -> Option<&IndexSetPattern> {
self.patterns self.index_set_patterns.iter().find(|p| &p.indexes == indexes)
.iter()
.find(|p| p.indexes == metric.indexes)
} }
/// Check if a type is a pattern (vs a primitive leaf type)
pub fn is_pattern_type(&self, type_name: &str) -> bool {
self.structural_patterns.iter().any(|p| p.name == type_name)
}
/// Build a lookup map from field signatures to pattern names
pub fn pattern_lookup(&self) -> HashMap<Vec<PatternField>, String> {
self.structural_patterns
.iter()
.map(|p| (p.fields.clone(), p.name.clone()))
.collect()
}
}
/// Detect structural patterns in the tree using a bottom-up approach.
/// For every branch node, create a signature from its children (sorted field names + types).
/// Patterns that appear 2+ times are deduplicated.
fn detect_structural_patterns(tree: &TreeNode) -> Vec<StructuralPattern> {
// Map from sorted fields signature to pattern name
let mut signature_to_pattern: HashMap<Vec<PatternField>, String> = HashMap::new();
// Count how many times each signature appears
let mut signature_counts: HashMap<Vec<PatternField>, usize> = HashMap::new();
// Process tree bottom-up to resolve all branch types
resolve_branch_patterns(tree, &mut signature_to_pattern, &mut signature_counts);
// Build final list of patterns (only those appearing 2+ times)
let mut patterns: Vec<StructuralPattern> = signature_to_pattern
.into_iter()
.filter(|(sig, _)| signature_counts.get(sig).copied().unwrap_or(0) >= 2)
.map(|(fields, name)| StructuralPattern { name, fields })
.collect();
// Sort by number of fields descending (larger patterns first)
patterns.sort_by(|a, b| b.fields.len().cmp(&a.fields.len()));
patterns
}
/// Recursively resolve branch patterns bottom-up.
/// Returns the pattern name for this node if it's a branch, or None if it's a leaf.
fn resolve_branch_patterns(
node: &TreeNode,
signature_to_pattern: &mut HashMap<Vec<PatternField>, String>,
signature_counts: &mut HashMap<Vec<PatternField>, usize>,
) -> Option<String> {
match node {
TreeNode::Leaf(_) => {
// Leaves don't have patterns, return None
None
}
TreeNode::Branch(children) => {
// First, recursively resolve all children
let mut fields: Vec<PatternField> = Vec::new();
for (child_name, child_node) in children {
let (rust_type, json_type, indexes) = match child_node {
TreeNode::Leaf(leaf) => (
leaf.value_type().to_string(),
schema_to_json_type(&leaf.schema),
leaf.indexes().clone(),
),
TreeNode::Branch(_) => {
// Branch: recursively get its pattern name
let pattern_name = resolve_branch_patterns(child_node, signature_to_pattern, signature_counts)
.unwrap_or_else(|| "Unknown".to_string());
(pattern_name.clone(), pattern_name, BTreeSet::new())
}
};
fields.push(PatternField {
name: child_name.clone(),
rust_type,
json_type,
indexes,
});
}
// Sort fields by name for consistent signatures
fields.sort_by(|a, b| a.name.cmp(&b.name));
// Increment count for this signature
*signature_counts.entry(fields.clone()).or_insert(0) += 1;
// Get or create pattern name for this signature
let pattern_name = signature_to_pattern
.entry(fields.clone())
.or_insert_with(|| generate_pattern_name_from_fields(&fields))
.clone();
Some(pattern_name)
}
}
}
/// Generate a sanitized pattern name from fields.
/// Names must be valid identifiers in all target languages (Rust, JS, Python).
fn generate_pattern_name_from_fields(fields: &[PatternField]) -> String {
// Join field names with underscores, then convert to PascalCase
let joined: Vec<&str> = fields.iter().map(|f| f.name.as_str()).collect();
let raw_name = joined.join("_");
// Sanitize: ensure it starts with a letter (prepend "P_" if starts with digit)
let sanitized = if raw_name.chars().next().map(|c| c.is_ascii_digit()).unwrap_or(false) {
format!("P_{}", raw_name)
} else {
raw_name
};
to_pascal_case(&sanitized)
}
/// Extract JSON type from JSON Schema
fn schema_to_json_type(schema: &serde_json::Value) -> String {
if let Some(ty) = schema.get("type").and_then(|v| v.as_str()) {
ty.to_string()
} else {
"object".to_string()
}
}
/// Get the field signature for a branch node's children
pub fn get_node_fields(
children: &std::collections::BTreeMap<String, TreeNode>,
pattern_lookup: &HashMap<Vec<PatternField>, String>,
) -> Vec<PatternField> {
let mut fields: Vec<PatternField> = children
.iter()
.map(|(name, node)| {
let (rust_type, json_type, indexes) = match node {
TreeNode::Leaf(leaf) => (
leaf.value_type().to_string(),
schema_to_json_type(&leaf.schema),
leaf.indexes().clone(),
),
TreeNode::Branch(grandchildren) => {
let child_fields = get_node_fields(grandchildren, pattern_lookup);
let pattern_name = pattern_lookup.get(&child_fields).cloned().unwrap_or_else(|| "Unknown".to_string());
(pattern_name.clone(), pattern_name, BTreeSet::new())
}
};
PatternField { name: name.clone(), rust_type, json_type, indexes }
})
.collect();
fields.sort_by(|a, b| a.name.cmp(&b.name));
fields
} }
/// Convert a metric name to PascalCase (for struct/class names) /// Convert a metric name to PascalCase (for struct/class names)
@@ -132,6 +275,73 @@ pub fn to_camel_case(s: &str) -> String {
} }
} }
/// Detect index patterns - collect all indexes and find sets that appear 2+ times
fn detect_index_patterns(tree: &TreeNode) -> (BTreeSet<Index>, Vec<IndexSetPattern>) {
let mut used_indexes: BTreeSet<Index> = BTreeSet::new();
let mut index_sets: Vec<BTreeSet<Index>> = Vec::new();
// Traverse tree and collect index information from leaves
collect_indexes_from_tree(tree, &mut used_indexes, &mut index_sets);
// Count occurrences of each unique index set
let mut index_set_counts: Vec<(BTreeSet<Index>, usize)> = Vec::new();
for index_set in index_sets {
if let Some(entry) = index_set_counts.iter_mut().find(|(s, _)| s == &index_set) {
entry.1 += 1;
} else {
index_set_counts.push((index_set, 1));
}
}
// Build patterns for index sets appearing 2+ times
let mut patterns: Vec<IndexSetPattern> = index_set_counts
.into_iter()
.filter(|(indexes, count)| *count >= 2 && !indexes.is_empty())
.map(|(indexes, _)| IndexSetPattern {
name: generate_index_set_name(&indexes),
indexes,
})
.collect();
// Sort by number of indexes descending
patterns.sort_by(|a, b| b.indexes.len().cmp(&a.indexes.len()));
(used_indexes, patterns)
}
/// Recursively collect indexes from tree leaves
fn collect_indexes_from_tree(
node: &TreeNode,
used_indexes: &mut BTreeSet<Index>,
index_sets: &mut Vec<BTreeSet<Index>>,
) {
match node {
TreeNode::Leaf(leaf) => {
// Add all indexes from this leaf to the global set
used_indexes.extend(leaf.indexes().iter().cloned());
// Collect this index set
index_sets.push(leaf.indexes().clone());
}
TreeNode::Branch(children) => {
for (_, child) in children {
collect_indexes_from_tree(child, used_indexes, index_sets);
}
}
}
}
/// Generate a name for an index set pattern
fn generate_index_set_name(indexes: &BTreeSet<Index>) -> String {
if indexes.len() == 1 {
let index = indexes.iter().next().unwrap();
return format!("{}Accessor", to_pascal_case(index.serialize_short()));
}
// For multiple indexes, create a descriptive name
let names: Vec<&str> = indexes.iter().map(|i| i.serialize_short()).collect();
format!("{}Accessor", to_pascal_case(&names.join("_")))
}
/// Convert a serde_json::Value (JSON Schema) to a JSDoc type annotation /// Convert a serde_json::Value (JSON Schema) to a JSDoc type annotation
pub fn schema_to_jsdoc(schema: &serde_json::Value) -> String { pub fn schema_to_jsdoc(schema: &serde_json::Value) -> String {
if let Some(ty) = schema.get("type").and_then(|v| v.as_str()) { if let Some(ty) = schema.get("type").and_then(|v| v.as_str()) {

View File

@@ -1 +1,2 @@
bottlenecks.md bottlenecks.md
BUG.md

View File

@@ -34,10 +34,30 @@ impl<'a> AddressLookup<'a> {
match map.entry(type_index) { match map.entry(type_index) {
Entry::Occupied(entry) => { Entry::Occupied(entry) => {
// Address is in cache. Need to determine if it's been processed
// by process_received (added to a cohort) or just loaded this block.
//
// - If wrapper is New AND funded_txo_count == 0: hasn't received yet,
// was just created in process_outputs this block → New
// - If wrapper is New AND funded_txo_count > 0: received in previous
// block but still in cache (no flush) → Loaded
// - If wrapper is FromLoaded/FromEmpty: loaded from storage → use wrapper
let source = match entry.get() { let source = match entry.get() {
WithAddressDataSource::New(_) => AddressSource::New, WithAddressDataSource::New(data) => {
if data.funded_txo_count == 0 {
AddressSource::New
} else {
AddressSource::Loaded
}
}
WithAddressDataSource::FromLoaded(..) => AddressSource::Loaded, WithAddressDataSource::FromLoaded(..) => AddressSource::Loaded,
WithAddressDataSource::FromEmpty(..) => AddressSource::FromEmpty, WithAddressDataSource::FromEmpty(_, data) => {
if data.utxo_count() == 0 {
AddressSource::FromEmpty
} else {
AddressSource::Loaded
}
}
}; };
(entry.into_mut(), source) (entry.into_mut(), source)
} }

View File

@@ -62,13 +62,30 @@ pub fn process_received(
if AmountBucket::from(prev_balance) != AmountBucket::from(new_balance) { if AmountBucket::from(prev_balance) != AmountBucket::from(new_balance) {
// Crossing cohort boundary - subtract from old, add to new // Crossing cohort boundary - subtract from old, add to new
cohorts let cohort_state = cohorts
.amount_range .amount_range
.get_mut(prev_balance) .get_mut(prev_balance)
.state .state
.as_mut() .as_mut()
.unwrap() .unwrap();
.subtract(addr_data);
// Debug info for tracking down underflow issues
if cohort_state.inner.supply.utxo_count < addr_data.utxo_count() as u64 {
panic!(
"process_received: cohort underflow detected!\n\
output_type={:?}, type_index={:?}\n\
prev_balance={}, new_balance={}, total_value={}\n\
Address: {:?}",
output_type,
type_index,
prev_balance,
new_balance,
total_value,
addr_data
);
}
cohort_state.subtract(addr_data);
addr_data.receive_outputs(total_value, price, output_count); addr_data.receive_outputs(total_value, price, output_count);
cohorts cohorts
.amount_range .amount_range

View File

@@ -8,7 +8,7 @@
use brk_error::Result; use brk_error::Result;
use brk_grouper::{ByAddressType, Filtered}; use brk_grouper::{ByAddressType, Filtered};
use brk_types::{CheckedSub, Dollars, Height, Sats, Timestamp, TypeIndex}; use brk_types::{CheckedSub, Dollars, Height, Sats, Timestamp, TypeIndex};
use vecdb::VecIndex; use vecdb::{VecIndex, unlikely};
use super::super::address::HeightToAddressTypeToVec; use super::super::address::HeightToAddressTypeToVec;
use super::super::cohorts::AddressCohorts; use super::super::cohorts::AddressCohorts;
@@ -63,13 +63,36 @@ pub fn process_sent(
if will_be_empty || filters_differ { if will_be_empty || filters_differ {
// Subtract from old cohort // Subtract from old cohort
cohorts let cohort_state = cohorts
.amount_range .amount_range
.get_mut(prev_balance) .get_mut(prev_balance)
.state .state
.as_mut() .as_mut()
.unwrap() .unwrap();
.subtract(addr_data);
// Debug info for tracking down underflow issues
if unlikely(
cohort_state.inner.supply.utxo_count < addr_data.utxo_count() as u64,
) {
panic!(
"process_sent: cohort underflow detected!\n\
Block context: prev_height={:?}, output_type={:?}, type_index={:?}\n\
prev_balance={}, new_balance={}, value={}\n\
will_be_empty={}, filters_differ={}\n\
Address: {:?}",
prev_height,
output_type,
type_index,
prev_balance,
new_balance,
value,
will_be_empty,
filters_differ,
addr_data
);
}
cohort_state.subtract(addr_data);
// Update address data // Update address data
addr_data.send(value, prev_price)?; addr_data.send(value, prev_price)?;

View File

@@ -2,6 +2,7 @@ use std::path::Path;
use brk_error::Result; use brk_error::Result;
use brk_types::{Dollars, Height, LoadedAddressData, Sats}; use brk_types::{Dollars, Height, LoadedAddressData, Sats};
use vecdb::unlikely;
use crate::stateful::states::{RealizedState, SupplyState}; use crate::stateful::states::{RealizedState, SupplyState};
@@ -136,11 +137,46 @@ impl AddressCohortState {
} }
pub fn subtract(&mut self, addressdata: &LoadedAddressData) { pub fn subtract(&mut self, addressdata: &LoadedAddressData) {
self.addr_count = self.addr_count.checked_sub(1).unwrap(); let addr_supply: SupplyState = addressdata.into();
let realized_price = addressdata.realized_price();
// Check for potential underflow before it happens
if unlikely(self.inner.supply.utxo_count < addr_supply.utxo_count) {
panic!(
"AddressCohortState::subtract underflow!\n\
Cohort state: addr_count={}, supply={}\n\
Address being subtracted: {}\n\
Address supply: {}\n\
Realized price: {}\n\
This means the address is not properly tracked in this cohort.",
self.addr_count, self.inner.supply, addressdata, addr_supply, realized_price
);
}
if unlikely(self.inner.supply.value < addr_supply.value) {
panic!(
"AddressCohortState::subtract value underflow!\n\
Cohort state: addr_count={}, supply={}\n\
Address being subtracted: {}\n\
Address supply: {}\n\
Realized price: {}\n\
This means the address is not properly tracked in this cohort.",
self.addr_count, self.inner.supply, addressdata, addr_supply, realized_price
);
}
self.addr_count = self.addr_count.checked_sub(1).unwrap_or_else(|| {
panic!(
"AddressCohortState::subtract addr_count underflow! addr_count=0\n\
Address being subtracted: {}\n\
Realized price: {}",
addressdata, realized_price
)
});
self.inner.decrement_( self.inner.decrement_(
&addressdata.into(), &addr_supply,
addressdata.realized_cap, addressdata.realized_cap,
addressdata.realized_price(), realized_price,
); );
} }

View File

@@ -9,7 +9,7 @@ use brk_types::{Dollars, Height, Sats};
use derive_deref::{Deref, DerefMut}; use derive_deref::{Deref, DerefMut};
use pco::standalone::{simple_decompress, simpler_compress}; use pco::standalone::{simple_decompress, simpler_compress};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use vecdb::Bytes; use vecdb::{Bytes, unlikely};
use crate::{grouped::PERCENTILES_LEN, utils::OptionExt}; use crate::{grouped::PERCENTILES_LEN, utils::OptionExt};
@@ -87,6 +87,25 @@ impl PriceToAmount {
pub fn decrement(&mut self, price: Dollars, supply_state: &SupplyState) { pub fn decrement(&mut self, price: Dollars, supply_state: &SupplyState) {
if let Some(amount) = self.state.um().get_mut(&price) { if let Some(amount) = self.state.um().get_mut(&price) {
if unlikely(*amount < supply_state.value) {
let amount = *amount;
panic!(
"PriceToAmount::decrement underflow!\n\
Path: {:?}\n\
Price: {}\n\
Bucket amount: {}\n\
Trying to decrement by: {}\n\
Supply state: utxo_count={}, value={}\n\
All buckets: {:?}",
self.pathbuf,
price,
amount,
supply_state.value,
supply_state.utxo_count,
supply_state.value,
self.state.u().iter().collect::<Vec<_>>()
);
}
*amount -= supply_state.value; *amount -= supply_state.value;
if *amount == Sats::ZERO { if *amount == Sats::ZERO {
self.state.um().remove(&price); self.state.um().remove(&price);
@@ -95,8 +114,18 @@ impl PriceToAmount {
buckets.decrement(price, supply_state.value); buckets.decrement(price, supply_state.value);
} }
} else { } else {
dbg!(price, &self.pathbuf); panic!(
unreachable!(); "PriceToAmount::decrement price not found!\n\
Path: {:?}\n\
Price: {}\n\
Supply state: utxo_count={}, value={}\n\
All buckets: {:?}",
self.pathbuf,
price,
supply_state.utxo_count,
supply_state.value,
self.state.u().iter().collect::<Vec<_>>()
);
} }
} }

View File

@@ -39,8 +39,22 @@ impl AddAssign<&SupplyState> for SupplyState {
impl SubAssign<&SupplyState> for SupplyState { impl SubAssign<&SupplyState> for SupplyState {
fn sub_assign(&mut self, rhs: &Self) { fn sub_assign(&mut self, rhs: &Self) {
self.utxo_count = self.utxo_count.checked_sub(rhs.utxo_count).unwrap(); self.utxo_count = self.utxo_count.checked_sub(rhs.utxo_count).unwrap_or_else(|| {
self.value = self.value.checked_sub(rhs.value).unwrap(); panic!(
"SupplyState underflow: cohort utxo_count {} < address utxo_count {}. \
This indicates a desync between cohort state and address data. \
Try deleting the compute cache and restarting fresh.",
self.utxo_count, rhs.utxo_count
)
});
self.value = self.value.checked_sub(rhs.value).unwrap_or_else(|| {
panic!(
"SupplyState underflow: cohort value {} < address value {}. \
This indicates a desync between cohort state and address data. \
Try deleting the compute cache and restarting fresh.",
self.value, rhs.value
)
});
} }
} }

View File

@@ -11,6 +11,7 @@ build = "build.rs"
[dependencies] [dependencies]
aide = { workspace = true } aide = { workspace = true }
axum = { workspace = true } axum = { workspace = true }
brk_binder = { workspace = true }
brk_computer = { workspace = true } brk_computer = { workspace = true }
brk_error = { workspace = true } brk_error = { workspace = true }
brk_fetcher = { workspace = true } brk_fetcher = { workspace = true }

View File

@@ -89,6 +89,7 @@ impl Server {
.on_failure(()) .on_failure(())
.on_eos(()); .on_eos(());
let vecs = state.query.inner().vecs();
let router = ApiRouter::new() let router = ApiRouter::new()
.add_api_routes() .add_api_routes()
.add_mcp_routes(&state.query, mcp) .add_mcp_routes(&state.query, mcp)
@@ -133,10 +134,25 @@ impl Server {
info!("Starting server on port {port}..."); info!("Starting server on port {port}...");
let mut openapi = create_openapi(); let mut openapi = create_openapi();
let router = router.finish_api(&mut openapi);
let clients_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.join("brk_binder")
.join("clients");
if clients_path.exists() {
let openapi_json = serde_json::to_string(&openapi).unwrap();
if let Err(e) = brk_binder::generate_clients(vecs, &openapi_json, &clients_path) {
error!("Failed to generate clients: {e}");
} else {
info!("Generated clients at {}", clients_path.display());
}
}
serve( serve(
listener, listener,
router router
.finish_api(&mut openapi)
.layer(Extension(Arc::new(openapi))) .layer(Extension(Arc::new(openapi)))
.into_make_service(), .into_make_service(),
) )

View File

@@ -51,7 +51,13 @@ impl LoadedAddressData {
#[inline] #[inline]
pub fn utxo_count(&self) -> u32 { pub fn utxo_count(&self) -> u32 {
self.funded_txo_count - self.spent_txo_count self.funded_txo_count.checked_sub(self.spent_txo_count).unwrap_or_else(|| {
panic!(
"LoadedAddressData corruption: spent_txo_count ({}) > funded_txo_count ({}). \
Address data: {:?}",
self.spent_txo_count, self.funded_txo_count, self
)
})
} }
#[inline] #[inline]

View File

@@ -110,11 +110,9 @@ impl SaturatingAdd for Sats {
impl SubAssign for Sats { impl SubAssign for Sats {
fn sub_assign(&mut self, rhs: Self) { fn sub_assign(&mut self, rhs: Self) {
*self = self.checked_sub(rhs).unwrap(); *self = self.checked_sub(rhs).unwrap_or_else(|| {
// .unwrap_or_else(|| { panic!("Sats underflow: {} - {} would be negative", self, rhs);
// dbg!((*self, rhs)); });
// unreachable!();
// });
} }
} }