From 4a0ce6337fd673a51580f34a780ee9494cf69922 Mon Sep 17 00:00:00 2001 From: nym21 Date: Sat, 20 Dec 2025 10:16:06 +0100 Subject: [PATCH] global: snapshot --- Cargo.lock | 29 +- crates/brk_binder/.gitignore | 1 + crates/brk_binder/Cargo.toml | 1 + crates/brk_binder/DESIGN.md | 289 +++------ crates/brk_binder/src/generator/javascript.rs | 564 +++++++++++++----- crates/brk_binder/src/generator/mod.rs | 22 +- crates/brk_binder/src/generator/openapi.rs | 214 +++++++ crates/brk_binder/src/generator/python.rs | 426 ++++++++++++- crates/brk_binder/src/generator/rust.rs | 531 ++++++++++++++++- crates/brk_binder/src/generator/types.rs | 346 ++++++++--- crates/brk_computer/.gitignore | 1 + .../src/stateful/process/lookup.rs | 24 +- .../src/stateful/process/received.rs | 23 +- .../brk_computer/src/stateful/process/sent.rs | 31 +- .../src/stateful/states/address_cohort.rs | 42 +- .../src/stateful/states/price_to_amount.rs | 35 +- .../src/stateful/states/supply.rs | 18 +- crates/brk_server/Cargo.toml | 1 + crates/brk_server/src/lib.rs | 18 +- crates/brk_types/src/loadedaddressdata.rs | 8 +- crates/brk_types/src/sats.rs | 8 +- 21 files changed, 2164 insertions(+), 468 deletions(-) create mode 100644 crates/brk_binder/.gitignore create mode 100644 crates/brk_binder/src/generator/openapi.rs diff --git a/Cargo.lock b/Cargo.lock index 48c3b091b..58656ebad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -568,6 +568,7 @@ version = "0.1.0-alpha.0" dependencies = [ "brk_query", "brk_types", + "oas3", "schemars", "serde_json", "vecdb", @@ -1203,6 +1204,7 @@ version = "0.1.0-alpha.0" dependencies = [ "aide", "axum", + "brk_binder", "brk_computer", "brk_error", "brk_fetcher", @@ -1364,9 +1366,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.49" +version = "1.2.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" +checksum = "9f50d563227a1c37cc0a263f64eca3334388c01c5e4c4861a9def205c614383c" dependencies = [ "find-msvc-tools", "jobserver", @@ -3219,6 +3221,23 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "oas3" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f67c885c7b19aaf652e84102035258ecb4e0425a4f71037e187798e367bd87" +dependencies = [ + "derive_more", + "http", + "log", + "once_cell", + "regex", + "semver", + "serde", + "serde_json", + "url", +] + [[package]] name = "objc2" version = "0.6.3" @@ -4198,7 +4217,7 @@ dependencies = [ [[package]] name = "rawdb" -version = "0.4.3" +version = "0.4.4" dependencies = [ "libc", "log", @@ -5388,7 +5407,7 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vecdb" -version = "0.4.3" +version = "0.4.4" dependencies = [ "ctrlc", "log", @@ -5407,7 +5426,7 @@ dependencies = [ [[package]] name = "vecdb_derive" -version = "0.4.3" +version = "0.4.4" dependencies = [ "quote", "syn 2.0.111", diff --git a/crates/brk_binder/.gitignore b/crates/brk_binder/.gitignore new file mode 100644 index 000000000..f94cf1915 --- /dev/null +++ b/crates/brk_binder/.gitignore @@ -0,0 +1 @@ +clients/ diff --git a/crates/brk_binder/Cargo.toml b/crates/brk_binder/Cargo.toml index 564e724af..168cc5160 100644 --- a/crates/brk_binder/Cargo.toml +++ b/crates/brk_binder/Cargo.toml @@ -11,6 +11,7 @@ build = "build.rs" [dependencies] brk_query = { workspace = true } brk_types = { workspace = true } +oas3 = "0.20" schemars = { workspace = true } serde_json = { workspace = true } vecdb = { workspace = true } diff --git a/crates/brk_binder/DESIGN.md b/crates/brk_binder/DESIGN.md index 24d95b985..87d299eef 100644 --- a/crates/brk_binder/DESIGN.md +++ b/crates/brk_binder/DESIGN.md @@ -8,27 +8,29 @@ Generate typed API clients for **Rust, JavaScript, and Python** with: ## Current State -### What Exists +### What's Working ✅ -1. **`js.rs`**: Generates compressed metric catalogs for JS (constants only, no HTTP client) -2. **`tree.rs`**: (kept for reference, not compiled) Brainstorming output for pattern extraction -3. **`generator/`**: Module structure for client generation - - `types.rs`: Intermediate representation (`ClientMetadata`, `MetricInfo`, `IndexPattern`, `schema_to_jsdoc`) - - `rust.rs`: Rust client generation (stub) - - `javascript.rs`: JavaScript + JSDoc client generation ✅ IMPLEMENTED - - `python.rs`: Python client generation (stub) +1. **JS + JSDoc generator**: Generates `client.js` with full JSDoc type annotations +2. **Python generator**: Generates `client.py` with type hints and httpx +3. **Rust generator**: Generates `client.rs` with strong typing and reqwest +4. **schemars integration**: JSON schemas embedded in `MetricLeafWithSchema` for type info +5. **Tree navigation**: `client.tree.blocks.difficulty.fetch()` pattern +6. **OpenAPI integration**: All GET endpoints generate typed methods +7. **Server integration**: brk_server calls brk_binder on startup (when clients/ dir exists) -### What's Working +### Generated Output -- **JS + JSDoc generator**: Generates `client.js` with full JSDoc type annotations -- **schemars integration**: JSON schemas embedded in `MetricLeafWithSchema` for type info -- **Tree navigation**: `client.tree.blocks.difficulty.fetch()` pattern +When `crates/brk_binder/clients/` directory exists, running the server generates: -### What's Missing - -- OpenAPI integration for non-metric endpoints -- Python client implementation -- Rust client implementation +``` +crates/brk_binder/clients/ +├── javascript/ +│ └── client.js # JS + JSDoc with tree + API methods +├── python/ +│ └── client.py # Python with type hints + httpx +└── rust/ + └── client.rs # Rust with reqwest + strong typing +``` ## Target Architecture @@ -56,13 +58,13 @@ const data = await client.tree.supply.active.by_date.fetch(); ```python # Python client = BrkClient("http://localhost:3000") -data = await client.tree.supply.active.by_date.fetch() +data = client.tree.supply.active.by_date.fetch() ``` ```rust // Rust -let client = BrkClient::new("http://localhost:3000"); -let data = client.tree.supply.active.by_date.fetch().await?; +let client = BrkClient::new("http://localhost:3000")?; +let data = client.tree().supply.active.by_date.fetch()?; ``` ## Implementation Details @@ -78,19 +80,11 @@ Each tree leaf becomes a "smart node" holding a client reference: * @template T */ class MetricNode { - /** - * @param {BrkClientBase} client - * @param {string} path - */ constructor(client, path) { this._client = client; this._path = path; } - /** - * Fetch the metric value - * @returns {Promise} - */ async fetch() { return this._client.get(this._path); } @@ -100,30 +94,30 @@ class MetricNode { ```python # Python class MetricNode(Generic[T]): - def __init__(self, client: BrkClient, path: str): + def __init__(self, client: BrkClientBase, path: str): self._client = client self._path = path - async def fetch(self) -> T: - return await self._client.get(self._path) + def fetch(self) -> T: + return self._client.get(self._path) ``` ```rust // Rust -pub struct MetricNode { - client: Arc, +pub struct MetricNode<'a, T> { + client: &'a BrkClientBase, path: &'static str, - _phantom: PhantomData, + _marker: PhantomData, } -impl MetricNode { - pub async fn fetch(&self) -> Result { - self.client.get(&self.path).await +impl<'a, T: DeserializeOwned> MetricNode<'a, T> { + pub fn fetch(&self) -> Result { + self.client.get(self.path) } } ``` -### Pattern Reuse (from tree.rs) +### Pattern Reuse To avoid 20k+ individual types, reuse structural patterns: @@ -142,17 +136,6 @@ struct Supply { } ``` -### Rust Client: Using brk_types - -The Rust client should import `brk_types` rather than generating duplicate types: - -```rust -use brk_types::{Height, Sats, DateIndex, ...}; - -// Response types come from brk_types -pub struct MetricNode { ... } -``` - ## Type Discovery Solution ✅ IMPLEMENTED ### The Problem @@ -163,45 +146,9 @@ Type information was erased at runtime because metrics are stored as `&dyn AnyEx Use `std::any::type_name::()` with caching to extract short type names. -> **Note**: Unlike `PrintableIndex` which needs `to_possible_strings()` for parsing from -> multiple string representations, for values we only need output, so `type_name` suffices. - #### Implementation (vecdb) -Added `short_type_name()` helper in `traits/printable.rs`: - -```rust -pub fn short_type_name() -> &'static str { - static CACHE: OnceLock>> = OnceLock::new(); - - let full: &'static str = std::any::type_name::(); - // ... caching logic, extracts "Sats" from "brk_types::sats::Sats" -} -``` - -Added `value_type_to_string()` to `AnyVec` trait in `traits/any.rs`: - -```rust -pub trait AnyVec: Send + Sync { - // ... existing methods - fn value_type_to_string(&self) -> &'static str; -} -``` - -Implemented in all vec variants: -- `variants/eager/mod.rs` -- `variants/lazy/from1/mod.rs`, `from2/mod.rs`, `from3/mod.rs` -- `variants/raw/inner/mod.rs` -- `variants/compressed/inner/mod.rs` -- `variants/macros.rs` (for wrapper types) - -```rust -fn value_type_to_string(&self) -> &'static str { - short_type_name::() -} -``` - -**No changes needed to brk_types** - works automatically for all types. +Added `short_type_name()` helper and `value_type_to_string()` to `AnyVec` trait. ### Result @@ -219,26 +166,11 @@ for (metric_name, index_to_vec) in &vecs.metric_to_index_to_vec { } ``` -This enables fully typed client generation. - ## TreeNode Enhancement ✅ IMPLEMENTED -### The Problem - -`TreeNode::Leaf` originally held just a `String` (the metric name), losing type and index information. - -### The Solution - Changed `TreeNode::Leaf(String)` to `TreeNode::Leaf(MetricLeafWithSchema)` where: ```rust -#[derive(Debug, Clone, Serialize, PartialEq, Eq, JsonSchema)] -pub struct MetricLeaf { - pub name: String, - pub value_type: String, - pub indexes: BTreeSet, -} - #[derive(Debug, Clone, Serialize, JsonSchema)] pub struct MetricLeafWithSchema { #[serde(flatten)] @@ -248,113 +180,68 @@ pub struct MetricLeafWithSchema { } ``` -#### Implementation +## OpenAPI Integration ✅ IMPLEMENTED -**brk_types/src/treenode.rs**: -- Added `MetricLeaf` struct with `name`, `value_type`, and `indexes` -- Added `MetricLeafWithSchema` wrapper with JSON schema -- Helper methods: `name()`, `value_type()`, `indexes()`, `is_same_metric()`, `merge_indexes()` -- Updated `TreeNode` enum to use `Leaf(MetricLeafWithSchema)` +### Flow -**brk_traversable/src/lib.rs**: -- Added `make_leaf()` helper that creates `MetricLeafWithSchema` with schema from schemars -- Updated all `Traversable::to_tree_node()` implementations with `JsonSchema` bounds -- Schema generated via `schemars::SchemaGenerator::default().into_root_schema_for::()` +1. brk_server creates OpenAPI spec via aide +2. On startup, serializes spec to JSON string +3. Passes JSON to `brk_binder::generate_clients()` +4. brk_binder parses with `oas3` crate (supports OpenAPI 3.1) +5. Generates typed methods for all GET endpoints -**vecdb** (schemars feature): -- Added optional `schemars` dependency -- Added `AnySchemaVec` trait with blanket impl for `TypedVec where T: JsonSchema` +### Why oas3? -### Result - -The catalog tree now includes full type information and JSON schema at each leaf: - -```rust -TreeNode::Leaf(MetricLeafWithSchema { - leaf: MetricLeaf { - name: "difficulty".to_string(), - value_type: "StoredF64".to_string(), - indexes: btreeset![Index::Height, Index::Date], - }, - schema: json!({ "type": "number" }), // schemars-generated -}) -``` - -When trees are merged/simplified, indexes are unioned together. - -### 2. Async Runtime - -- TypeScript: Native `Promise` -- Python: `asyncio` or sync variant? -- Rust: `tokio` assumed, or feature-flag for other runtimes? - -### 3. Error Handling - -- HTTP errors (4xx, 5xx) -- Deserialization errors -- Network errors -- Should errors be typed per language? - -### 4. Additional Client Features - -- Request timeout configuration -- Retry logic -- Rate limiting -- Caching -- Batch requests (fetch multiple metrics at once) +aide generates OpenAPI 3.1 specs. The `openapiv3` crate only supports 3.0.x. +The `oas3` crate supports OpenAPI 3.1.x parsing. ## Tasks ### Phase 0: Type Infrastructure ✅ COMPLETE -- [x] **vecdb**: Add `short_type_name()` helper in `traits/printable.rs` -- [x] **vecdb**: Add `value_type_to_string()` to `AnyVec` trait -- [x] **vecdb**: Implement in all vec variants (eager, lazy, raw, compressed, macros) -- [x] **vecdb**: Add optional `schemars` feature with `AnySchemaVec` trait -- [x] **brk_types**: Enhance `TreeNode::Leaf` to include `MetricLeafWithSchema` -- [x] **brk_types**: Add `JsonSchema` derives to all value types -- [x] **brk_traversable**: Update all `to_tree_node()` implementations with schemars integration -- [x] **brk_query**: Export `Vecs` publicly for client generation -- [x] **brk_binder**: Set up generator module structure -- [x] **brk**: Verify compilation +- [x] vecdb: Add `short_type_name()` and `value_type_to_string()` +- [x] vecdb: Add optional `schemars` feature with `AnySchemaVec` trait +- [x] brk_types: Enhance `TreeNode::Leaf` to include `MetricLeafWithSchema` +- [x] brk_traversable: Update all `to_tree_node()` with schemars integration +- [x] brk_binder: Set up generator module structure ### Phase 1: JavaScript Client ✅ COMPLETE - [x] Define `MetricNode` class with JSDoc generics - [x] Define `BrkClient` with base HTTP functionality -- [x] Implement `ClientMetadata::from_vecs()` to extract metadata - [x] Generate `client.js` with full JSDoc type annotations -- [x] Use `schema_to_jsdoc()` to convert JSON schemas to JSDoc types - [x] Tree navigation: `client.tree.category.metric.fetch()` +- [x] API methods from OpenAPI endpoints -### Phase 2: OpenAPI Integration (NEXT) +### Phase 2: OpenAPI Integration ✅ COMPLETE -- [ ] Add `openapiv3` crate dependency -- [ ] Parse OpenAPI spec from aide (brk_server generates this) -- [ ] Extract non-metric endpoint definitions (health, info, catalog, etc.) -- [ ] Generate methods for each endpoint with proper types -- [ ] Merge with tree-based metric access +- [x] Add `oas3` crate dependency (OpenAPI 3.1 support) +- [x] brk_server passes OpenAPI JSON to brk_binder on startup +- [x] Parse OpenAPI spec and extract endpoint definitions +- [x] Generate typed methods for each GET endpoint -### Phase 3: Python Client +### Phase 3: Python Client ✅ COMPLETE -- [ ] Define `MetricNode` class with type hints -- [ ] Define `BrkClient` with httpx/aiohttp -- [ ] Generate typed methods from OpenAPI -- [ ] Generate tree navigation +- [x] Define `MetricNode` class with type hints +- [x] Define `BrkClient` with httpx +- [x] Generate typed methods from OpenAPI +- [x] Generate tree navigation -### Phase 4: Rust Client +### Phase 4: Rust Client ✅ COMPLETE -- [ ] Define `MetricNode` struct using `brk_types` -- [ ] Define `BrkClient` with reqwest -- [ ] Import types from `brk_types` instead of generating -- [ ] Generate tree navigation with proper lifetimes +- [x] Define `MetricNode` struct with lifetimes +- [x] Define `BrkClient` with reqwest (blocking) +- [x] Generate tree navigation with proper lifetimes +- [x] Generate typed methods from OpenAPI ### Phase 5: Polish +- [x] Switch from `openapiv3` to `oas3` crate - [ ] Error types per language - [ ] Documentation generation - [ ] Tests - [ ] Example usage in each language +- [ ] Async Rust client variant ## File Structure @@ -362,17 +249,27 @@ When trees are merged/simplified, indexes are unioned together. crates/brk_binder/ ├── src/ │ ├── lib.rs -│ ├── js.rs # JS constants generation (existing) -│ ├── tree.rs # Pattern extraction (reference only, not compiled) +│ ├── js.rs # JS constants generation (existing) │ └── generator/ -│ ├── mod.rs -│ ├── types.rs # ClientMetadata, MetricInfo, IndexPattern, schema_to_jsdoc -│ ├── javascript.rs # JavaScript + JSDoc client generation ✅ -│ ├── python.rs # Python client generation (stub) -│ └── rust.rs # Rust client generation (stub) +│ ├── mod.rs # generate_clients() entry point +│ ├── types.rs # ClientMetadata, MetricInfo, IndexPattern +│ ├── openapi.rs # OpenAPI 3.1 spec parsing (oas3) +│ ├── javascript.rs # JavaScript + JSDoc client ✅ +│ ├── python.rs # Python client ✅ +│ └── rust.rs # Rust client ✅ +├── clients/ # Generated output (gitignored) +│ ├── javascript/ +│ ├── python/ +│ └── rust/ ├── Cargo.toml ├── README.md -└── DESIGN.md # This file +└── DESIGN.md + +crates/brk_server/ +└── src/ + ├── lib.rs # Calls brk_binder::generate_clients() on startup + └── api/ + └── openapi.rs # create_openapi() for aide ``` ## Dependencies @@ -381,11 +278,19 @@ crates/brk_binder/ [dependencies] brk_query = { workspace = true } brk_types = { workspace = true } +oas3 = "0.20" # OpenAPI 3.1 spec parsing schemars = { workspace = true } serde_json = { workspace = true } -vecdb = { workspace = true } - -# For OpenAPI integration (Phase 2): -# openapiv3 = "2" # OpenAPI parsing -# serde_yaml = "0.9" # If parsing YAML specs +``` + +## Usage + +To generate clients: + +```bash +# Create the output directory +mkdir -p crates/brk_binder/clients + +# Run the server (generates clients on startup) +cargo run -p brk_server ``` diff --git a/crates/brk_binder/src/generator/javascript.rs b/crates/brk_binder/src/generator/javascript.rs index 023adc73e..ed9d372d8 100644 --- a/crates/brk_binder/src/generator/javascript.rs +++ b/crates/brk_binder/src/generator/javascript.rs @@ -1,69 +1,105 @@ +use std::collections::HashSet; use std::fmt::Write as FmtWrite; use std::fs; use std::io; use std::path::Path; -use brk_types::TreeNode; +use brk_types::{Index, TreeNode}; -use super::{schema_to_jsdoc, to_camel_case, ClientMetadata, IndexPattern}; +use super::{ClientMetadata, Endpoint, IndexSetPattern, PatternField, StructuralPattern, get_node_fields, to_camel_case, to_pascal_case, to_snake_case}; -/// Generate JavaScript + JSDoc client from metadata -pub fn generate_javascript_client(metadata: &ClientMetadata, output_dir: &Path) -> io::Result<()> { +/// Generate JavaScript + JSDoc client from metadata and OpenAPI endpoints +pub fn generate_javascript_client( + metadata: &ClientMetadata, + endpoints: &[Endpoint], + output_dir: &Path, +) -> io::Result<()> { let mut output = String::new(); // Header writeln!(output, "// Auto-generated BRK JavaScript client").unwrap(); writeln!(output, "// Do not edit manually\n").unwrap(); - // Generate pattern JSDoc typedefs for index groupings - generate_pattern_typedefs(&mut output, &metadata.patterns); - // Generate the base client class generate_base_client(&mut output); - // Generate tree JSDoc typedefs from catalog - generate_tree_typedefs(&mut output, &metadata.catalog); + // Generate index accessor factory functions + generate_index_accessors(&mut output, &metadata.index_set_patterns); - // Generate the main client class with tree - generate_main_client(&mut output, &metadata.catalog); + // Generate structural pattern factory functions + generate_structural_patterns(&mut output, &metadata.structural_patterns, metadata); + + // Generate tree JSDoc typedefs + generate_tree_typedefs(&mut output, &metadata.catalog, metadata); + + // Generate the main client class with tree and API methods + generate_main_client(&mut output, &metadata.catalog, metadata, endpoints); fs::write(output_dir.join("client.js"), output)?; Ok(()) } -/// Generate JSDoc typedefs for common index patterns -fn generate_pattern_typedefs(output: &mut String, patterns: &[IndexPattern]) { - writeln!(output, "// Index pattern typedefs").unwrap(); - writeln!(output, "// Reusable patterns for metrics with the same index groupings\n").unwrap(); - - for pattern in patterns { - let pattern_name = pattern_to_name(pattern); - writeln!(output, "/**").unwrap(); - writeln!(output, " * @template T").unwrap(); - writeln!(output, " * @typedef {{Object}} {}", pattern_name).unwrap(); - - for index in &pattern.indexes { - let field_name = to_camel_case(&index.serialize_long()); - writeln!(output, " * @property {{MetricNode}} {}", field_name).unwrap(); - } - - writeln!(output, " */\n").unwrap(); - } -} - /// Generate the base BrkClient class with HTTP functionality fn generate_base_client(output: &mut String) { writeln!( output, r#"/** * @typedef {{Object}} BrkClientOptions - * @property {{string}} baseUrl - The base URL for the API - * @property {{number}} [timeout] - Request timeout in ms (default: 30000) + * @property {{string}} baseUrl - Base URL for the API + * @property {{number}} [timeout] - Request timeout in milliseconds */ /** - * Base HTTP client + * Custom error class for BRK client errors + */ +class BrkError extends Error {{ + /** + * @param {{string}} message + * @param {{number}} [status] + */ + constructor(message, status) {{ + super(message); + this.name = 'BrkError'; + this.status = status; + }} +}} + +/** + * A metric node that can fetch data for different indexes. + * @template T + */ +class MetricNode {{ + /** + * @param {{BrkClientBase}} client + * @param {{string}} path + */ + constructor(client, path) {{ + this.client = client; + this.path = path; + }} + + /** + * Fetch all data points for this metric. + * @returns {{Promise}} + */ + async get() {{ + return this.client.get(this.path); + }} + + /** + * Fetch data points within a date range. + * @param {{string}} from + * @param {{string}} to + * @returns {{Promise}} + */ + async getRange(from, to) {{ + return this.client.get(`${{this.path}}?from=${{from}}&to=${{to}}`); + }} +}} + +/** + * Base HTTP client for making requests */ class BrkClientBase {{ /** @@ -71,15 +107,16 @@ class BrkClientBase {{ */ constructor(options) {{ if (typeof options === 'string') {{ - this.baseUrl = options.replace(/\/$/, ''); + this.baseUrl = options; this.timeout = 30000; }} else {{ - this.baseUrl = options.baseUrl.replace(/\/$/, ''); - this.timeout = options.timeout ?? 30000; + this.baseUrl = options.baseUrl; + this.timeout = options.timeout || 30000; }} }} /** + * Make a GET request * @template T * @param {{string}} path * @returns {{Promise}} @@ -91,127 +128,241 @@ class BrkClientBase {{ try {{ const response = await fetch(`${{this.baseUrl}}${{path}}`, {{ signal: controller.signal, - headers: {{ 'Accept': 'application/json' }}, }}); if (!response.ok) {{ - throw new BrkError(`HTTP ${{response.status}}: ${{response.statusText}}`, response.status); + throw new BrkError(`HTTP error: ${{response.status}}`, response.status); }} - return await response.json(); + return response.json(); + }} catch (error) {{ + if (error.name === 'AbortError') {{ + throw new BrkError('Request timeout'); + }} + throw error; }} finally {{ clearTimeout(timeoutId); }} }} }} -/** - * Error class for BRK API errors - */ -class BrkError extends Error {{ - /** - * @param {{string}} message - * @param {{number}} [statusCode] - */ - constructor(message, statusCode) {{ - super(message); - this.name = 'BrkError'; - this.statusCode = statusCode; - }} -}} - -/** - * Metric node with fetch capability - * @template T - */ -class MetricNode {{ - /** - * @param {{BrkClientBase}} client - * @param {{string}} path - */ - constructor(client, path) {{ - this._client = client; - this._path = path; - }} - - /** - * Fetch the metric value - * @returns {{Promise}} - */ - async fetch() {{ - return this._client.get(this._path); - }} - - toString() {{ - return this._path; - }} -}} - "# ) .unwrap(); } -/// Generate JSDoc typedefs for the catalog tree -fn generate_tree_typedefs(output: &mut String, catalog: &TreeNode) { - writeln!(output, "// Catalog tree typedefs\n").unwrap(); - generate_node_typedef(output, "CatalogTree", catalog, ""); +/// Generate index accessor factory functions +fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) { + if patterns.is_empty() { + return; + } + + writeln!(output, "// Index accessor factory functions\n").unwrap(); + + for pattern in patterns { + // Generate JSDoc typedef for the accessor + writeln!(output, "/**").unwrap(); + writeln!(output, " * @template T").unwrap(); + writeln!(output, " * @typedef {{Object}} {}", pattern.name).unwrap(); + for index in &pattern.indexes { + let field_name = index_to_camel_case(index); + writeln!(output, " * @property {{MetricNode}} {}", field_name).unwrap(); + } + writeln!(output, " */\n").unwrap(); + + // Generate factory function + writeln!(output, "/**").unwrap(); + writeln!(output, " * Create a {} accessor", pattern.name).unwrap(); + writeln!(output, " * @template T").unwrap(); + writeln!(output, " * @param {{BrkClientBase}} client").unwrap(); + writeln!(output, " * @param {{string}} basePath").unwrap(); + writeln!(output, " * @returns {{{}}}", pattern.name).unwrap(); + writeln!(output, " */").unwrap(); + writeln!(output, "function create{}(client, basePath) {{", pattern.name).unwrap(); + writeln!(output, " return {{").unwrap(); + + for (i, index) in pattern.indexes.iter().enumerate() { + let field_name = index_to_camel_case(index); + let path_segment = index.serialize_short(); + let comma = if i < pattern.indexes.len() - 1 { "," } else { "" }; + writeln!( + output, + " {}: new MetricNode(client, `${{basePath}}/{}`){}", + field_name, path_segment, comma + ).unwrap(); + } + + writeln!(output, " }};").unwrap(); + writeln!(output, "}}\n").unwrap(); + } } -/// Recursively generate typedef for a tree node -fn generate_node_typedef(output: &mut String, name: &str, node: &TreeNode, path: &str) { - match node { - TreeNode::Leaf(_leaf) => { - // Leaf nodes are MetricNode - // No separate typedef needed, handled inline +/// Convert an Index to a camelCase field name (e.g., DateIndex -> byDate) +fn index_to_camel_case(index: &Index) -> String { + let short = index.serialize_short(); + format!("by{}", to_pascal_case(&to_snake_case(short))) +} + +/// Generate structural pattern factory functions +fn generate_structural_patterns(output: &mut String, patterns: &[StructuralPattern], metadata: &ClientMetadata) { + if patterns.is_empty() { + return; + } + + writeln!(output, "// Reusable structural pattern factories\n").unwrap(); + + for pattern in patterns { + // Generate JSDoc typedef + writeln!(output, "/**").unwrap(); + writeln!(output, " * @typedef {{Object}} {}", pattern.name).unwrap(); + for field in &pattern.fields { + let js_type = field_to_js_type(field, metadata); + writeln!(output, " * @property {{{}}} {}", js_type, to_camel_case(&field.name)).unwrap(); } - TreeNode::Branch(children) => { - writeln!(output, "/**").unwrap(); - writeln!(output, " * @typedef {{Object}} {}", name).unwrap(); + writeln!(output, " */\n").unwrap(); - for (child_name, child_node) in children { - let field_name = to_camel_case(child_name); + // Generate factory function + writeln!(output, "/**").unwrap(); + writeln!(output, " * Create a {} pattern node", pattern.name).unwrap(); + writeln!(output, " * @param {{BrkClientBase}} client").unwrap(); + writeln!(output, " * @param {{string}} basePath").unwrap(); + writeln!(output, " * @returns {{{}}}", pattern.name).unwrap(); + writeln!(output, " */").unwrap(); + writeln!(output, "function create{}(client, basePath) {{", pattern.name).unwrap(); + writeln!(output, " return {{").unwrap(); - match child_node { - TreeNode::Leaf(leaf) => { - let js_type = schema_to_jsdoc(&leaf.schema); - writeln!( - output, - " * @property {{MetricNode<{}>}} {}", - js_type, field_name - ) - .unwrap(); - } - TreeNode::Branch(_) => { - let child_type_name = format!("{}_{}", name, to_pascal_case(child_name)); - writeln!(output, " * @property {{{}}} {}", child_type_name, field_name) - .unwrap(); - } - } + for (i, field) in pattern.fields.iter().enumerate() { + let comma = if i < pattern.fields.len() - 1 { "," } else { "" }; + if metadata.is_pattern_type(&field.rust_type) { + writeln!( + output, + " {}: create{}(client, `${{basePath}}/{}`){}", + to_camel_case(&field.name), field.rust_type, field.name, comma + ).unwrap(); + } else if field_uses_accessor(field, metadata) { + let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap(); + writeln!( + output, + " {}: create{}(client, `${{basePath}}/{}`){}", + to_camel_case(&field.name), accessor.name, field.name, comma + ).unwrap(); + } else { + writeln!( + output, + " {}: new MetricNode(client, `${{basePath}}/{}`){}", + to_camel_case(&field.name), field.name, comma + ).unwrap(); } + } - writeln!(output, " */\n").unwrap(); + writeln!(output, " }};").unwrap(); + writeln!(output, "}}\n").unwrap(); + } +} - // Generate child typedefs - for (child_name, child_node) in children { - if let TreeNode::Branch(_) = child_node { +/// Convert pattern field to JavaScript/JSDoc type +fn field_to_js_type(field: &PatternField, metadata: &ClientMetadata) -> String { + if metadata.is_pattern_type(&field.rust_type) { + // Pattern type - use pattern name directly + field.rust_type.clone() + } else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) { + // Leaf with a reusable accessor pattern + let js_type = json_type_to_js(&field.json_type); + format!("{}<{}>", accessor.name, js_type) + } else { + // Leaf with unique index set - use MetricNode directly + let js_type = json_type_to_js(&field.json_type); + format!("MetricNode<{}>", js_type) + } +} + +/// Check if a field should use an index accessor +fn field_uses_accessor(field: &PatternField, metadata: &ClientMetadata) -> bool { + metadata.find_index_set_pattern(&field.indexes).is_some() +} + +/// Convert JSON Schema type to JSDoc type +fn json_type_to_js(json_type: &str) -> &str { + match json_type { + "integer" | "number" => "number", + "boolean" => "boolean", + "string" => "string", + "array" => "Array", + "object" => "Object", + _ => "*", + } +} + +/// Generate tree typedefs +fn generate_tree_typedefs( + output: &mut String, + catalog: &TreeNode, + metadata: &ClientMetadata, +) { + writeln!(output, "// Catalog tree typedefs\n").unwrap(); + + let pattern_lookup = metadata.pattern_lookup(); + let mut generated = HashSet::new(); + generate_tree_typedef(output, "CatalogTree", catalog, &pattern_lookup, metadata, &mut generated); +} + +/// Recursively generate tree typedefs +fn generate_tree_typedef( + output: &mut String, + name: &str, + node: &TreeNode, + pattern_lookup: &std::collections::HashMap, String>, + metadata: &ClientMetadata, + generated: &mut HashSet, +) { + if let TreeNode::Branch(children) = node { + // Build signature + let fields = get_node_fields(children, pattern_lookup); + + // Skip if this matches a pattern (already generated) + if pattern_lookup.contains_key(&fields) && pattern_lookup.get(&fields) != Some(&name.to_string()) { + return; + } + + if generated.contains(name) { + return; + } + generated.insert(name.to_string()); + + writeln!(output, "/**").unwrap(); + writeln!(output, " * @typedef {{Object}} {}", name).unwrap(); + + for field in &fields { + let js_type = field_to_js_type(field, metadata); + writeln!(output, " * @property {{{}}} {}", js_type, to_camel_case(&field.name)).unwrap(); + } + + writeln!(output, " */\n").unwrap(); + + // Generate child typedefs + for (child_name, child_node) in children { + if let TreeNode::Branch(grandchildren) = child_node { + let child_fields = get_node_fields(grandchildren, pattern_lookup); + if !pattern_lookup.contains_key(&child_fields) { let child_type_name = format!("{}_{}", name, to_pascal_case(child_name)); - let child_path = if path.is_empty() { - format!("/{}", child_name) - } else { - format!("{}/{}", path, child_name) - }; - generate_node_typedef(output, &child_type_name, child_node, &child_path); + generate_tree_typedef(output, &child_type_name, child_node, pattern_lookup, metadata, generated); } } } } } -/// Generate the main client class with initialized tree -fn generate_main_client(output: &mut String, catalog: &TreeNode) { +/// Generate main client +fn generate_main_client( + output: &mut String, + catalog: &TreeNode, + metadata: &ClientMetadata, + endpoints: &[Endpoint], +) { + let pattern_lookup = metadata.pattern_lookup(); + writeln!(output, "/**").unwrap(); - writeln!(output, " * Main BRK client with catalog tree").unwrap(); + writeln!(output, " * Main BRK client with catalog tree and API methods").unwrap(); writeln!(output, " * @extends BrkClientBase").unwrap(); writeln!(output, " */").unwrap(); writeln!(output, "class BrkClient extends BrkClientBase {{").unwrap(); @@ -221,27 +372,39 @@ fn generate_main_client(output: &mut String, catalog: &TreeNode) { writeln!(output, " constructor(options) {{").unwrap(); writeln!(output, " super(options);").unwrap(); writeln!(output, " /** @type {{CatalogTree}} */").unwrap(); - writeln!(output, " this.tree = this._buildTree();").unwrap(); + writeln!(output, " this.tree = this._buildTree('');").unwrap(); writeln!(output, " }}\n").unwrap(); // Generate _buildTree method writeln!(output, " /**").unwrap(); writeln!(output, " * @private").unwrap(); + writeln!(output, " * @param {{string}} basePath").unwrap(); writeln!(output, " * @returns {{CatalogTree}}").unwrap(); writeln!(output, " */").unwrap(); - writeln!(output, " _buildTree() {{").unwrap(); + writeln!(output, " _buildTree(basePath) {{").unwrap(); writeln!(output, " return {{").unwrap(); - generate_tree_initializer(output, catalog, "", 3); + generate_tree_initializer(output, catalog, "", 3, &pattern_lookup, metadata); writeln!(output, " }};").unwrap(); - writeln!(output, " }}").unwrap(); + writeln!(output, " }}\n").unwrap(); + + // Generate API methods + generate_api_methods(output, endpoints); + writeln!(output, "}}\n").unwrap(); - // Export for ES modules + // Export writeln!(output, "export {{ BrkClient, BrkClientBase, BrkError, MetricNode }};").unwrap(); } -/// Generate the tree initializer code -fn generate_tree_initializer(output: &mut String, node: &TreeNode, path: &str, indent: usize) { +/// Generate tree initializer +fn generate_tree_initializer( + output: &mut String, + node: &TreeNode, + path: &str, + indent: usize, + pattern_lookup: &std::collections::HashMap, String>, + metadata: &ClientMetadata, +) { let indent_str = " ".repeat(indent); if let TreeNode::Branch(children) = node { @@ -256,43 +419,118 @@ fn generate_tree_initializer(output: &mut String, node: &TreeNode, path: &str, i let comma = if i < children.len() - 1 { "," } else { "" }; match child_node { - TreeNode::Leaf(_) => { - writeln!( - output, - "{}{}: new MetricNode(this, '{}'){}", - indent_str, field_name, child_path, comma - ) - .unwrap(); + TreeNode::Leaf(leaf) => { + if let Some(accessor) = metadata.find_index_set_pattern(leaf.indexes()) { + writeln!( + output, + "{}{}: create{}(this, '{}'){}", + indent_str, field_name, accessor.name, child_path, comma + ).unwrap(); + } else { + writeln!( + output, + "{}{}: new MetricNode(this, '{}'){}", + indent_str, field_name, child_path, comma + ).unwrap(); + } } - TreeNode::Branch(_) => { - writeln!(output, "{}{}: {{", indent_str, field_name).unwrap(); - generate_tree_initializer(output, child_node, &child_path, indent + 1); - writeln!(output, "{}}}{}", indent_str, comma).unwrap(); + TreeNode::Branch(grandchildren) => { + let child_fields = get_node_fields(grandchildren, pattern_lookup); + if let Some(pattern_name) = pattern_lookup.get(&child_fields) { + writeln!( + output, + "{}{}: create{}(this, '{}'){}", + indent_str, field_name, pattern_name, child_path, comma + ).unwrap(); + } else { + writeln!(output, "{}{}: {{", indent_str, field_name).unwrap(); + generate_tree_initializer(output, child_node, &child_path, indent + 1, pattern_lookup, metadata); + writeln!(output, "{}}}{}", indent_str, comma).unwrap(); + } } } } } } -/// Convert pattern to a JSDoc typedef name -fn pattern_to_name(pattern: &IndexPattern) -> String { - let index_names: Vec = pattern - .indexes - .iter() - .map(|i| to_pascal_case(&i.serialize_long())) - .collect(); - format!("Pattern_{}", index_names.join("_")) +/// Generate API methods +fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) { + for endpoint in endpoints { + if endpoint.method != "GET" { + continue; + } + + let method_name = endpoint_to_method_name(endpoint); + let return_type = endpoint.response_type.as_deref().unwrap_or("*"); + + writeln!(output, " /**").unwrap(); + if let Some(summary) = &endpoint.summary { + writeln!(output, " * {}", summary).unwrap(); + } + + for param in &endpoint.path_params { + let desc = param.description.as_deref().unwrap_or(""); + writeln!(output, " * @param {{{}}} {} {}", param.param_type, param.name, desc).unwrap(); + } + for param in &endpoint.query_params { + let optional = if param.required { "" } else { "=" }; + let desc = param.description.as_deref().unwrap_or(""); + writeln!(output, " * @param {{{}{}}} [{}] {}", param.param_type, optional, param.name, desc).unwrap(); + } + + writeln!(output, " * @returns {{Promise<{}>}}", return_type).unwrap(); + writeln!(output, " */").unwrap(); + + let params = build_method_params(endpoint); + writeln!(output, " async {}({}) {{", method_name, params).unwrap(); + + let path = build_path_template(&endpoint.path, &endpoint.path_params); + + if endpoint.query_params.is_empty() { + writeln!(output, " return this.get(`{}`);", path).unwrap(); + } else { + writeln!(output, " const params = new URLSearchParams();").unwrap(); + for param in &endpoint.query_params { + if param.required { + writeln!(output, " params.set('{}', String({}));", param.name, param.name).unwrap(); + } else { + writeln!(output, " if ({} !== undefined) params.set('{}', String({}));", param.name, param.name, param.name).unwrap(); + } + } + writeln!(output, " const query = params.toString();").unwrap(); + writeln!(output, " return this.get(`{}${{query ? '?' + query : ''}}`);", path).unwrap(); + } + + writeln!(output, " }}\n").unwrap(); + } } -/// Convert string to PascalCase -fn to_pascal_case(s: &str) -> String { - s.split('_') - .map(|word| { - let mut chars = word.chars(); - match chars.next() { - None => String::new(), - Some(first) => first.to_uppercase().collect::() + chars.as_str(), - } - }) - .collect() +fn endpoint_to_method_name(endpoint: &Endpoint) -> String { + if let Some(op_id) = &endpoint.operation_id { + return to_camel_case(op_id); + } + let parts: Vec<&str> = endpoint.path.split('/').filter(|s| !s.is_empty() && !s.starts_with('{')).collect(); + format!("get{}", to_pascal_case(&parts.join("_"))) } + +fn build_method_params(endpoint: &Endpoint) -> String { + let mut params = Vec::new(); + for param in &endpoint.path_params { + params.push(param.name.clone()); + } + for param in &endpoint.query_params { + params.push(param.name.clone()); + } + params.join(", ") +} + +fn build_path_template(path: &str, path_params: &[super::Parameter]) -> String { + let mut result = path.to_string(); + for param in path_params { + let placeholder = format!("{{{}}}", param.name); + let interpolation = format!("${{{}}}", param.name); + result = result.replace(&placeholder, &interpolation); + } + result +} + diff --git a/crates/brk_binder/src/generator/mod.rs b/crates/brk_binder/src/generator/mod.rs index 60a23f7a1..9e1dfd778 100644 --- a/crates/brk_binder/src/generator/mod.rs +++ b/crates/brk_binder/src/generator/mod.rs @@ -1,35 +1,41 @@ mod javascript; +mod openapi; mod python; mod rust; mod types; -pub use javascript::generate_javascript_client; -pub use python::generate_python_client; -pub use rust::generate_rust_client; +pub use javascript::*; +pub use openapi::*; +pub use python::*; +pub use rust::*; pub use types::*; use brk_query::Vecs; use std::io; use std::path::Path; -/// Generate all client libraries from the query vecs -pub fn generate_clients(vecs: &Vecs, output_dir: &Path) -> io::Result<()> { +/// Generate all client libraries from the query vecs and OpenAPI JSON +pub fn generate_clients(vecs: &Vecs, openapi_json: &str, output_dir: &Path) -> io::Result<()> { let metadata = ClientMetadata::from_vecs(vecs); + // Parse OpenAPI spec from JSON + let spec = parse_openapi_json(openapi_json)?; + let endpoints = extract_endpoints(&spec); + // Generate Rust client let rust_path = output_dir.join("rust"); std::fs::create_dir_all(&rust_path)?; - generate_rust_client(&metadata, &rust_path)?; + generate_rust_client(&metadata, &endpoints, &rust_path)?; // Generate JavaScript client let js_path = output_dir.join("javascript"); std::fs::create_dir_all(&js_path)?; - generate_javascript_client(&metadata, &js_path)?; + generate_javascript_client(&metadata, &endpoints, &js_path)?; // Generate Python client let python_path = output_dir.join("python"); std::fs::create_dir_all(&python_path)?; - generate_python_client(&metadata, &python_path)?; + generate_python_client(&metadata, &endpoints, &python_path)?; Ok(()) } diff --git a/crates/brk_binder/src/generator/openapi.rs b/crates/brk_binder/src/generator/openapi.rs new file mode 100644 index 000000000..f750a5e3c --- /dev/null +++ b/crates/brk_binder/src/generator/openapi.rs @@ -0,0 +1,214 @@ +use std::io; + +use oas3::spec::{ObjectOrReference, Operation, ParameterIn, PathItem, Schema, SchemaTypeSet}; +use oas3::Spec; +use serde_json::Value; + +/// Endpoint information extracted from OpenAPI spec +#[derive(Debug, Clone)] +pub struct Endpoint { + /// HTTP method (GET, POST, etc.) + pub method: String, + /// Path template (e.g., "/blocks/{hash}") + pub path: String, + /// Operation ID (e.g., "getBlockByHash") + pub operation_id: Option, + /// Summary/description + pub summary: Option, + /// Tags for grouping + pub tags: Vec, + /// Path parameters + pub path_params: Vec, + /// Query parameters + pub query_params: Vec, + /// Response type (simplified) + pub response_type: Option, +} + +/// Parameter information +#[derive(Debug, Clone)] +pub struct Parameter { + pub name: String, + pub required: bool, + pub param_type: String, + pub description: Option, +} + +/// Parse OpenAPI spec from JSON string +/// +/// Pre-processes the JSON to handle oas3 limitations: +/// - Removes unsupported siblings from `$ref` objects (oas3 only supports `summary` and `description`) +pub fn parse_openapi_json(json: &str) -> io::Result { + let mut value: Value = + serde_json::from_str(json).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + // Clean up for oas3 compatibility + clean_for_oas3(&mut value); + + let cleaned_json = serde_json::to_string(&value) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + oas3::from_json(&cleaned_json).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) +} + +/// Clean up OpenAPI spec for oas3 compatibility. +/// - Removes unsupported siblings from $ref objects (oas3 only supports summary and description) +/// - Converts boolean schemas to object schemas (oas3 doesn't handle `"schema": true`) +fn clean_for_oas3(value: &mut Value) { + match value { + Value::Object(map) => { + // Handle $ref with unsupported siblings + if map.contains_key("$ref") { + map.retain(|k, _| k == "$ref" || k == "summary" || k == "description"); + } else { + // Convert boolean schemas to empty object schemas + if let Some(schema) = map.get_mut("schema") { + if schema.is_boolean() { + *schema = Value::Object(serde_json::Map::new()); + } + } + for v in map.values_mut() { + clean_for_oas3(v); + } + } + } + Value::Array(arr) => { + for v in arr { + clean_for_oas3(v); + } + } + _ => {} + } +} + +/// Extract all endpoints from OpenAPI spec +pub fn extract_endpoints(spec: &Spec) -> Vec { + let mut endpoints = Vec::new(); + + let Some(paths) = &spec.paths else { + return endpoints; + }; + + for (path, path_item) in paths { + for (method, operation) in get_operations(path_item) { + if let Some(endpoint) = extract_endpoint(path, &method, operation) { + endpoints.push(endpoint); + } + } + } + + endpoints +} + +fn get_operations(path_item: &PathItem) -> Vec<(String, &Operation)> { + let mut ops = Vec::new(); + if let Some(op) = &path_item.get { + ops.push(("GET".to_string(), op)); + } + if let Some(op) = &path_item.post { + ops.push(("POST".to_string(), op)); + } + if let Some(op) = &path_item.put { + ops.push(("PUT".to_string(), op)); + } + if let Some(op) = &path_item.delete { + ops.push(("DELETE".to_string(), op)); + } + if let Some(op) = &path_item.patch { + ops.push(("PATCH".to_string(), op)); + } + ops +} + +fn extract_endpoint(path: &str, method: &str, operation: &Operation) -> Option { + let path_params = extract_parameters(operation, ParameterIn::Path); + let query_params = extract_parameters(operation, ParameterIn::Query); + + let response_type = extract_response_type(operation); + + Some(Endpoint { + method: method.to_string(), + path: path.to_string(), + operation_id: operation.operation_id.clone(), + summary: operation.summary.clone().or_else(|| operation.description.clone()), + tags: operation.tags.clone(), + path_params, + query_params, + response_type, + }) +} + +fn extract_parameters(operation: &Operation, location: ParameterIn) -> Vec { + operation + .parameters + .iter() + .filter_map(|p| match p { + ObjectOrReference::Object(param) if param.location == location => Some(Parameter { + name: param.name.clone(), + required: param.required.unwrap_or(false), + param_type: "string".to_string(), // Simplified + description: param.description.clone(), + }), + _ => None, + }) + .collect() +} + +fn extract_response_type(operation: &Operation) -> Option { + let responses = operation.responses.as_ref()?; + + // Look for 200 OK response + let response = responses.get("200")?; + + match response { + ObjectOrReference::Object(response) => { + // Look for JSON content + let content = response.content.get("application/json")?; + + match &content.schema { + Some(ObjectOrReference::Ref { ref_path, .. }) => { + // Extract type name from reference like "#/components/schemas/Block" + Some(ref_path.rsplit('/').next()?.to_string()) + } + Some(ObjectOrReference::Object(schema)) => schema_to_type_name(schema), + None => None, + } + } + ObjectOrReference::Ref { .. } => None, + } +} + +fn schema_type_from_schema(schema: &Schema) -> Option { + match schema { + Schema::Boolean(_) => Some("boolean".to_string()), + Schema::Object(obj_or_ref) => match obj_or_ref.as_ref() { + ObjectOrReference::Object(obj_schema) => schema_to_type_name(obj_schema), + ObjectOrReference::Ref { ref_path, .. } => { + ref_path.rsplit('/').next().map(|s| s.to_string()) + } + }, + } +} + +fn schema_to_type_name(schema: &oas3::spec::ObjectSchema) -> Option { + let schema_type = schema.schema_type.as_ref()?; + + match schema_type { + SchemaTypeSet::Single(t) => match t { + oas3::spec::SchemaType::String => Some("string".to_string()), + oas3::spec::SchemaType::Number => Some("number".to_string()), + oas3::spec::SchemaType::Integer => Some("number".to_string()), + oas3::spec::SchemaType::Boolean => Some("boolean".to_string()), + oas3::spec::SchemaType::Array => { + let inner = match &schema.items { + Some(boxed_schema) => schema_type_from_schema(boxed_schema), + None => Some("*".to_string()), + }; + inner.map(|t| format!("{}[]", t)) + } + oas3::spec::SchemaType::Object => Some("Object".to_string()), + oas3::spec::SchemaType::Null => Some("null".to_string()), + }, + SchemaTypeSet::Multiple(_) => Some("*".to_string()), + } +} diff --git a/crates/brk_binder/src/generator/python.rs b/crates/brk_binder/src/generator/python.rs index 4f72ef05c..115e0bfe0 100644 --- a/crates/brk_binder/src/generator/python.rs +++ b/crates/brk_binder/src/generator/python.rs @@ -1,10 +1,428 @@ +use std::collections::HashSet; +use std::fmt::Write as FmtWrite; +use std::fs; use std::io; use std::path::Path; -use super::ClientMetadata; +use brk_types::{Index, TreeNode}; + +use super::{ClientMetadata, Endpoint, IndexSetPattern, PatternField, StructuralPattern, get_node_fields, to_pascal_case, to_snake_case}; + +/// Generate Python client from metadata and OpenAPI endpoints +pub fn generate_python_client( + metadata: &ClientMetadata, + endpoints: &[Endpoint], + output_dir: &Path, +) -> io::Result<()> { + let mut output = String::new(); + + // Header + writeln!(output, "# Auto-generated BRK Python client").unwrap(); + writeln!(output, "# Do not edit manually\n").unwrap(); + writeln!(output, "from __future__ import annotations").unwrap(); + writeln!(output, "from typing import TypeVar, Generic, Any, Optional, List").unwrap(); + writeln!(output, "from dataclasses import dataclass").unwrap(); + writeln!(output, "import httpx\n").unwrap(); + + // Type variable for generic MetricNode + writeln!(output, "T = TypeVar('T')\n").unwrap(); + + // Generate base client class + generate_base_client(&mut output); + + // Generate MetricNode class + generate_metric_node(&mut output); + + // Generate index accessor classes + generate_index_accessors(&mut output, &metadata.index_set_patterns); + + // Generate structural pattern classes + generate_structural_patterns(&mut output, &metadata.structural_patterns, metadata); + + // Generate tree classes + generate_tree_classes(&mut output, &metadata.catalog, metadata); + + // Generate main client with tree and API methods + generate_main_client(&mut output, endpoints); + + fs::write(output_dir.join("client.py"), output)?; -/// Generate Python client from metadata -pub fn generate_python_client(_metadata: &ClientMetadata, _output_dir: &Path) -> io::Result<()> { - // TODO: Implement Python client generation Ok(()) } + +/// Generate the base BrkClient class with HTTP functionality +fn generate_base_client(output: &mut String) { + writeln!( + output, + r#"class BrkError(Exception): + """Custom error class for BRK client errors.""" + + def __init__(self, message: str, status: Optional[int] = None): + super().__init__(message) + self.status = status + + +class BrkClientBase: + """Base HTTP client for making requests.""" + + def __init__(self, base_url: str, timeout: float = 30.0): + self.base_url = base_url + self.timeout = timeout + self._client = httpx.Client(timeout=timeout) + + def get(self, path: str) -> Any: + """Make a GET request.""" + try: + response = self._client.get(f"{{self.base_url}}{{path}}") + response.raise_for_status() + return response.json() + except httpx.HTTPStatusError as e: + raise BrkError(f"HTTP error: {{e.response.status_code}}", e.response.status_code) + except httpx.RequestError as e: + raise BrkError(str(e)) + + def close(self): + """Close the HTTP client.""" + self._client.close() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + +"# + ) + .unwrap(); +} + +/// Generate the MetricNode class +fn generate_metric_node(output: &mut String) { + writeln!( + output, + r#"class MetricNode(Generic[T]): + """A metric node that can fetch data for different indexes.""" + + def __init__(self, client: BrkClientBase, path: str): + self._client = client + self._path = path + + def get(self) -> List[T]: + """Fetch all data points for this metric.""" + return self._client.get(self._path) + + def get_range(self, from_date: str, to_date: str) -> List[T]: + """Fetch data points within a date range.""" + return self._client.get(f"{{self._path}}?from={{from_date}}&to={{to_date}}") + +"# + ) + .unwrap(); +} + +/// Generate index accessor classes +fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) { + if patterns.is_empty() { + return; + } + + writeln!(output, "# Index accessor classes\n").unwrap(); + + for pattern in patterns { + writeln!(output, "class {}(Generic[T]):", pattern.name).unwrap(); + writeln!(output, " \"\"\"Index accessor for metrics with {} indexes.\"\"\"", pattern.indexes.len()).unwrap(); + writeln!(output, " ").unwrap(); + writeln!(output, " def __init__(self, client: BrkClientBase, base_path: str):").unwrap(); + + for index in &pattern.indexes { + let field_name = index_to_snake_case(index); + let path_segment = index.serialize_short(); + writeln!( + output, + " self.{}: MetricNode[T] = MetricNode(client, f'{{base_path}}/{}')", + field_name, path_segment + ).unwrap(); + } + + writeln!(output).unwrap(); + } +} + +/// Convert an Index to a snake_case field name (e.g., DateIndex -> by_date) +fn index_to_snake_case(index: &Index) -> String { + let short = index.serialize_short(); + format!("by_{}", to_snake_case(short)) +} + +/// Generate structural pattern classes +fn generate_structural_patterns(output: &mut String, patterns: &[StructuralPattern], metadata: &ClientMetadata) { + if patterns.is_empty() { + return; + } + + writeln!(output, "# Reusable structural pattern classes\n").unwrap(); + + for pattern in patterns { + writeln!(output, "class {}:", pattern.name).unwrap(); + writeln!(output, " \"\"\"Pattern struct for repeated tree structure.\"\"\"").unwrap(); + writeln!(output, " ").unwrap(); + writeln!(output, " def __init__(self, client: BrkClientBase, base_path: str):").unwrap(); + + for field in &pattern.fields { + let py_type = field_to_python_type(field, metadata); + if metadata.is_pattern_type(&field.rust_type) { + writeln!( + output, + " self.{}: {} = {}(client, f'{{base_path}}/{}')", + to_snake_case(&field.name), py_type, field.rust_type, field.name + ).unwrap(); + } else if field_uses_accessor(field, metadata) { + let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap(); + writeln!( + output, + " self.{}: {} = {}(client, f'{{base_path}}/{}')", + to_snake_case(&field.name), py_type, accessor.name, field.name + ).unwrap(); + } else { + writeln!( + output, + " self.{}: {} = MetricNode(client, f'{{base_path}}/{}')", + to_snake_case(&field.name), py_type, field.name + ).unwrap(); + } + } + + writeln!(output).unwrap(); + } +} + +/// Convert pattern field to Python type annotation +fn field_to_python_type(field: &PatternField, metadata: &ClientMetadata) -> String { + if metadata.is_pattern_type(&field.rust_type) { + // Pattern type - use pattern name directly + field.rust_type.clone() + } else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) { + // Leaf with a reusable accessor pattern + let py_type = json_type_to_python(&field.json_type); + format!("{}[{}]", accessor.name, py_type) + } else { + // Leaf with unique index set - use MetricNode directly + let py_type = json_type_to_python(&field.json_type); + format!("MetricNode[{}]", py_type) + } +} + +/// Check if a field should use an index accessor +fn field_uses_accessor(field: &PatternField, metadata: &ClientMetadata) -> bool { + metadata.find_index_set_pattern(&field.indexes).is_some() +} + +/// Convert JSON Schema type to Python type +fn json_type_to_python(json_type: &str) -> &str { + match json_type { + "integer" => "int", + "number" => "float", + "boolean" => "bool", + "string" => "str", + "array" => "List", + "object" => "dict", + _ => "Any", + } +} + +/// Generate tree classes +fn generate_tree_classes( + output: &mut String, + catalog: &TreeNode, + metadata: &ClientMetadata, +) { + writeln!(output, "# Catalog tree classes\n").unwrap(); + + let pattern_lookup = metadata.pattern_lookup(); + let mut generated = HashSet::new(); + generate_tree_class(output, "CatalogTree", catalog, &pattern_lookup, metadata, &mut generated); +} + +/// Recursively generate tree classes +fn generate_tree_class( + output: &mut String, + name: &str, + node: &TreeNode, + pattern_lookup: &std::collections::HashMap, String>, + metadata: &ClientMetadata, + generated: &mut HashSet, +) { + if let TreeNode::Branch(children) = node { + // Build signature + let fields = get_node_fields(children, pattern_lookup); + + // Skip if this matches a pattern (already generated) + if pattern_lookup.contains_key(&fields) && pattern_lookup.get(&fields) != Some(&name.to_string()) { + return; + } + + if generated.contains(name) { + return; + } + generated.insert(name.to_string()); + + writeln!(output, "class {}:", name).unwrap(); + writeln!(output, " \"\"\"Catalog tree node.\"\"\"").unwrap(); + writeln!(output, " ").unwrap(); + writeln!(output, " def __init__(self, client: BrkClientBase, base_path: str = ''):").unwrap(); + + for field in &fields { + let py_type = field_to_python_type(field, metadata); + if metadata.is_pattern_type(&field.rust_type) { + writeln!( + output, + " self.{}: {} = {}(client, f'{{base_path}}/{}')", + to_snake_case(&field.name), py_type, field.rust_type, field.name + ).unwrap(); + } else if field_uses_accessor(field, metadata) { + let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap(); + writeln!( + output, + " self.{}: {} = {}(client, f'{{base_path}}/{}')", + to_snake_case(&field.name), py_type, accessor.name, field.name + ).unwrap(); + } else { + writeln!( + output, + " self.{}: {} = MetricNode(client, f'{{base_path}}/{}')", + to_snake_case(&field.name), py_type, field.name + ).unwrap(); + } + } + + writeln!(output).unwrap(); + + // Generate child classes + for (child_name, child_node) in children { + if let TreeNode::Branch(grandchildren) = child_node { + let child_fields = get_node_fields(grandchildren, pattern_lookup); + if !pattern_lookup.contains_key(&child_fields) { + let child_class_name = format!("{}_{}", name, to_pascal_case(child_name)); + generate_tree_class(output, &child_class_name, child_node, pattern_lookup, metadata, generated); + } + } + } + } +} + +/// Generate the main client class +fn generate_main_client(output: &mut String, endpoints: &[Endpoint]) { + writeln!(output, "class BrkClient(BrkClientBase):").unwrap(); + writeln!(output, " \"\"\"Main BRK client with catalog tree and API methods.\"\"\"").unwrap(); + writeln!(output, " ").unwrap(); + writeln!(output, " def __init__(self, base_url: str = 'http://localhost:3000', timeout: float = 30.0):").unwrap(); + writeln!(output, " super().__init__(base_url, timeout)").unwrap(); + writeln!(output, " self.tree = CatalogTree(self)").unwrap(); + writeln!(output).unwrap(); + + // Generate API methods + generate_api_methods(output, endpoints); +} + +/// Generate API methods from OpenAPI endpoints +fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) { + for endpoint in endpoints { + if endpoint.method != "GET" { + continue; + } + + let method_name = endpoint_to_method_name(endpoint); + let return_type = endpoint.response_type.as_deref().unwrap_or("Any"); + + // Build method signature + let params = build_method_params(endpoint); + writeln!(output, " def {}(self{}) -> {}:", method_name, params, return_type).unwrap(); + + // Docstring + if let Some(summary) = &endpoint.summary { + writeln!(output, " \"\"\"{}\"\"\"", summary).unwrap(); + } + + // Build path + let path = build_path_template(&endpoint.path, &endpoint.path_params); + + if endpoint.query_params.is_empty() { + writeln!(output, " return self.get(f'{}')", path).unwrap(); + } else { + writeln!(output, " params = []").unwrap(); + for param in &endpoint.query_params { + if param.required { + writeln!(output, " params.append(f'{}={{{}}}')", param.name, param.name).unwrap(); + } else { + writeln!(output, " if {} is not None: params.append(f'{}={{{}}}')", param.name, param.name, param.name).unwrap(); + } + } + writeln!(output, " query = '&'.join(params)").unwrap(); + writeln!(output, " return self.get(f'{}{{\"?\" + query if query else \"\"}}')", path).unwrap(); + } + + writeln!(output).unwrap(); + } +} + +fn endpoint_to_method_name(endpoint: &Endpoint) -> String { + if let Some(op_id) = &endpoint.operation_id { + return to_snake_case(op_id); + } + let parts: Vec<&str> = endpoint.path.split('/').filter(|s| !s.is_empty() && !s.starts_with('{')).collect(); + format!("get_{}", parts.join("_")) +} + +fn build_method_params(endpoint: &Endpoint) -> String { + let mut params = Vec::new(); + for param in &endpoint.path_params { + params.push(format!(", {}: str", param.name)); + } + for param in &endpoint.query_params { + if param.required { + params.push(format!(", {}: str", param.name)); + } else { + params.push(format!(", {}: Optional[str] = None", param.name)); + } + } + params.join("") +} + +fn build_path_template(path: &str, path_params: &[super::Parameter]) -> String { + let mut result = path.to_string(); + for param in path_params { + let placeholder = format!("{{{}}}", param.name); + let interpolation = format!("{{{{{}}}}}", param.name); + result = result.replace(&placeholder, &interpolation); + } + result +} + +/// Convert JSON Schema to Python type hint +pub fn schema_to_python_type(schema: &serde_json::Value) -> String { + if let Some(ty) = schema.get("type").and_then(|v| v.as_str()) { + match ty { + "null" => "None".to_string(), + "boolean" => "bool".to_string(), + "integer" => "int".to_string(), + "number" => "float".to_string(), + "string" => "str".to_string(), + "array" => { + if let Some(items) = schema.get("items") { + format!("List[{}]", schema_to_python_type(items)) + } else { + "List[Any]".to_string() + } + } + "object" => "dict[str, Any]".to_string(), + _ => "Any".to_string(), + } + } else if schema.get("anyOf").is_some() || schema.get("oneOf").is_some() { + "Any".to_string() + } else if let Some(reference) = schema.get("$ref").and_then(|v| v.as_str()) { + reference.rsplit('/').next().unwrap_or("Any").to_string() + } else { + "Any".to_string() + } +} + diff --git a/crates/brk_binder/src/generator/rust.rs b/crates/brk_binder/src/generator/rust.rs index 3ca7f8ac6..7ae2060cd 100644 --- a/crates/brk_binder/src/generator/rust.rs +++ b/crates/brk_binder/src/generator/rust.rs @@ -1,10 +1,533 @@ +use std::collections::HashSet; +use std::fmt::Write as FmtWrite; +use std::fs; use std::io; use std::path::Path; -use super::ClientMetadata; +use brk_types::{Index, TreeNode}; + +use super::{ClientMetadata, Endpoint, IndexSetPattern, PatternField, StructuralPattern, get_node_fields, to_pascal_case, to_snake_case}; + +/// Generate Rust client from metadata and OpenAPI endpoints +pub fn generate_rust_client( + metadata: &ClientMetadata, + endpoints: &[Endpoint], + output_dir: &Path, +) -> io::Result<()> { + let mut output = String::new(); + + // Header + writeln!(output, "// Auto-generated BRK Rust client").unwrap(); + writeln!(output, "// Do not edit manually\n").unwrap(); + writeln!(output, "#![allow(non_camel_case_types)]").unwrap(); + writeln!(output, "#![allow(dead_code)]\n").unwrap(); + + // Imports + generate_imports(&mut output); + + // Generate base client + generate_base_client(&mut output); + + // Generate MetricNode + generate_metric_node(&mut output); + + // Generate index accessor structs (for each unique set of indexes) + generate_index_accessors(&mut output, &metadata.index_set_patterns); + + // Generate pattern structs (reusable, appearing 2+ times) + generate_pattern_structs(&mut output, &metadata.structural_patterns, metadata); + + // Generate tree - each node uses its pattern or is generated inline + generate_tree(&mut output, &metadata.catalog, metadata); + + // Generate main client with API methods + generate_main_client(&mut output, endpoints); + + fs::write(output_dir.join("client.rs"), output)?; -/// Generate Rust client from metadata -pub fn generate_rust_client(_metadata: &ClientMetadata, _output_dir: &Path) -> io::Result<()> { - // TODO: Implement Rust client generation Ok(()) } + +fn generate_imports(output: &mut String) { + writeln!( + output, + r#"use std::marker::PhantomData; +use serde::de::DeserializeOwned; +use brk_types::*; + +"# + ) + .unwrap(); +} + +fn generate_base_client(output: &mut String) { + writeln!( + output, + r#"/// Error type for BRK client operations. +#[derive(Debug)] +pub struct BrkError {{ + pub message: String, +}} + +impl std::fmt::Display for BrkError {{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{ + write!(f, "{{}}", self.message) + }} +}} + +impl std::error::Error for BrkError {{}} + +/// Result type for BRK client operations. +pub type Result = std::result::Result; + +/// Options for configuring the BRK client. +#[derive(Debug, Clone)] +pub struct BrkClientOptions {{ + pub base_url: String, + pub timeout_ms: u64, +}} + +impl Default for BrkClientOptions {{ + fn default() -> Self {{ + Self {{ + base_url: "http://localhost:3000".to_string(), + timeout_ms: 30000, + }} + }} +}} + +/// Base HTTP client for making requests. +#[derive(Debug, Clone)] +pub struct BrkClientBase {{ + base_url: String, + client: reqwest::blocking::Client, +}} + +impl BrkClientBase {{ + /// Create a new client with the given base URL. + pub fn new(base_url: impl Into) -> Result {{ + let base_url = base_url.into(); + let client = reqwest::blocking::Client::new(); + Ok(Self {{ base_url, client }}) + }} + + /// Create a new client with options. + pub fn with_options(options: BrkClientOptions) -> Result {{ + let client = reqwest::blocking::Client::builder() + .timeout(std::time::Duration::from_millis(options.timeout_ms)) + .build() + .map_err(|e| BrkError {{ message: e.to_string() }})?; + Ok(Self {{ + base_url: options.base_url, + client, + }}) + }} + + /// Make a GET request. + pub fn get(&self, path: &str) -> Result {{ + let url = format!("{{}}{{}}", self.base_url, path); + self.client + .get(&url) + .send() + .map_err(|e| BrkError {{ message: e.to_string() }})? + .json() + .map_err(|e| BrkError {{ message: e.to_string() }}) + }} +}} + +"# + ) + .unwrap(); +} + +fn generate_metric_node(output: &mut String) { + writeln!( + output, + r#"/// A metric node that can fetch data for different indexes. +pub struct MetricNode<'a, T> {{ + client: &'a BrkClientBase, + path: String, + _marker: PhantomData, +}} + +impl<'a, T: DeserializeOwned> MetricNode<'a, T> {{ + pub fn new(client: &'a BrkClientBase, path: String) -> Self {{ + Self {{ + client, + path, + _marker: PhantomData, + }} + }} + + /// Fetch all data points for this metric. + pub fn get(&self) -> Result> {{ + self.client.get(&self.path) + }} + + /// Fetch data points within a date range. + pub fn get_range(&self, from: &str, to: &str) -> Result> {{ + let path = format!("{{}}?from={{}}&to={{}}", self.path, from, to); + self.client.get(&path) + }} +}} + +"# + ) + .unwrap(); +} + +/// Generate index accessor structs for each unique set of indexes +fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) { + if patterns.is_empty() { + return; + } + + writeln!(output, "// Index accessor structs\n").unwrap(); + + for pattern in patterns { + writeln!(output, "/// Index accessor for metrics with {} indexes.", pattern.indexes.len()).unwrap(); + writeln!(output, "pub struct {}<'a, T> {{", pattern.name).unwrap(); + + for index in &pattern.indexes { + let field_name = index_to_field_name(index); + writeln!(output, " pub {}: MetricNode<'a, T>,", field_name).unwrap(); + } + + writeln!(output, " _marker: PhantomData,").unwrap(); + writeln!(output, "}}\n").unwrap(); + + // Generate impl block with constructor + writeln!(output, "impl<'a, T: DeserializeOwned> {}<'a, T> {{", pattern.name).unwrap(); + writeln!(output, " pub fn new(client: &'a BrkClientBase, base_path: &str) -> Self {{").unwrap(); + writeln!(output, " Self {{").unwrap(); + + for index in &pattern.indexes { + let field_name = index_to_field_name(index); + let path_segment = index.serialize_short(); + writeln!( + output, + " {}: MetricNode::new(client, format!(\"{{base_path}}/{}\")),", + field_name, path_segment + ).unwrap(); + } + + writeln!(output, " _marker: PhantomData,").unwrap(); + writeln!(output, " }}").unwrap(); + writeln!(output, " }}").unwrap(); + writeln!(output, "}}\n").unwrap(); + } +} + +/// Convert an Index to a snake_case field name (e.g., DateIndex -> by_date) +fn index_to_field_name(index: &Index) -> String { + let short = index.serialize_short(); + format!("by_{}", to_snake_case(short)) +} + +/// Generate pattern structs (those appearing 2+ times) +fn generate_pattern_structs(output: &mut String, patterns: &[StructuralPattern], metadata: &ClientMetadata) { + if patterns.is_empty() { + return; + } + + writeln!(output, "// Reusable pattern structs\n").unwrap(); + + for pattern in patterns { + writeln!(output, "/// Pattern struct for repeated tree structure.").unwrap(); + writeln!(output, "pub struct {}<'a> {{", pattern.name).unwrap(); + + for field in &pattern.fields { + let field_name = to_snake_case(&field.name); + let type_annotation = field_to_type_annotation(field, metadata); + writeln!(output, " pub {}: {},", field_name, type_annotation).unwrap(); + } + + writeln!(output, "}}\n").unwrap(); + + // Generate impl block with constructor + writeln!(output, "impl<'a> {}<'a> {{", pattern.name).unwrap(); + writeln!(output, " pub fn new(client: &'a BrkClientBase, base_path: &str) -> Self {{").unwrap(); + writeln!(output, " Self {{").unwrap(); + + for field in &pattern.fields { + let field_name = to_snake_case(&field.name); + if metadata.is_pattern_type(&field.rust_type) { + writeln!( + output, + " {}: {}::new(client, &format!(\"{{base_path}}/{}\"))," , + field_name, field.rust_type, field.name + ).unwrap(); + } else if field_uses_accessor(field, metadata) { + let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap(); + writeln!( + output, + " {}: {}::new(client, &format!(\"{{base_path}}/{}\"))," , + field_name, accessor.name, field.name + ).unwrap(); + } else { + writeln!( + output, + " {}: MetricNode::new(client, format!(\"{{base_path}}/{}\"))," , + field_name, field.name + ).unwrap(); + } + } + + writeln!(output, " }}").unwrap(); + writeln!(output, " }}").unwrap(); + writeln!(output, "}}\n").unwrap(); + } +} + +/// Convert a PatternField to the full type annotation +fn field_to_type_annotation(field: &PatternField, metadata: &ClientMetadata) -> String { + if metadata.is_pattern_type(&field.rust_type) { + format!("{}<'a>", field.rust_type) + } else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) { + // Leaf with a reusable accessor pattern + format!("{}<'a, {}>", accessor.name, field.rust_type) + } else { + // Leaf with unique index set - use MetricNode directly + format!("MetricNode<'a, {}>", field.rust_type) + } +} + +/// Check if a field should use an index accessor +fn field_uses_accessor(field: &PatternField, metadata: &ClientMetadata) -> bool { + metadata.find_index_set_pattern(&field.indexes).is_some() +} + +/// Generate the catalog tree structure +fn generate_tree( + output: &mut String, + catalog: &TreeNode, + metadata: &ClientMetadata, +) { + writeln!(output, "// Catalog tree\n").unwrap(); + + let pattern_lookup = metadata.pattern_lookup(); + let mut generated = HashSet::new(); + generate_tree_node(output, "CatalogTree", catalog, &pattern_lookup, metadata, &mut generated); +} + +/// Recursively generate tree nodes +fn generate_tree_node( + output: &mut String, + name: &str, + node: &TreeNode, + pattern_lookup: &std::collections::HashMap, String>, + metadata: &ClientMetadata, + generated: &mut HashSet, +) { + if let TreeNode::Branch(children) = node { + // Build the signature for this node + let mut fields: Vec = children + .iter() + .map(|(child_name, child_node)| { + let (rust_type, json_type, indexes) = match child_node { + TreeNode::Leaf(leaf) => ( + leaf.value_type().to_string(), + leaf.schema.get("type").and_then(|v| v.as_str()).unwrap_or("object").to_string(), + leaf.indexes().clone(), + ), + TreeNode::Branch(grandchildren) => { + // Get pattern name for this child + let child_fields = get_node_fields(grandchildren, pattern_lookup); + let pattern_name = pattern_lookup + .get(&child_fields) + .cloned() + .unwrap_or_else(|| format!("{}_{}", name, to_pascal_case(child_name))); + (pattern_name.clone(), pattern_name, std::collections::BTreeSet::new()) + } + }; + PatternField { + name: child_name.clone(), + rust_type, + json_type, + indexes, + } + }) + .collect(); + fields.sort_by(|a, b| a.name.cmp(&b.name)); + + // Check if this matches a reusable pattern + if let Some(pattern_name) = pattern_lookup.get(&fields) { + // This node matches a pattern that will be generated separately + // Don't generate it here, it's already in pattern_structs + if pattern_name != name { + return; + } + } + + // Generate this struct if not already generated + if generated.contains(name) { + return; + } + generated.insert(name.to_string()); + + writeln!(output, "/// Catalog tree node.").unwrap(); + writeln!(output, "pub struct {}<'a> {{", name).unwrap(); + + for field in &fields { + let field_name = to_snake_case(&field.name); + let type_annotation = field_to_type_annotation(field, metadata); + writeln!(output, " pub {}: {},", field_name, type_annotation).unwrap(); + } + + writeln!(output, "}}\n").unwrap(); + + // Generate impl block + writeln!(output, "impl<'a> {}<'a> {{", name).unwrap(); + writeln!(output, " pub fn new(client: &'a BrkClientBase, base_path: &str) -> Self {{").unwrap(); + writeln!(output, " Self {{").unwrap(); + + for field in &fields { + let field_name = to_snake_case(&field.name); + if metadata.is_pattern_type(&field.rust_type) { + writeln!( + output, + " {}: {}::new(client, &format!(\"{{base_path}}/{}\"))," , + field_name, field.rust_type, field.name + ).unwrap(); + } else if field_uses_accessor(field, metadata) { + let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap(); + writeln!( + output, + " {}: {}::new(client, &format!(\"{{base_path}}/{}\"))," , + field_name, accessor.name, field.name + ).unwrap(); + } else { + writeln!( + output, + " {}: MetricNode::new(client, format!(\"{{base_path}}/{}\"))," , + field_name, field.name + ).unwrap(); + } + } + + writeln!(output, " }}").unwrap(); + writeln!(output, " }}").unwrap(); + writeln!(output, "}}\n").unwrap(); + + // Recursively generate child nodes that aren't patterns + for (child_name, child_node) in children { + if let TreeNode::Branch(grandchildren) = child_node { + let child_fields = get_node_fields(grandchildren, pattern_lookup); + if !pattern_lookup.contains_key(&child_fields) { + let child_struct_name = format!("{}_{}", name, to_pascal_case(child_name)); + generate_tree_node(output, &child_struct_name, child_node, pattern_lookup, metadata, generated); + } + } + } + } +} + +/// Generate the main client struct +fn generate_main_client(output: &mut String, endpoints: &[Endpoint]) { + writeln!( + output, + r#"/// Main BRK client with catalog tree and API methods. +pub struct BrkClient {{ + base: BrkClientBase, +}} + +impl BrkClient {{ + /// Create a new client with the given base URL. + pub fn new(base_url: impl Into) -> Result {{ + Ok(Self {{ + base: BrkClientBase::new(base_url)?, + }}) + }} + + /// Create a new client with options. + pub fn with_options(options: BrkClientOptions) -> Result {{ + Ok(Self {{ + base: BrkClientBase::with_options(options)?, + }}) + }} + + /// Get the catalog tree for navigating metrics. + pub fn tree(&self) -> CatalogTree<'_> {{ + CatalogTree::new(&self.base, "") + }} +"# + ) + .unwrap(); + + // Generate API methods + generate_api_methods(output, endpoints); + + writeln!(output, "}}").unwrap(); +} + +/// Generate API methods from OpenAPI endpoints +fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) { + for endpoint in endpoints { + if endpoint.method != "GET" { + continue; + } + + let method_name = endpoint_to_method_name(endpoint); + let return_type = endpoint.response_type.as_deref().unwrap_or("serde_json::Value"); + + // Build doc comment + writeln!(output, " /// {}", endpoint.summary.as_deref().unwrap_or(&method_name)).unwrap(); + + // Build method signature + let params = build_method_params(endpoint); + writeln!(output, " pub fn {}(&self{}) -> Result<{}> {{", method_name, params, return_type).unwrap(); + + // Build path + let path = build_path_template(&endpoint.path, &endpoint.path_params); + + if endpoint.query_params.is_empty() { + writeln!(output, " self.base.get(&format!(\"{}\"))", path).unwrap(); + } else { + writeln!(output, " let mut query = Vec::new();").unwrap(); + for param in &endpoint.query_params { + if param.required { + writeln!(output, " query.push(format!(\"{}={{}}\", {}));", param.name, param.name).unwrap(); + } else { + writeln!(output, " if let Some(v) = {} {{ query.push(format!(\"{}={{}}\", v)); }}", param.name, param.name).unwrap(); + } + } + writeln!(output, " let query_str = if query.is_empty() {{ String::new() }} else {{ format!(\"?{{}}\", query.join(\"&\")) }};").unwrap(); + writeln!(output, " self.base.get(&format!(\"{}{{}}\", query_str))", path).unwrap(); + } + + writeln!(output, " }}\n").unwrap(); + } +} + +fn endpoint_to_method_name(endpoint: &Endpoint) -> String { + if let Some(op_id) = &endpoint.operation_id { + return to_snake_case(op_id); + } + let parts: Vec<&str> = endpoint.path.split('/').filter(|s| !s.is_empty() && !s.starts_with('{')).collect(); + format!("get_{}", parts.join("_")) +} + +fn build_method_params(endpoint: &Endpoint) -> String { + let mut params = Vec::new(); + for param in &endpoint.path_params { + params.push(format!(", {}: &str", param.name)); + } + for param in &endpoint.query_params { + if param.required { + params.push(format!(", {}: &str", param.name)); + } else { + params.push(format!(", {}: Option<&str>", param.name)); + } + } + params.join("") +} + +fn build_path_template(path: &str, path_params: &[super::Parameter]) -> String { + let mut result = path.to_string(); + for param in path_params { + let placeholder = format!("{{{}}}", param.name); + let interpolation = format!("{{{}}}", param.name); + result = result.replace(&placeholder, &interpolation); + } + result +} diff --git a/crates/brk_binder/src/generator/types.rs b/crates/brk_binder/src/generator/types.rs index a2cbd8910..1374a2315 100644 --- a/crates/brk_binder/src/generator/types.rs +++ b/crates/brk_binder/src/generator/types.rs @@ -1,4 +1,5 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeSet, HashMap}; +use std::hash::{Hash, Hasher}; use brk_query::Vecs; use brk_types::{Index, TreeNode}; @@ -6,94 +7,236 @@ use brk_types::{Index, TreeNode}; /// Metadata extracted from brk_query for client generation #[derive(Debug)] pub struct ClientMetadata { - /// All metrics with their available indexes and value type - pub metrics: BTreeMap, /// The catalog tree structure (with schemas in leaves) pub catalog: TreeNode, - /// Discovered patterns (sets of indexes that appear together frequently) - pub patterns: Vec, + /// Structural patterns - tree node shapes that repeat + pub structural_patterns: Vec, + /// All indexes used across the catalog + pub used_indexes: BTreeSet, + /// Index set patterns - sets of indexes that appear together on metrics + pub index_set_patterns: Vec, } -/// Information about a single metric +/// A pattern of indexes that appear together on multiple metrics #[derive(Debug, Clone)] -pub struct MetricInfo { - /// Metric name (e.g., "difficulty", "supply_total") +pub struct IndexSetPattern { + /// Pattern name (e.g., "DateHeightIndexes") pub name: String, - /// Available indexes for this metric + /// The set of indexes pub indexes: BTreeSet, - /// Value type name (e.g., "Sats", "StoredF64") - pub value_type: String, } -/// A pattern of indexes that appears multiple times across metrics +/// A structural pattern - a branch structure that appears multiple times in the tree #[derive(Debug, Clone)] -pub struct IndexPattern { - /// Unique identifier for this pattern - pub id: usize, - /// The set of indexes in this pattern - pub indexes: BTreeSet, - /// How many metrics use this exact pattern - pub usage_count: usize, +pub struct StructuralPattern { + /// Pattern name - sanitized for all languages (e.g., "BaseCumulativeSum") + pub name: String, + /// Ordered list of child fields (sorted by field name) + pub fields: Vec, } +/// A field in a structural pattern +#[derive(Debug, Clone, PartialOrd, Ord)] +pub struct PatternField { + /// Field name + pub name: String, + /// Rust type: brk_types type for leaves ("Sats", "StoredF64") or pattern name for branches + pub rust_type: String, + /// JSON type from schema: "integer", "number", "string", "boolean", or pattern name for branches + pub json_type: String, + /// For leaves: the set of supported indexes. Empty for branches. + pub indexes: BTreeSet, +} + +// Manual implementations of Hash/Eq/PartialEq that exclude `indexes` +// since indexes aren't part of the structural pattern identity +impl Hash for PatternField { + fn hash(&self, state: &mut H) { + self.name.hash(state); + self.rust_type.hash(state); + self.json_type.hash(state); + // indexes excluded from hash + } +} + +impl PartialEq for PatternField { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + && self.rust_type == other.rust_type + && self.json_type == other.json_type + // indexes excluded from equality + } +} + +impl Eq for PatternField {} + + impl ClientMetadata { /// Extract metadata from brk_query::Vecs pub fn from_vecs(vecs: &Vecs) -> Self { - let mut metrics = BTreeMap::new(); - let mut pattern_counts: BTreeMap, usize> = BTreeMap::new(); - - // Extract metric information - for (name, index_to_vec) in &vecs.metric_to_index_to_vec { - let indexes: BTreeSet = index_to_vec.keys().copied().collect(); - - // Get value type from the first available vec - let value_type = index_to_vec - .values() - .next() - .map(|v| v.value_type_to_string().to_string()) - .unwrap_or_else(|| "unknown".to_string()); - - // Count pattern usage - *pattern_counts.entry(indexes.clone()).or_insert(0) += 1; - - metrics.insert( - name.to_string(), - MetricInfo { - name: name.to_string(), - indexes, - value_type, - }, - ); - } - - // Extract patterns that are used by multiple metrics - let mut patterns: Vec = pattern_counts - .into_iter() - .filter(|(_, count)| *count >= 2) // Only patterns used by 2+ metrics - .enumerate() - .map(|(id, (indexes, usage_count))| IndexPattern { - id, - indexes, - usage_count, - }) - .collect(); - - // Sort by usage count descending - patterns.sort_by(|a, b| b.usage_count.cmp(&a.usage_count)); + let catalog = vecs.catalog().clone(); + let structural_patterns = detect_structural_patterns(&catalog); + let (used_indexes, index_set_patterns) = detect_index_patterns(&catalog); ClientMetadata { - metrics, - catalog: vecs.catalog().clone(), - patterns, + catalog, + structural_patterns, + used_indexes, + index_set_patterns, } } - /// Find the pattern that matches a metric's indexes, if any - pub fn find_pattern_for_metric(&self, metric: &MetricInfo) -> Option<&IndexPattern> { - self.patterns - .iter() - .find(|p| p.indexes == metric.indexes) + /// Check if an index set matches a pattern + pub fn find_index_set_pattern(&self, indexes: &BTreeSet) -> Option<&IndexSetPattern> { + self.index_set_patterns.iter().find(|p| &p.indexes == indexes) } + + /// Check if a type is a pattern (vs a primitive leaf type) + pub fn is_pattern_type(&self, type_name: &str) -> bool { + self.structural_patterns.iter().any(|p| p.name == type_name) + } + + /// Build a lookup map from field signatures to pattern names + pub fn pattern_lookup(&self) -> HashMap, String> { + self.structural_patterns + .iter() + .map(|p| (p.fields.clone(), p.name.clone())) + .collect() + } +} + +/// Detect structural patterns in the tree using a bottom-up approach. +/// For every branch node, create a signature from its children (sorted field names + types). +/// Patterns that appear 2+ times are deduplicated. +fn detect_structural_patterns(tree: &TreeNode) -> Vec { + // Map from sorted fields signature to pattern name + let mut signature_to_pattern: HashMap, String> = HashMap::new(); + // Count how many times each signature appears + let mut signature_counts: HashMap, usize> = HashMap::new(); + + // Process tree bottom-up to resolve all branch types + resolve_branch_patterns(tree, &mut signature_to_pattern, &mut signature_counts); + + // Build final list of patterns (only those appearing 2+ times) + let mut patterns: Vec = signature_to_pattern + .into_iter() + .filter(|(sig, _)| signature_counts.get(sig).copied().unwrap_or(0) >= 2) + .map(|(fields, name)| StructuralPattern { name, fields }) + .collect(); + + // Sort by number of fields descending (larger patterns first) + patterns.sort_by(|a, b| b.fields.len().cmp(&a.fields.len())); + + patterns +} + +/// Recursively resolve branch patterns bottom-up. +/// Returns the pattern name for this node if it's a branch, or None if it's a leaf. +fn resolve_branch_patterns( + node: &TreeNode, + signature_to_pattern: &mut HashMap, String>, + signature_counts: &mut HashMap, usize>, +) -> Option { + match node { + TreeNode::Leaf(_) => { + // Leaves don't have patterns, return None + None + } + TreeNode::Branch(children) => { + // First, recursively resolve all children + let mut fields: Vec = Vec::new(); + + for (child_name, child_node) in children { + let (rust_type, json_type, indexes) = match child_node { + TreeNode::Leaf(leaf) => ( + leaf.value_type().to_string(), + schema_to_json_type(&leaf.schema), + leaf.indexes().clone(), + ), + TreeNode::Branch(_) => { + // Branch: recursively get its pattern name + let pattern_name = resolve_branch_patterns(child_node, signature_to_pattern, signature_counts) + .unwrap_or_else(|| "Unknown".to_string()); + (pattern_name.clone(), pattern_name, BTreeSet::new()) + } + }; + + fields.push(PatternField { + name: child_name.clone(), + rust_type, + json_type, + indexes, + }); + } + + // Sort fields by name for consistent signatures + fields.sort_by(|a, b| a.name.cmp(&b.name)); + + // Increment count for this signature + *signature_counts.entry(fields.clone()).or_insert(0) += 1; + + // Get or create pattern name for this signature + let pattern_name = signature_to_pattern + .entry(fields.clone()) + .or_insert_with(|| generate_pattern_name_from_fields(&fields)) + .clone(); + + Some(pattern_name) + } + } +} + +/// Generate a sanitized pattern name from fields. +/// Names must be valid identifiers in all target languages (Rust, JS, Python). +fn generate_pattern_name_from_fields(fields: &[PatternField]) -> String { + // Join field names with underscores, then convert to PascalCase + let joined: Vec<&str> = fields.iter().map(|f| f.name.as_str()).collect(); + let raw_name = joined.join("_"); + + // Sanitize: ensure it starts with a letter (prepend "P_" if starts with digit) + let sanitized = if raw_name.chars().next().map(|c| c.is_ascii_digit()).unwrap_or(false) { + format!("P_{}", raw_name) + } else { + raw_name + }; + + to_pascal_case(&sanitized) +} + +/// Extract JSON type from JSON Schema +fn schema_to_json_type(schema: &serde_json::Value) -> String { + if let Some(ty) = schema.get("type").and_then(|v| v.as_str()) { + ty.to_string() + } else { + "object".to_string() + } +} + +/// Get the field signature for a branch node's children +pub fn get_node_fields( + children: &std::collections::BTreeMap, + pattern_lookup: &HashMap, String>, +) -> Vec { + let mut fields: Vec = children + .iter() + .map(|(name, node)| { + let (rust_type, json_type, indexes) = match node { + TreeNode::Leaf(leaf) => ( + leaf.value_type().to_string(), + schema_to_json_type(&leaf.schema), + leaf.indexes().clone(), + ), + TreeNode::Branch(grandchildren) => { + let child_fields = get_node_fields(grandchildren, pattern_lookup); + let pattern_name = pattern_lookup.get(&child_fields).cloned().unwrap_or_else(|| "Unknown".to_string()); + (pattern_name.clone(), pattern_name, BTreeSet::new()) + } + }; + PatternField { name: name.clone(), rust_type, json_type, indexes } + }) + .collect(); + fields.sort_by(|a, b| a.name.cmp(&b.name)); + fields } /// Convert a metric name to PascalCase (for struct/class names) @@ -132,6 +275,73 @@ pub fn to_camel_case(s: &str) -> String { } } +/// Detect index patterns - collect all indexes and find sets that appear 2+ times +fn detect_index_patterns(tree: &TreeNode) -> (BTreeSet, Vec) { + let mut used_indexes: BTreeSet = BTreeSet::new(); + let mut index_sets: Vec> = Vec::new(); + + // Traverse tree and collect index information from leaves + collect_indexes_from_tree(tree, &mut used_indexes, &mut index_sets); + + // Count occurrences of each unique index set + let mut index_set_counts: Vec<(BTreeSet, usize)> = Vec::new(); + for index_set in index_sets { + if let Some(entry) = index_set_counts.iter_mut().find(|(s, _)| s == &index_set) { + entry.1 += 1; + } else { + index_set_counts.push((index_set, 1)); + } + } + + // Build patterns for index sets appearing 2+ times + let mut patterns: Vec = index_set_counts + .into_iter() + .filter(|(indexes, count)| *count >= 2 && !indexes.is_empty()) + .map(|(indexes, _)| IndexSetPattern { + name: generate_index_set_name(&indexes), + indexes, + }) + .collect(); + + // Sort by number of indexes descending + patterns.sort_by(|a, b| b.indexes.len().cmp(&a.indexes.len())); + + (used_indexes, patterns) +} + +/// Recursively collect indexes from tree leaves +fn collect_indexes_from_tree( + node: &TreeNode, + used_indexes: &mut BTreeSet, + index_sets: &mut Vec>, +) { + match node { + TreeNode::Leaf(leaf) => { + // Add all indexes from this leaf to the global set + used_indexes.extend(leaf.indexes().iter().cloned()); + // Collect this index set + index_sets.push(leaf.indexes().clone()); + } + TreeNode::Branch(children) => { + for (_, child) in children { + collect_indexes_from_tree(child, used_indexes, index_sets); + } + } + } +} + +/// Generate a name for an index set pattern +fn generate_index_set_name(indexes: &BTreeSet) -> String { + if indexes.len() == 1 { + let index = indexes.iter().next().unwrap(); + return format!("{}Accessor", to_pascal_case(index.serialize_short())); + } + + // For multiple indexes, create a descriptive name + let names: Vec<&str> = indexes.iter().map(|i| i.serialize_short()).collect(); + format!("{}Accessor", to_pascal_case(&names.join("_"))) +} + /// Convert a serde_json::Value (JSON Schema) to a JSDoc type annotation pub fn schema_to_jsdoc(schema: &serde_json::Value) -> String { if let Some(ty) = schema.get("type").and_then(|v| v.as_str()) { diff --git a/crates/brk_computer/.gitignore b/crates/brk_computer/.gitignore index b2e818da8..d171c7d5a 100644 --- a/crates/brk_computer/.gitignore +++ b/crates/brk_computer/.gitignore @@ -1 +1,2 @@ bottlenecks.md +BUG.md diff --git a/crates/brk_computer/src/stateful/process/lookup.rs b/crates/brk_computer/src/stateful/process/lookup.rs index 070a37003..dd4f6845e 100644 --- a/crates/brk_computer/src/stateful/process/lookup.rs +++ b/crates/brk_computer/src/stateful/process/lookup.rs @@ -34,10 +34,30 @@ impl<'a> AddressLookup<'a> { match map.entry(type_index) { Entry::Occupied(entry) => { + // Address is in cache. Need to determine if it's been processed + // by process_received (added to a cohort) or just loaded this block. + // + // - If wrapper is New AND funded_txo_count == 0: hasn't received yet, + // was just created in process_outputs this block → New + // - If wrapper is New AND funded_txo_count > 0: received in previous + // block but still in cache (no flush) → Loaded + // - If wrapper is FromLoaded/FromEmpty: loaded from storage → use wrapper let source = match entry.get() { - WithAddressDataSource::New(_) => AddressSource::New, + WithAddressDataSource::New(data) => { + if data.funded_txo_count == 0 { + AddressSource::New + } else { + AddressSource::Loaded + } + } WithAddressDataSource::FromLoaded(..) => AddressSource::Loaded, - WithAddressDataSource::FromEmpty(..) => AddressSource::FromEmpty, + WithAddressDataSource::FromEmpty(_, data) => { + if data.utxo_count() == 0 { + AddressSource::FromEmpty + } else { + AddressSource::Loaded + } + } }; (entry.into_mut(), source) } diff --git a/crates/brk_computer/src/stateful/process/received.rs b/crates/brk_computer/src/stateful/process/received.rs index b09bdeb04..98e84d377 100644 --- a/crates/brk_computer/src/stateful/process/received.rs +++ b/crates/brk_computer/src/stateful/process/received.rs @@ -62,13 +62,30 @@ pub fn process_received( if AmountBucket::from(prev_balance) != AmountBucket::from(new_balance) { // Crossing cohort boundary - subtract from old, add to new - cohorts + let cohort_state = cohorts .amount_range .get_mut(prev_balance) .state .as_mut() - .unwrap() - .subtract(addr_data); + .unwrap(); + + // Debug info for tracking down underflow issues + if cohort_state.inner.supply.utxo_count < addr_data.utxo_count() as u64 { + panic!( + "process_received: cohort underflow detected!\n\ + output_type={:?}, type_index={:?}\n\ + prev_balance={}, new_balance={}, total_value={}\n\ + Address: {:?}", + output_type, + type_index, + prev_balance, + new_balance, + total_value, + addr_data + ); + } + + cohort_state.subtract(addr_data); addr_data.receive_outputs(total_value, price, output_count); cohorts .amount_range diff --git a/crates/brk_computer/src/stateful/process/sent.rs b/crates/brk_computer/src/stateful/process/sent.rs index 247bca0d1..dee8c2f36 100644 --- a/crates/brk_computer/src/stateful/process/sent.rs +++ b/crates/brk_computer/src/stateful/process/sent.rs @@ -8,7 +8,7 @@ use brk_error::Result; use brk_grouper::{ByAddressType, Filtered}; use brk_types::{CheckedSub, Dollars, Height, Sats, Timestamp, TypeIndex}; -use vecdb::VecIndex; +use vecdb::{VecIndex, unlikely}; use super::super::address::HeightToAddressTypeToVec; use super::super::cohorts::AddressCohorts; @@ -63,13 +63,36 @@ pub fn process_sent( if will_be_empty || filters_differ { // Subtract from old cohort - cohorts + let cohort_state = cohorts .amount_range .get_mut(prev_balance) .state .as_mut() - .unwrap() - .subtract(addr_data); + .unwrap(); + + // Debug info for tracking down underflow issues + if unlikely( + cohort_state.inner.supply.utxo_count < addr_data.utxo_count() as u64, + ) { + panic!( + "process_sent: cohort underflow detected!\n\ + Block context: prev_height={:?}, output_type={:?}, type_index={:?}\n\ + prev_balance={}, new_balance={}, value={}\n\ + will_be_empty={}, filters_differ={}\n\ + Address: {:?}", + prev_height, + output_type, + type_index, + prev_balance, + new_balance, + value, + will_be_empty, + filters_differ, + addr_data + ); + } + + cohort_state.subtract(addr_data); // Update address data addr_data.send(value, prev_price)?; diff --git a/crates/brk_computer/src/stateful/states/address_cohort.rs b/crates/brk_computer/src/stateful/states/address_cohort.rs index 9e2d8ce98..5d2112929 100644 --- a/crates/brk_computer/src/stateful/states/address_cohort.rs +++ b/crates/brk_computer/src/stateful/states/address_cohort.rs @@ -2,6 +2,7 @@ use std::path::Path; use brk_error::Result; use brk_types::{Dollars, Height, LoadedAddressData, Sats}; +use vecdb::unlikely; use crate::stateful::states::{RealizedState, SupplyState}; @@ -136,11 +137,46 @@ impl AddressCohortState { } pub fn subtract(&mut self, addressdata: &LoadedAddressData) { - self.addr_count = self.addr_count.checked_sub(1).unwrap(); + let addr_supply: SupplyState = addressdata.into(); + let realized_price = addressdata.realized_price(); + + // Check for potential underflow before it happens + if unlikely(self.inner.supply.utxo_count < addr_supply.utxo_count) { + panic!( + "AddressCohortState::subtract underflow!\n\ + Cohort state: addr_count={}, supply={}\n\ + Address being subtracted: {}\n\ + Address supply: {}\n\ + Realized price: {}\n\ + This means the address is not properly tracked in this cohort.", + self.addr_count, self.inner.supply, addressdata, addr_supply, realized_price + ); + } + if unlikely(self.inner.supply.value < addr_supply.value) { + panic!( + "AddressCohortState::subtract value underflow!\n\ + Cohort state: addr_count={}, supply={}\n\ + Address being subtracted: {}\n\ + Address supply: {}\n\ + Realized price: {}\n\ + This means the address is not properly tracked in this cohort.", + self.addr_count, self.inner.supply, addressdata, addr_supply, realized_price + ); + } + + self.addr_count = self.addr_count.checked_sub(1).unwrap_or_else(|| { + panic!( + "AddressCohortState::subtract addr_count underflow! addr_count=0\n\ + Address being subtracted: {}\n\ + Realized price: {}", + addressdata, realized_price + ) + }); + self.inner.decrement_( - &addressdata.into(), + &addr_supply, addressdata.realized_cap, - addressdata.realized_price(), + realized_price, ); } diff --git a/crates/brk_computer/src/stateful/states/price_to_amount.rs b/crates/brk_computer/src/stateful/states/price_to_amount.rs index e765fa884..85858544c 100644 --- a/crates/brk_computer/src/stateful/states/price_to_amount.rs +++ b/crates/brk_computer/src/stateful/states/price_to_amount.rs @@ -9,7 +9,7 @@ use brk_types::{Dollars, Height, Sats}; use derive_deref::{Deref, DerefMut}; use pco::standalone::{simple_decompress, simpler_compress}; use serde::{Deserialize, Serialize}; -use vecdb::Bytes; +use vecdb::{Bytes, unlikely}; use crate::{grouped::PERCENTILES_LEN, utils::OptionExt}; @@ -87,6 +87,25 @@ impl PriceToAmount { pub fn decrement(&mut self, price: Dollars, supply_state: &SupplyState) { if let Some(amount) = self.state.um().get_mut(&price) { + if unlikely(*amount < supply_state.value) { + let amount = *amount; + panic!( + "PriceToAmount::decrement underflow!\n\ + Path: {:?}\n\ + Price: {}\n\ + Bucket amount: {}\n\ + Trying to decrement by: {}\n\ + Supply state: utxo_count={}, value={}\n\ + All buckets: {:?}", + self.pathbuf, + price, + amount, + supply_state.value, + supply_state.utxo_count, + supply_state.value, + self.state.u().iter().collect::>() + ); + } *amount -= supply_state.value; if *amount == Sats::ZERO { self.state.um().remove(&price); @@ -95,8 +114,18 @@ impl PriceToAmount { buckets.decrement(price, supply_state.value); } } else { - dbg!(price, &self.pathbuf); - unreachable!(); + panic!( + "PriceToAmount::decrement price not found!\n\ + Path: {:?}\n\ + Price: {}\n\ + Supply state: utxo_count={}, value={}\n\ + All buckets: {:?}", + self.pathbuf, + price, + supply_state.utxo_count, + supply_state.value, + self.state.u().iter().collect::>() + ); } } diff --git a/crates/brk_computer/src/stateful/states/supply.rs b/crates/brk_computer/src/stateful/states/supply.rs index 31322b679..a8ec8fe7f 100644 --- a/crates/brk_computer/src/stateful/states/supply.rs +++ b/crates/brk_computer/src/stateful/states/supply.rs @@ -39,8 +39,22 @@ impl AddAssign<&SupplyState> for SupplyState { impl SubAssign<&SupplyState> for SupplyState { fn sub_assign(&mut self, rhs: &Self) { - self.utxo_count = self.utxo_count.checked_sub(rhs.utxo_count).unwrap(); - self.value = self.value.checked_sub(rhs.value).unwrap(); + self.utxo_count = self.utxo_count.checked_sub(rhs.utxo_count).unwrap_or_else(|| { + panic!( + "SupplyState underflow: cohort utxo_count {} < address utxo_count {}. \ + This indicates a desync between cohort state and address data. \ + Try deleting the compute cache and restarting fresh.", + self.utxo_count, rhs.utxo_count + ) + }); + self.value = self.value.checked_sub(rhs.value).unwrap_or_else(|| { + panic!( + "SupplyState underflow: cohort value {} < address value {}. \ + This indicates a desync between cohort state and address data. \ + Try deleting the compute cache and restarting fresh.", + self.value, rhs.value + ) + }); } } diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 6d9dc0cce..3f0c275e2 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -11,6 +11,7 @@ build = "build.rs" [dependencies] aide = { workspace = true } axum = { workspace = true } +brk_binder = { workspace = true } brk_computer = { workspace = true } brk_error = { workspace = true } brk_fetcher = { workspace = true } diff --git a/crates/brk_server/src/lib.rs b/crates/brk_server/src/lib.rs index d36da938a..462266e57 100644 --- a/crates/brk_server/src/lib.rs +++ b/crates/brk_server/src/lib.rs @@ -89,6 +89,7 @@ impl Server { .on_failure(()) .on_eos(()); + let vecs = state.query.inner().vecs(); let router = ApiRouter::new() .add_api_routes() .add_mcp_routes(&state.query, mcp) @@ -133,10 +134,25 @@ impl Server { info!("Starting server on port {port}..."); let mut openapi = create_openapi(); + let router = router.finish_api(&mut openapi); + + let clients_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("brk_binder") + .join("clients"); + if clients_path.exists() { + let openapi_json = serde_json::to_string(&openapi).unwrap(); + if let Err(e) = brk_binder::generate_clients(vecs, &openapi_json, &clients_path) { + error!("Failed to generate clients: {e}"); + } else { + info!("Generated clients at {}", clients_path.display()); + } + } + serve( listener, router - .finish_api(&mut openapi) .layer(Extension(Arc::new(openapi))) .into_make_service(), ) diff --git a/crates/brk_types/src/loadedaddressdata.rs b/crates/brk_types/src/loadedaddressdata.rs index 25e9c3acb..b79660b7f 100644 --- a/crates/brk_types/src/loadedaddressdata.rs +++ b/crates/brk_types/src/loadedaddressdata.rs @@ -51,7 +51,13 @@ impl LoadedAddressData { #[inline] pub fn utxo_count(&self) -> u32 { - self.funded_txo_count - self.spent_txo_count + self.funded_txo_count.checked_sub(self.spent_txo_count).unwrap_or_else(|| { + panic!( + "LoadedAddressData corruption: spent_txo_count ({}) > funded_txo_count ({}). \ + Address data: {:?}", + self.spent_txo_count, self.funded_txo_count, self + ) + }) } #[inline] diff --git a/crates/brk_types/src/sats.rs b/crates/brk_types/src/sats.rs index 33fea5bea..820b00acb 100644 --- a/crates/brk_types/src/sats.rs +++ b/crates/brk_types/src/sats.rs @@ -110,11 +110,9 @@ impl SaturatingAdd for Sats { impl SubAssign for Sats { fn sub_assign(&mut self, rhs: Self) { - *self = self.checked_sub(rhs).unwrap(); - // .unwrap_or_else(|| { - // dbg!((*self, rhs)); - // unreachable!(); - // }); + *self = self.checked_sub(rhs).unwrap_or_else(|| { + panic!("Sats underflow: {} - {} would be negative", self, rhs); + }); } }