Compare commits
381 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 70e7e24b4f | |||
| 13ab7d39d7 | |||
| 3cae817915 | |||
| c33444a92e | |||
| 3e9b1cc2b2 | |||
| ac6175688d | |||
| a6f8108165 | |||
| 8cff55a405 | |||
| bd376f86ea | |||
| d9f28e85af | |||
| ed18fd55e1 | |||
| 5b06098368 | |||
| e89a67b9a7 | |||
| 445959f5b9 | |||
| 647f177f31 | |||
| 705dbdbd7e | |||
| 31d2f8ef37 | |||
| 236b4097c5 | |||
| f5790d5c8a | |||
| f08ac7f916 | |||
| e77d338357 | |||
| 5d6325ae30 | |||
| 9ba77dac0f | |||
| f9856cf0aa | |||
| de93f08e93 | |||
| d538280f4b | |||
| bbb74b76c8 | |||
| eadf93b804 | |||
| f29443fc15 | |||
| 75a023bdd8 | |||
| d30344ee3c | |||
| 02d635d48b | |||
| 40ec356cc3 | |||
| 5a5d4da57d | |||
| efb247d104 | |||
| 457b0e24c5 | |||
| 6e0ac138d8 | |||
| 26c6c92bb8 | |||
| e1ad45f44b | |||
| aebca14d78 | |||
| 42b0d7a174 | |||
| a37c2474fe | |||
| 5f308e9da7 | |||
| 3aadced85d | |||
| 9375d5aded | |||
| 2c8205146c | |||
| 8d5a2b911d | |||
| 7d5de7bf24 | |||
| 4b1410855a | |||
| 78a4d1af65 | |||
| 5e3519aad4 | |||
| 4386ef47fe | |||
| 135a18d56f | |||
| 71f45479b9 | |||
| bcb8d5bed6 | |||
| 8f19bf7350 | |||
| 25860636f0 | |||
| 8c2402cacb | |||
| 4b910ceaa7 | |||
| 4a0ce6337f | |||
| e134ed11a9 | |||
| 03b83846ef | |||
| 7c86c803fa | |||
| a31d9dc15e | |||
| 57749da919 | |||
| 9ad3acbdf9 | |||
| 6fa53aca9f | |||
| bd53168c4e | |||
| 08d17b4a09 | |||
| c5657b9c31 | |||
| 549e2da05b | |||
| c5e912593a | |||
| a86085c2db | |||
| edbec6fd5c | |||
| a76139c0ea | |||
| 59f1296d56 | |||
| 14ae41c7ba | |||
| df09b3aa28 | |||
| f9fad2d775 | |||
| fa609c73ba | |||
| 9b2f334130 | |||
| a006cefd71 | |||
| 4b2ada14a0 | |||
| 1ad8d8a631 | |||
| 3ca83a2289 | |||
| 2ccf0ef856 | |||
| f7f065c6e0 | |||
| 593af69230 | |||
| 032f3cb66b | |||
| 692a1889ab | |||
| 825a4a77c0 | |||
| 882a3525af | |||
| b491b1f41f | |||
| db5d784ff7 | |||
| db57db4bd9 | |||
| c5e9b75261 | |||
| c59ac62e45 | |||
| 9c8b9b1a3b | |||
| 158b0254ed | |||
| 3526a177fc | |||
| e755f2856a | |||
| 2ec3ca8308 | |||
| 1cf75b48b5 | |||
| abde9ed162 | |||
| 998db1beed | |||
| 79e352d06e | |||
| b8f77433b9 | |||
| 96b967f6fb | |||
| 68c71e62d6 | |||
| 60a38b4108 | |||
| f4a1384dc4 | |||
| b88f0bab56 | |||
| f23907768f | |||
| f280b03cab | |||
| 554c0e565d | |||
| cfc5f7633b | |||
| 82050c7c01 | |||
| f4edb695de | |||
| dc2fa233ab | |||
| a1f31a14be | |||
| d27cc02e8c | |||
| fcc74ba212 | |||
| f48ad577d3 | |||
| 60c73f5635 | |||
| 24248215e9 | |||
| b6ec133368 | |||
| 35e567cfb6 | |||
| 25c697cca1 | |||
| 30dc695741 | |||
| 9e41d51702 | |||
| dc86514329 | |||
| c644781d18 | |||
| eedc0dd075 | |||
| c8c62b504b | |||
| 8467e218ae | |||
| e8f77ab2e5 | |||
| 1d2c927d94 | |||
| 81da73bc53 | |||
| 2dcbd8df99 | |||
| 37f5f50867 | |||
| f6a2a0540b | |||
| dc2e847f58 | |||
| e77fe0253e | |||
| 3d3787a8d9 | |||
| 11b323ef00 | |||
| df577ca7f5 | |||
| a2ba4d89f3 | |||
| 2ad55bf558 | |||
| cf08e470ef | |||
| 82e59d409e | |||
| 7d01e9e91e | |||
| 1e4acfe124 | |||
| 4f1653b086 | |||
| 6cd60a064b | |||
| 8072c4670c | |||
| 4ffa2e3993 | |||
| 9b230d23dd | |||
| baa7c9cc22 | |||
| 33a92cfad4 | |||
| e9f6295014 | |||
| 71078b5bdd | |||
| 6cce92af22 | |||
| d3b8520c41 | |||
| 5425085953 | |||
| db0298ac1b | |||
| 7bfca87caf | |||
| 5f87594ead | |||
| bb46481d7f | |||
| 1821d5d57b | |||
| 6ad15221de | |||
| 83d74da556 | |||
| 114228e8eb | |||
| a53f89c849 | |||
| 7ff79c3164 | |||
| db344749b6 | |||
| 1c6ece48a8 | |||
| b622285999 | |||
| 5fde0101bf | |||
| a6062d4c39 | |||
| 66f1e92cb6 | |||
| d9c4653f82 | |||
| cfdf8fdbca | |||
| 138b2bd357 | |||
| 16b14b1fe1 | |||
| c4ce718bb2 | |||
| 62d4b35c93 | |||
| 7407c032e5 | |||
| 9d03fdf31d | |||
| dfe5148f17 | |||
| 0d5b792c57 | |||
| 2279aa8f18 | |||
| d45686128e | |||
| 5b6ce5d8ee | |||
| aad34c4d52 | |||
| 470082cc65 | |||
| 6554f35710 | |||
| 335fe24a54 | |||
| 3831ef7b25 | |||
| 8127337a09 | |||
| 9a59c2e541 | |||
| 27adca5653 | |||
| 2c5b502da9 | |||
| 23f6397a97 | |||
| 43117825d7 | |||
| cc5701ea62 | |||
| 9524eafea1 | |||
| c28a0f96f7 | |||
| 301dee96dc | |||
| 185fc7b6ed | |||
| 6d194dbb71 | |||
| d34f4bdd12 | |||
| 17dc4bde5e | |||
| ce50b14591 | |||
| f7bd319954 | |||
| e9c0121a18 | |||
| 01aa425f81 | |||
| 38d5c7dff6 | |||
| e3b4b9b618 | |||
| a5951c58f3 | |||
| 504d6eaa9f | |||
| 6253fa30ef | |||
| 47f7cef4f4 | |||
| 72bba06e71 | |||
| 9b92c5ce38 | |||
| dfa077a1c9 | |||
| 18fb2e7d4d | |||
| a610fd53e2 | |||
| 16abce1f2d | |||
| f3b42f34a6 | |||
| 6483d324de | |||
| 5ab97050dd | |||
| 17eed70903 | |||
| 88067c03b7 | |||
| 7c1e5b913f | |||
| 0014235e91 | |||
| a39b7be1d1 | |||
| de98c5f706 | |||
| 10b496e845 | |||
| bbe7bf390d | |||
| 4777b3400a | |||
| acaa70e944 | |||
| 4049d694f7 | |||
| e155a3dacf | |||
| a224e4c4d8 | |||
| edaeda5424 | |||
| 09d974913d | |||
| f82edb290a | |||
| 3d8b33ae94 | |||
| 565ecbd436 | |||
| 3359dfcc29 | |||
| 1c2afd14dd | |||
| fe5343c1d6 | |||
| 08cfefc02a | |||
| f6d9332c48 | |||
| cc6913c854 | |||
| 8c75fbd0a4 | |||
| 0de6d62409 | |||
| 5ba7ce5b7c | |||
| e106d30852 | |||
| 30affc884b | |||
| 745717ea49 | |||
| 4efd98b758 | |||
| 36640e3710 | |||
| 311c4fd29d | |||
| f50374f983 | |||
| 82ceb7f021 | |||
| 0aba3bc1d8 | |||
| f6c984ff3c | |||
| 4091ab6b6c | |||
| fb9fd5b51a | |||
| 9389700a01 | |||
| 016c1b2233 | |||
| 38b8a08297 | |||
| c9ffd3ad99 | |||
| 61f960de28 | |||
| da1ff2cacc | |||
| 05036c682f | |||
| 7d47bc8042 | |||
| 98cfd160ef | |||
| b5e3262b67 | |||
| 009fb35c4c | |||
| 8648d3131a | |||
| 00c316c35d | |||
| 5f8de8e756 | |||
| ee5dc8fc41 | |||
| a61926988a | |||
| bd8c4dfb6b | |||
| ce9b4bc4dd | |||
| 8b12b00114 | |||
| 1775cc1d54 | |||
| e4bd09df24 | |||
| 5e8c7da4df | |||
| c85592eefe | |||
| 05861c9113 | |||
| 3508d1e315 | |||
| e3177b8054 | |||
| 03e3760152 | |||
| 4740610923 | |||
| e28a0cde55 | |||
| 5b855fd835 | |||
| a2f5704581 | |||
| f7aa9424db | |||
| aa8b47a3dd | |||
| 11911c1898 | |||
| 4814c1971d | |||
| be9569f3fb | |||
| 900e72f95a | |||
| d2827f188b | |||
| cf9903b759 | |||
| 23f96461f4 | |||
| 9f2fd26e98 | |||
| 78d837c080 | |||
| 241b9312b7 | |||
| ed70ad7378 | |||
| 00213176d8 | |||
| 406650a45a | |||
| 56750ccf3c | |||
| dfc286b393 | |||
| 49a66f72fc | |||
| 3f237689da | |||
| cf1fb483b3 | |||
| b10f5e3f67 | |||
| c4fc24c513 | |||
| 3ac9c2d95e | |||
| e5ab4dafc0 | |||
| 10ae1911c3 | |||
| 73ebcdf0d6 | |||
| 5347523921 | |||
| 7ef70b953b | |||
| ccaca524fe | |||
| dd51f91cab | |||
| 537d98b41b | |||
| 9c4cadfc04 | |||
| 2001370441 | |||
| cc87b22757 | |||
| c0a65b30ad | |||
| c07e66c086 | |||
| a0cfc1be2b | |||
| 1505454793 | |||
| e1dff66283 | |||
| 5be801a086 | |||
| 94d4b05c29 | |||
| cebb889f7e | |||
| c4ed6ed034 | |||
| ec960bfefa | |||
| 79f689dde1 | |||
| 3b3654df56 | |||
| c66f008f07 | |||
| 37d9498d90 | |||
| 1ff67093db | |||
| daed37ccb8 | |||
| d41d807b4f | |||
| d6fa5c8a55 | |||
| 2dd608dfed | |||
| a98546f605 | |||
| 3567559d4e | |||
| 216476ee45 | |||
| 3fc28c07fb | |||
| 85f6ef063d | |||
| 1e71e2d68f | |||
| b24a29895f | |||
| 0167a2ae59 | |||
| 2c867103ca | |||
| 8c289df336 | |||
| 4489920cbf | |||
| 029a85081b | |||
| 1bc739d07f | |||
| c229e218f6 | |||
| a66f4ad4bd | |||
| 1dd687dab7 | |||
| 50ff6e2745 | |||
| 811dec713b | |||
| 617d6f4bd7 | |||
| 57cd2d6252 | |||
| 5de9757d46 | |||
| f89276d7b8 | |||
| 30ba034206 | |||
| fa1e5aaa7f | |||
| 870c70180f | |||
| d83a833b4d | |||
| ec3a2f29f0 |
@@ -1,4 +1,4 @@
|
||||
# This file was autogenerated by dist: https://opensource.axo.dev/cargo-dist/
|
||||
# This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist
|
||||
#
|
||||
# Copyright 2022-2024, axodotdev
|
||||
# SPDX-License-Identifier: MIT or Apache-2.0
|
||||
@@ -47,7 +47,7 @@ on:
|
||||
jobs:
|
||||
# Run 'dist plan' (or host) to determine what tasks we need to do
|
||||
plan:
|
||||
runs-on: "ubuntu-latest"
|
||||
runs-on: "ubuntu-22.04"
|
||||
outputs:
|
||||
val: ${{ steps.plan.outputs.manifest }}
|
||||
tag: ${{ !github.event.pull_request && github.ref_name || '' }}
|
||||
@@ -58,12 +58,13 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install dist
|
||||
# we specify bash to get pipefail; it guards against the `curl` command
|
||||
# failing. otherwise `sh` won't catch that `curl` returned non-0
|
||||
shell: bash
|
||||
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.28.0/cargo-dist-installer.sh | sh"
|
||||
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.30.2/cargo-dist-installer.sh | sh"
|
||||
- name: Cache dist
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
@@ -117,6 +118,7 @@ jobs:
|
||||
git config --global core.longpaths true
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install Rust non-interactively if not already installed
|
||||
if: ${{ matrix.container }}
|
||||
@@ -168,13 +170,14 @@ jobs:
|
||||
needs:
|
||||
- plan
|
||||
- build-local-artifacts
|
||||
runs-on: "ubuntu-latest"
|
||||
runs-on: "ubuntu-22.04"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install cached dist
|
||||
uses: actions/download-artifact@v4
|
||||
@@ -214,16 +217,17 @@ jobs:
|
||||
- plan
|
||||
- build-local-artifacts
|
||||
- build-global-artifacts
|
||||
# Only run if we're "publishing", and only if local and global didn't fail (skipped is fine)
|
||||
if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
|
||||
# Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine)
|
||||
if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
runs-on: "ubuntu-latest"
|
||||
runs-on: "ubuntu-22.04"
|
||||
outputs:
|
||||
val: ${{ steps.host.outputs.manifest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install cached dist
|
||||
uses: actions/download-artifact@v4
|
||||
@@ -282,10 +286,11 @@ jobs:
|
||||
# still allowing individual publish jobs to skip themselves (for prereleases).
|
||||
# "host" however must run to completion, no skipping allowed!
|
||||
if: ${{ always() && needs.host.result == 'success' }}
|
||||
runs-on: "ubuntu-latest"
|
||||
runs-on: "ubuntu-22.04"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
|
||||
@@ -3,18 +3,39 @@
|
||||
|
||||
# Builds
|
||||
target
|
||||
dist
|
||||
vecid-to-indexes.js
|
||||
websites/dist
|
||||
bridge/
|
||||
/ids.txt
|
||||
|
||||
# Copies
|
||||
*\ copy*
|
||||
|
||||
# Ignored
|
||||
_*
|
||||
|
||||
# Editors
|
||||
.vscode
|
||||
.zed
|
||||
!__*.py
|
||||
/*.md
|
||||
/api.json
|
||||
|
||||
# Logs
|
||||
.log
|
||||
*.log*
|
||||
|
||||
# Environment variables/configs
|
||||
.env
|
||||
|
||||
# Profiling
|
||||
profile.json.gz
|
||||
flamegraph.svg
|
||||
*.trace
|
||||
|
||||
# AI
|
||||
.claude/settings*
|
||||
|
||||
# Expand
|
||||
expand.rs
|
||||
|
||||
# Benchmarks
|
||||
[0-9]/
|
||||
/benches
|
||||
|
||||
# AI
|
||||
.claude
|
||||
|
||||
@@ -1,287 +0,0 @@
|
||||
<!--
|
||||
# v0.X.Y | WIP
|
||||

|
||||
-->
|
||||
|
||||
# v0.X.0 | WIP | A new beginning
|
||||
|
||||

|
||||
|
||||
Full rewrite
|
||||
|
||||
# [kibo-v0.5.0](https://github.com/bitcoinresearchkit/brk/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
|
||||
|
||||

|
||||
|
||||
## Datasets
|
||||
|
||||
- Added `Sell Side Risk Ratio` to all entities
|
||||
- Added `Open`, `High` and `Low` datasets
|
||||
- Added `Satoshis Per Dollar`
|
||||
- Added `All Time High`
|
||||
- Added `All Time High Date`
|
||||
- Added `Days Since All Time High`
|
||||
- Added `Max Days Between All Time Highs`
|
||||
- Added `Max Years Between All Time Highs`
|
||||
- Added `Drawdown`
|
||||
- Added `Adjusted Value Created`, `Adjusted Value Destroyed` and `Adjusted Spent Output Profit Ratio` to all entities
|
||||
- Added `Realized Profit To Loss Ratio` to all entities
|
||||
- Added `Hash Price Min`
|
||||
- Added `Hash Price Rebound`
|
||||
- Removed all year datasets (25) in favor for epoch datasets (5), the former was too granular to be really useful
|
||||
- Removed datasets split by liquidity for all datasets **already split by any address kind**, while fun to have, they took time to compute, ram, and space to store and no one was actually checking them
|
||||
- Fixed a lot of values in split by liquidity datasets
|
||||
|
||||
## Website
|
||||
|
||||
- Updated the design yet again which made the website for something more minimal and easier on the eyes
|
||||
- Added a *Save In Bitcoin* (DCA) simulation page
|
||||
- ~Added a dashboard~ Added the latest values to the tree next to each option instead, while less values are visible at a time, it's much more readable and organised
|
||||
- Added a library of PDFs
|
||||
- Fixed service worker not passing 304 (not modified) response and instead serving cached responses
|
||||
- Fixed history not being properly registered
|
||||
- Fixed window being moveable on iOS when in standalone mode when it shouldn't be
|
||||
- Added `Compare` section to all groups, to compare all datasets within a group
|
||||
- Updated `Solid Signals` library, which had an important breaking change on the `createEffect` function which might bring some bugs
|
||||
- Fixed some datasets paths
|
||||
- A lot of code reorg and file splits
|
||||
- Adopted a framework like approach to load pages while still being pure JS without a build step
|
||||
- Probably more that was forgotten
|
||||
|
||||
## Parser
|
||||
|
||||
- Added a `/datasets/last` json file with all the latest values
|
||||
- Added `--rpcconnect` parameter to the config
|
||||
- Added handling of SIGINT and SIGTERM terminal signals which menas you can now safely CTRL+C or kill the parser while it's exporting
|
||||
- Added config print at the start of the program
|
||||
- Compressed `empty_address_data` struct to save space (should shave of between up to 50% of the `address_index_to_empty_address_data` database)
|
||||
- Doubled the number of `txid_to_tx_data` databases from 4096 to 8192
|
||||
- ~Added `--recompute_computed true` argument, to allow recomputation of computed datasets in case of a bug~ Buggy for now
|
||||
- Fixed not saved arguments, not being processed properly
|
||||
- Fixed bug in `generic_map.multi_insert_simple_average`
|
||||
- Added defragmentation option `--first-defragment true` of databases to save space (which can save up to 50%)
|
||||
- Fixed bug in the computation of averages in `GenericMap`
|
||||
- Added support and paramer for cookie files with `--rpccookiefile`, and auto find if the path is `--datadir/.cookie`
|
||||
- Increased number of retries and time between them when fetching price from exchanges APIs
|
||||
|
||||
## Server
|
||||
|
||||
- Fixed links in several places missing the `/api` part and thus not working
|
||||
- Fixed broken last values routes
|
||||
- Added support for the `/datasets/last` file via the `/api/last` route
|
||||
- Added support for `.json` (won't change anything) and `.csv` (will download a csv file) extension at the end of datasets routes
|
||||
- Added `all=true` query parameter to dataset routes to get to full history
|
||||
|
||||
## Biter
|
||||
|
||||
- Moved back to this repo
|
||||
|
||||
# [kibo-v0.4.0](https://github.com/bitcoinresearchkit/brk/tree/a64c544815d9ef785e2fc1323582f774f16b9200) | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
|
||||
|
||||

|
||||
|
||||
## Brand
|
||||
|
||||
- **Satonomics** is now **kibo** 🎉
|
||||
|
||||
## Website
|
||||
|
||||
- Complete redesign of the website
|
||||
- Rewrote the whole application and removed `node`/`npm`/`pnpm` dependencies in favor for pure `HTML`/`CSS`/`Javascript`
|
||||
- Website is now served by the server
|
||||
- Added Trading View attribution link to the settings frame and file in the lightweight charts folder
|
||||
- Many other changes
|
||||
|
||||
## Parser
|
||||
|
||||
- Changed the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind`
|
||||
- Added datasets compression thanks to [zstd](https://crates.io/crates/zstd) to reduce disk usage
|
||||
- Use the Bitcoin RPC server for various calls instead of running cli commands and then parsing the JSON from the output
|
||||
- **Important database changes that will need a full rescan**:
|
||||
- Changed databases page size from 1MB to 4KB for improved disk usage
|
||||
- Split txid_to_tx_data database in 4096 chunks (from 256) for improved disk usage
|
||||
- Split address_index_to_X databases to chunks of 25_000 instead of 50_000
|
||||
- Removed local Multisig database
|
||||
- Updated the config, run with `-h` to see possible args
|
||||
- Moved outputs from `/target/outputs` to `/out` to allow to run commands like `cargo clean` without side effects
|
||||
- Various first run fixes
|
||||
- Added to `-h` which arguments are saved, which is all of them at the time of writing
|
||||
|
||||
## Server
|
||||
|
||||
- Updated the code to support compressed binaries
|
||||
- Added serving of the website
|
||||
- Improved `Cache-Control` behavior
|
||||
|
||||
# [satonomics-v0.3.0](https://github.com/bitcoinresearchkit/brk/tree/b68b016091c45b071218fba01bac5b76e8eaf18c) | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26
|
||||
|
||||

|
||||
|
||||
## Parser
|
||||
|
||||
- Global
|
||||
- Improved self-hosting by:
|
||||
- Fixing an incredibly annoying bug that made the program panic because of a wrong utxo/address durable state after a or many new datasets were added/changed after a first successful parse of the chain
|
||||
- Fixing a bug that would crash the program if launched for the first time ever
|
||||
- Auto fetch prices from the main Satonomics instance if missing instead of only trying Kraken's and Binance's API which are limited to the last 16 hours
|
||||
- Merged the core of `HeightMap` and `DateMap` structs into `GenericMap`
|
||||
- Added `Height` struct and many others
|
||||
- Reorganized outputs of both the parser and the server for ease of use and easier sync compatibility
|
||||
- CLI
|
||||
- Added an argument parser for improved UX with several options
|
||||
- Datasets
|
||||
- Added the following datasets for all entities:
|
||||
- Value destroyed
|
||||
- Value created
|
||||
- Spent Output Profit Ratio (SOPR)
|
||||
- Added the following ratio datasets and their variations to all prices {realized, moving average, any cointime, etc}:
|
||||
- Market Price to {X}
|
||||
- Market Price to {X} Ratio
|
||||
- Market Price to {X} Ratio 1 Week SMA
|
||||
- Market Price to {X} Ratio 1 Month SMA
|
||||
- Market Price to {X} Ratio 1 Year SMA
|
||||
- Market Price to {X} Ratio 1 Year SMA Momentum Oscillator
|
||||
- Market Price to {X} Ratio 99th Percentile
|
||||
- Market Price to {X} Ratio 99.5th Percentile
|
||||
- Market Price to {X} Ratio 99.9th Percentile
|
||||
- Market Price to {X} Ratio 1st Percentile
|
||||
- Market Price to {X} Ratio 0.5th Percentile
|
||||
- {X} 1% Top Probability
|
||||
- {X} 0.5% Top Probability
|
||||
- {X} 0.1% Top Probability
|
||||
- {X} 1% Bottom Probability
|
||||
- {X} 0.5% Bottom Probability
|
||||
- {X} 0.1% Bottom Probability
|
||||
- Added block metadatasets and their variants (raw/sum/average/min/max/percentiles):
|
||||
- Block size
|
||||
- Block weight
|
||||
- Block VBytes
|
||||
- Block interval
|
||||
- Price
|
||||
- Improved error message when price cannot be found
|
||||
|
||||
## App
|
||||
|
||||
- General
|
||||
- Added chart scroll button for nice animations à la Wicked
|
||||
- Added scale mode switch (Linear/Logarithmic) at the bottom right of all charts
|
||||
- Added unit at the top left of all charts
|
||||
- Added a backup API in case the main one fails or is offline
|
||||
- Complete redesign of the datasets object
|
||||
- Removed import of routes in JSON in favor for hardcoded typed routes in string format which resulted in:
|
||||
- \+ A much lighter app
|
||||
- \+ Better Lighthouse score
|
||||
- \- Slower Typescript server
|
||||
- Fixed datasets with null values crashing their fetch function
|
||||
- Added a 'Go to a random chart' button in several places
|
||||
- Chart
|
||||
- Fixed series color being set to default ones after hovering the legend
|
||||
- Fixed chart starting showing candlesticks and quickly switching to a line when it should've started directly with the line
|
||||
- Separated the QRCode generator library from the main chunk and made it imported on click
|
||||
- Fixed timescale changing on small screen after changing charts
|
||||
- Folders
|
||||
- Added the size in the "filename" of address cohorts grouped by size
|
||||
- Favorites
|
||||
- Added a 'favorite' and 'unfavorite' button at the bottom
|
||||
- Settings
|
||||
- Removed the horizontal scroll bar which was unintended
|
||||
|
||||
## Server
|
||||
|
||||
- Run file
|
||||
- Only run with a watcher if `cargo watch` is available
|
||||
- Removed id_to_path file in favor for only `paths.d.ts` in `app/src/types`
|
||||
|
||||
# [satonomics-v0.2.0](https://github.com/bitcoinresearchkit/brk/tree/248187889283597c5dbb806292297453c25e97b8) | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08
|
||||
|
||||

|
||||
|
||||
## App
|
||||
|
||||
- General
|
||||
- Added the height version of all datasets and many optimizations to make them usable but only available on desktop and tablets for now
|
||||
- Added a light theme
|
||||
- Charts
|
||||
- Added split panes in order to have the vertical axis visible for all datasets
|
||||
- Added min and max values on the charts
|
||||
- Fixed legend hovering on mobile not resetting on touch end
|
||||
- Added "3 months" and yearly time scale setters (from year 2009 to today)
|
||||
- Hide scrollbar of timescale setters and instead added scroll buttons to the legend only visible on desktop
|
||||
- Improved Share/QR Code screen
|
||||
- Changed all Area series to Line series
|
||||
- Fixed horizontal scrollable legend not updating on preset change
|
||||
- Performance
|
||||
- Improved app's reactivity
|
||||
- Added some chunk splitting for a faster initial load
|
||||
- Global improvements that increased the Lighthouse's performance score
|
||||
- Settings
|
||||
- Finally made a proper component where you can chose the app's theme, between a moving or static background and its text opacity
|
||||
- Added donations section with a leaderboard
|
||||
- Added various links that are visible on the bottom side of the strip on desktop to mobile users
|
||||
- Added install instructions when not installed for Apple users
|
||||
- Misc
|
||||
- Support mini window size, could be useful for embedded views
|
||||
- Hopefully made scrollbars a little more subtle on WIndows and Linux, can't test
|
||||
- Generale style updates
|
||||
|
||||
## Parser
|
||||
|
||||
- Fixed ulimit only being run in Mac OS instead of whenever the program is detected
|
||||
|
||||
# [satonomics-v0.1.1](https://github.com/bitcoinresearchkit/brk/tree/e55b5195a9de9aea306903c94ed63cb1720fda5f) | [849240](https://mempool.space/block/000000000000000000002b8653988655071c07bb5f7181c038f9326bc86db741) - 2024/06/24
|
||||
|
||||

|
||||
|
||||
## Parser
|
||||
|
||||
- Fixed overflow in `Price` struct which caused many Realized Caps and Realized Prices to have completely bogus data
|
||||
- Fixed Realized Cap computation which was using rounded prices instead normal ones
|
||||
|
||||
## Server
|
||||
|
||||
- Added the chunk, date and time of the request to the terminal logs
|
||||
|
||||
## App
|
||||
|
||||
- Chart
|
||||
- Added double click option on a legend to toggle the visibility of all other series
|
||||
- Added highlight effect to a legend by darkening the color of all the other series on the chart while hovering it with the mouse
|
||||
- Added an API link in the legend for each dataset where applicable (when isn't generated locally)
|
||||
- Save fullscreen preference in local storage and url
|
||||
- Improved resize bar on desktop
|
||||
- Changed resize button logo
|
||||
- Changed the share button to visible on small screen too
|
||||
- Improved share screen
|
||||
- Fixed time range shifting not being the one in url params or saved in local storage
|
||||
- Fixed time range shifting on series toggling via the legend
|
||||
- Fixed time range shifting on fullscreen
|
||||
- Fixed time range shifting on resize of the sidebar
|
||||
- Set default view at first load to last 6 months
|
||||
- Added some padding around the datasets (year 1970 to 2100)
|
||||
- History
|
||||
- Changed background for the sticky dates from blur to a solid color as it didn't appear properly in Firefox
|
||||
- Build
|
||||
- Tried to add lazy loads to have split chunks after build, to have much faster load times and they worked great ! But they completely broke Safari on iOS, we can't have nice things
|
||||
- Removed many libraries and did some things manually instead to improve build size
|
||||
- Strip
|
||||
- Temporarily removed the Home button on the strip bar on desktop as there is no landing page yet
|
||||
- Settings
|
||||
- Added version
|
||||
- PWA
|
||||
- Fixed background update
|
||||
- Changed update check frequency to 1 minute (~1kb to fetch every minute which is very reasonable)
|
||||
- Added a nice banner to ask the user to install the update
|
||||
- Misc
|
||||
- Removed tracker even though it was a very privacy friendly as it appeared to not be working properly
|
||||
|
||||
## Price
|
||||
|
||||
- Deleted old price datasets and their backups
|
||||
|
||||
# [satonomics-v0.1.0](https://github.com/bitcoinresearchkit/brk/tree/a1a576d088c8f83ed32d48753a7611f70a964574) | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19
|
||||
|
||||

|
||||
|
||||
# satonomics-v0.0.1 | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20
|
||||
|
||||

|
||||
@@ -4,63 +4,86 @@ members = ["crates/*"]
|
||||
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
|
||||
package.license = "MIT"
|
||||
package.edition = "2024"
|
||||
package.version = "0.0.73"
|
||||
package.version = "0.1.0-alpha.2"
|
||||
package.homepage = "https://bitcoinresearchkit.org"
|
||||
package.repository = "https://github.com/bitcoinresearchkit/brk"
|
||||
package.readme = "README.md"
|
||||
|
||||
[profile.dev]
|
||||
lto = "thin"
|
||||
codegen-units = 16
|
||||
opt-level = 2
|
||||
split-debuginfo = "unpacked"
|
||||
|
||||
[profile.release]
|
||||
lto = "fat"
|
||||
codegen-units = 1
|
||||
panic = "abort"
|
||||
strip = true
|
||||
overflow-checks = false
|
||||
|
||||
[profile.bloaty]
|
||||
debug = true
|
||||
lto = false
|
||||
strip = false
|
||||
inherits = "release"
|
||||
|
||||
[profile.dist]
|
||||
inherits = "release"
|
||||
|
||||
[profile.profiling]
|
||||
inherits = "release"
|
||||
debug = true
|
||||
|
||||
[profile.dist]
|
||||
inherits = "release"
|
||||
|
||||
[workspace.dependencies]
|
||||
arc-swap = "1.7.1"
|
||||
axum = "0.8.4"
|
||||
bincode = { version = "2.0.1", features = ["serde"] }
|
||||
bitcoin = { version = "0.32.6", features = ["serde"] }
|
||||
aide = { version = "0.16.0-alpha.1", features = ["axum-json", "axum-query"] }
|
||||
axum = "0.8.8"
|
||||
bitcoin = { version = "0.32.8", features = ["serde"] }
|
||||
bitcoincore-rpc = "0.19.0"
|
||||
brk_bundler = { version = "0.0.73", path = "crates/brk_bundler" }
|
||||
brk_cli = { version = "0.0.73", path = "crates/brk_cli" }
|
||||
brk_computer = { version = "0.0.73", path = "crates/brk_computer" }
|
||||
brk_core = { version = "0.0.73", path = "crates/brk_core" }
|
||||
brk_exit = { version = "0.0.73", path = "crates/brk_exit" }
|
||||
brk_fetcher = { version = "0.0.73", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0.0.73", path = "crates/brk_indexer" }
|
||||
brk_interface = { version = "0.0.73", path = "crates/brk_interface" }
|
||||
brk_logger = { version = "0.0.73", path = "crates/brk_logger" }
|
||||
brk_mcp = { version = "0.0.73", path = "crates/brk_mcp" }
|
||||
brk_parser = { version = "0.0.73", path = "crates/brk_parser" }
|
||||
brk_rmcp = { version = "0.2.1", features = ["transport-streamable-http-server", "transport-worker"]}
|
||||
# brk_rmcp = { path = "../rust-sdk/crates/rmcp", features = ["transport-streamable-http-server", "transport-worker"]}
|
||||
brk_server = { version = "0.0.73", path = "crates/brk_server" }
|
||||
brk_store = { version = "0.0.73", path = "crates/brk_store" }
|
||||
brk_vec = { version = "0.0.73", path = "crates/brk_vec" }
|
||||
byteview = "=0.6.1"
|
||||
clap = { version = "4.5.40", features = ["string"] }
|
||||
clap_derive = "4.5.40"
|
||||
brk_alloc = { version = "0.1.0-alpha.2", path = "crates/brk_alloc" }
|
||||
brk_bencher = { version = "0.1.0-alpha.2", path = "crates/brk_bencher" }
|
||||
brk_bindgen = { version = "0.1.0-alpha.2", path = "crates/brk_bindgen" }
|
||||
brk_bundler = { version = "0.1.0-alpha.2", path = "crates/brk_bundler" }
|
||||
brk_cli = { version = "0.1.0-alpha.2", path = "crates/brk_cli" }
|
||||
brk_client = { version = "0.1.0-alpha.2", path = "crates/brk_client" }
|
||||
brk_cohort = { version = "0.1.0-alpha.2", path = "crates/brk_cohort" }
|
||||
brk_computer = { version = "0.1.0-alpha.2", path = "crates/brk_computer" }
|
||||
brk_error = { version = "0.1.0-alpha.2", path = "crates/brk_error" }
|
||||
brk_fetcher = { version = "0.1.0-alpha.2", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0.1.0-alpha.2", path = "crates/brk_indexer" }
|
||||
brk_query = { version = "0.1.0-alpha.2", path = "crates/brk_query", features = ["tokio"] }
|
||||
brk_iterator = { version = "0.1.0-alpha.2", path = "crates/brk_iterator" }
|
||||
brk_logger = { version = "0.1.0-alpha.2", path = "crates/brk_logger" }
|
||||
brk_mcp = { version = "0.1.0-alpha.2", path = "crates/brk_mcp" }
|
||||
brk_mempool = { version = "0.1.0-alpha.2", path = "crates/brk_mempool" }
|
||||
brk_reader = { version = "0.1.0-alpha.2", path = "crates/brk_reader" }
|
||||
brk_rpc = { version = "0.1.0-alpha.2", path = "crates/brk_rpc" }
|
||||
brk_server = { version = "0.1.0-alpha.2", path = "crates/brk_server" }
|
||||
brk_store = { version = "0.1.0-alpha.2", path = "crates/brk_store" }
|
||||
brk_types = { version = "0.1.0-alpha.2", path = "crates/brk_types" }
|
||||
brk_traversable = { version = "0.1.0-alpha.2", path = "crates/brk_traversable", features = ["pco", "derive"] }
|
||||
brk_traversable_derive = { version = "0.1.0-alpha.2", path = "crates/brk_traversable_derive" }
|
||||
byteview = "0.10.0"
|
||||
color-eyre = "0.6.5"
|
||||
derive_deref = "1.1.1"
|
||||
fjall = "2.11.1"
|
||||
jiff = "0.2.15"
|
||||
log = { version = "0.4.27" }
|
||||
minreq = { version = "2.14.0", features = ["https", "serde_json"] }
|
||||
rayon = "1.10.0"
|
||||
schemars = "1.0.4"
|
||||
serde = { version = "1.0.219" }
|
||||
serde_bytes = "0.11.17"
|
||||
serde_derive = "1.0.219"
|
||||
serde_json = { version = "1.0.140", features = ["float_roundtrip"] }
|
||||
tabled = "0.20.0"
|
||||
tokio = { version = "1.46.1", features = ["rt-multi-thread"] }
|
||||
zerocopy = { version = "0.8.26" }
|
||||
zerocopy-derive = "0.8.26"
|
||||
env_logger = "0.11.8"
|
||||
fjall = "3.0.0"
|
||||
jiff = "0.2.17"
|
||||
log = "0.4.29"
|
||||
minreq = { version = "2.14.1", features = ["https", "serde_json"] }
|
||||
parking_lot = "0.12.5"
|
||||
rayon = "1.11.0"
|
||||
rustc-hash = "2.1.1"
|
||||
schemars = "1.2.0"
|
||||
serde = "1.0.228"
|
||||
serde_bytes = "0.11.19"
|
||||
serde_derive = "1.0.228"
|
||||
serde_json = { version = "1.0.148", features = ["float_roundtrip"] }
|
||||
smallvec = "1.15.1"
|
||||
tokio = { version = "1.49.0", features = ["rt-multi-thread"] }
|
||||
vecdb = { version = "0.5.4", features = ["derive", "serde_json", "pco", "schemars"] }
|
||||
# vecdb = { path = "../anydb/crates/vecdb", features = ["derive", "serde_json", "pco", "schemars"] }
|
||||
# vecdb = { git = "https://github.com/anydb-rs/anydb", features = ["derive", "serde_json", "pco"] }
|
||||
|
||||
[workspace.metadata.release]
|
||||
shared-version = true
|
||||
@@ -69,20 +92,8 @@ pre-release-commit-message = "release: v{{version}}"
|
||||
tag-message = "release: v{{version}}"
|
||||
|
||||
[workspace.metadata.dist]
|
||||
cargo-dist-version = "0.28.0"
|
||||
cargo-dist-version = "0.30.2"
|
||||
ci = "github"
|
||||
allow-dirty = ["ci"]
|
||||
installers = []
|
||||
targets = [
|
||||
"aarch64-apple-darwin",
|
||||
"aarch64-unknown-linux-gnu",
|
||||
"x86_64-apple-darwin",
|
||||
"x86_64-unknown-linux-gnu",
|
||||
]
|
||||
|
||||
[workspace.metadata.dist.github-custom-runners]
|
||||
global = "ubuntu-latest"
|
||||
aarch64-apple-darwin.runner = "macos-14"
|
||||
x86_64-unknown-linux-gnu.runner = "ubuntu-latest"
|
||||
x86_64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" }
|
||||
aarch64-unknown-linux-gnu.runner = "ubuntu-latest"
|
||||
aarch64-unknown-linux-gnu.container = { image = "quay.io/pypa/manylinux_2_28_x86_64", host = "x86_64-unknown-linux-musl" }
|
||||
targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
# Bitcoin Research Kit
|
||||
|
||||
<p align="left">
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
<a href="https://crates.io/crates/brk">
|
||||
<img src="https://img.shields.io/crates/v/brk" alt="Version" />
|
||||
</a>
|
||||
<a href="https://docs.rs/brk">
|
||||
<img src="https://img.shields.io/docsrs/brk" alt="Documentation" />
|
||||
</a>
|
||||
<img src="https://img.shields.io/crates/size/brk" alt="Size" />
|
||||
<a href="https://deps.rs/crate/brk">
|
||||
<img src="https://deps.rs/crate/brk/latest/status.svg" alt="Dependency status">
|
||||
</a>
|
||||
<a href="https://discord.gg/HaR3wpH3nr">
|
||||
<img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
|
||||
</a>
|
||||
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
|
||||
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
|
||||
</a>
|
||||
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
|
||||
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
|
||||
</a>
|
||||
<a href="https://x.com/brkdotorg">
|
||||
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
|
||||
</a>
|
||||
</p>
|
||||
|
||||
The Bitcoin Research Kit is a high-performance toolchain designed to parse, index, compute, serve and visualize data from a Bitcoin node, enabling users to gain deeper insights into the Bitcoin network.
|
||||
|
||||
In other words it's an alternative to [Glassnode](https://glassnode.com), [mempool.space](https://mempool.space/) (soon) and [electrs](https://github.com/romanz/electrs) (soon) all in one package with a particular focus on simplicity and ease of use.
|
||||
|
||||
The toolkit can be used in various ways to accommodate as many needs as possible:
|
||||
|
||||
- **[Website](https://bitcoinresearchkit.org)** \
|
||||
Everyone is welcome to visit the official instance and showcase of the suite's capabilities. \
|
||||
It has a wide range of functionalities including charts, tables and simulations which you can visit for free and without the need for an account. \
|
||||
Also available at: [kibo.money](https://kibo.money) // [satonomics.xyz](https://satonomics.xyz)
|
||||
- **[API](https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#brk-server)** \
|
||||
Researchers and developers are free to use BRK's public API with  dataset variants at their disposal. \
|
||||
Just like the website, it's entirely free, with no authentication or rate-limiting.
|
||||
- **[AI](https://github.com/bitcoinresearchkit/brk/blob/main/crates/brk_mcp/README.md#brk-mcp)** \
|
||||
LLMs have to possibility to connect to BRK's backend through a [MCP](https://modelcontextprotocol.io/introduction). \
|
||||
It will give them access to the same tools as the API, with no restrictions, and allow you to have your very own data analysts. \
|
||||
One-shot output examples: [Document](https://claude.ai/public/artifacts/71194d29-f965-417c-ba09-fdf0e4ecb1d5) // [Dashboard](https://claude.ai/public/artifacts/beef143f-399a-4ed4-b8bf-c986b776de42) // [Dashboard 2](https://claude.ai/public/artifacts/5430ae49-bb3d-4fc1-ab24-f1e33deb40dc)
|
||||
- **[CLI](https://crates.io/crates/brk_cli)** \
|
||||
Node runners are strongly encouraged to try out and self-host their own instance using BRK's command line interface. \
|
||||
The CLI has multiple cogs available for users to tweak to adapt to all situations with even the possibility for web developers to create their own custom website which could later on be added as an alternative front-end.
|
||||
- **[Crates](https://crates.io/crates/brk)** \
|
||||
Rust developers have access to a wide range crates, each built upon one another with its own specific purpose, enabling independent use and offering great flexibility.
|
||||
PRs are welcome, especially if their goal is to introduce additional datasets.
|
||||
|
||||
The primary goal of this project is to be fully-featured and accessible for everyone, regardless of their background or financial situation - whether that person is an enthusiast, researcher, miner, analyst, or simply curious.
|
||||
|
||||
In contrast, existing alternatives tend to be either [very costly](https://studio.glassnode.com/pricing) or missing essential features, with the vast majority being closed-source and unverifiable, which fundamentally undermines the principles of Bitcoin.
|
||||
|
||||
## Crates
|
||||
|
||||
- [`brk`](https://crates.io/crates/brk): Wrapper around all other `brk-*` crates
|
||||
- [`brk_cli`](https://crates.io/crates/brk_cli): A command line interface to run a Bitcoin Research Kit instance
|
||||
- [`brk_computer`](https://crates.io/crates/brk_computer): A Bitcoin dataset computer, built on top of brk_indexer
|
||||
- [`brk_core`](https://crates.io/crates/brk_core): The Core (Structs and Errors) of the Bitcoin Research Kit
|
||||
- [`brk_exit`](https://crates.io/crates/brk_exit): An exit blocker built on top of ctrlc
|
||||
- [`brk_fetcher`](https://crates.io/crates/brk_fetcher): A Bitcoin price fetcher
|
||||
- [`brk_indexer`](https://crates.io/crates/brk_indexer): A Bitcoin Core indexer built on top of brk_parser
|
||||
- [`brk_logger`](https://crates.io/crates/brk_logger): A clean logger used in the Bitcoin Research Kit
|
||||
- [`brk_mcp`](https://crates.io/crates/brk_mcp): A Model Context Protocol (MCP) which gives LLMs access to all available tools in BRK
|
||||
- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust
|
||||
- [`brk_interface`](https://crates.io/crates/brk_interface): An interface to BRK's engine
|
||||
- [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer`
|
||||
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
|
||||
- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
|
||||
- [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/)
|
||||
|
||||
## Hosting as a service
|
||||
|
||||
If you'd like to have your own instance hosted for you please contact [hosting@bitcoinresearchkit.org](mailto:hosting@bitcoinresearchkit.org).
|
||||
|
||||
- 2 separate dedicated servers (1 GB/s each) with different ISPs and Cloudflare integration for enhanced performance and optimal availability
|
||||
- 99.99% SLA
|
||||
- Configured for speed
|
||||
- Updates delivered at your convenience
|
||||
- Direct communication for feature requests and support
|
||||
- Bitcoin Core or Knots with desired version
|
||||
- Optional subdomains: `*.bitcoinresearchkit.org`, `*.kibo.money` and `*.satonomics.xyz`
|
||||
- Logo featured in the Readme if desired
|
||||
|
||||
Pricing: `0.01 BTC / month` *or* `0.1 BTC / year`
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
Deepest gratitude to the [Open Sats](https://opensats.org/) public charity. Their grant — from December 2024 to the present — has been critical in sustaining this project.
|
||||
|
||||
Heartfelt thanks go out to every donor on [Nostr](https://primal.net/p/npub1jagmm3x39lmwfnrtvxcs9ac7g300y3dusv9lgzhk2e4x5frpxlrqa73v44) and [Geyser.fund](https://geyser.fund/project/brk) whose support has ensured the availability of the [bitcoinresearchkit.org](https://bitcoinresearchkit.org) public instance.
|
||||
|
||||
## Donate
|
||||
|
||||
[`bc1q09 8zsm89 m7kgyz e338vf ejhpdt 92ua9p 3peuve`](bitcoin:bc1q098zsm89m7kgyze338vfejhpdt92ua9p3peuve)
|
||||
|
||||
[`lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4`](lightning:lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4)
|
||||
|
||||
[Geyser Fund](https://geyser.fund/project/brk)
|
||||
@@ -1,92 +0,0 @@
|
||||
# TODO
|
||||
|
||||
- __crates__
|
||||
- _cli_
|
||||
- check disk space on first launch
|
||||
- add custom path support for config.toml
|
||||
- maybe add bitcoind download and launch support
|
||||
- via: https://github.com/rust-bitcoin/corepc/blob/master/node
|
||||
- test read/write speed, add warning if too low (<2gb/s)
|
||||
- pull latest version and notify is out of date
|
||||
- _computer_
|
||||
- **add rollback of states (in stateful)**
|
||||
- add support for per index computation
|
||||
- fix feerate which is always ZERO due to coinbase transaction
|
||||
- before computing multiple sources check their length, panic if not equal
|
||||
- add oracle price dataset (https://utxo.live/oracle/UTXOracle.py)
|
||||
- add address counts relative to all datasets
|
||||
- make decade, quarter, year datasets `computed` instead of `eager`
|
||||
- add 6 months (semester) interval datasets to builder
|
||||
- add revived/sent supply datasets
|
||||
- add `in-sats` version of all price datasets (average and co)
|
||||
- add `p2pk` group (sum of `p2pk33` and `p2pk65`)
|
||||
- add more date ranges (3-6 months and more)
|
||||
- add puell multiple dataset
|
||||
- add pi cycle dataset
|
||||
- add ema of price
|
||||
- add 7d and 30d ema to sell side risk ratio and sopr
|
||||
- add all possible charts from:
|
||||
- https://mainnet.observer
|
||||
- https://glassnode.com
|
||||
- https://checkonchain.com
|
||||
- https://researchbitcoin.net/exciting-update-coming-to-the-bitcoin-lab/
|
||||
- https://mempool.space/research
|
||||
- _indexer_
|
||||
- parse only the needed block number
|
||||
- maybe using https://developer.bitcoin.org/reference/rpc/getblockhash.html
|
||||
- _interface_
|
||||
- create pagination enum
|
||||
- from to
|
||||
- from option<count>
|
||||
- to option<count>
|
||||
- page + option<per page> default 1000 max 1000
|
||||
- from/to/count params don’t cap all combinations
|
||||
- example: from -10,000 count 10, won’t work if underlying vec isn’t 10k or more long
|
||||
- _parser_
|
||||
- save `vec` file instead of `json`
|
||||
- support lock file, process in read only if already opened in write mode
|
||||
- if less than X (10 maybe ?) get block using rpc instead of parsing the block files
|
||||
- _server_
|
||||
- api
|
||||
- add extensions support (.json .csv …)
|
||||
- if format instead of extension then don't download file
|
||||
- _vec_
|
||||
- add native lock file support (once it's available in stable rust)
|
||||
- improve compressed mode (slow reads)
|
||||
- add ema support
|
||||
- __docs__
|
||||
- _README_
|
||||
- add a comparison table with alternatives
|
||||
- add contribution section where help is needed
|
||||
- documentation/mcp/datasets/different front ends
|
||||
- add faq
|
||||
- __websites__
|
||||
- _default_
|
||||
- explorer
|
||||
- blocks
|
||||
- transactions
|
||||
- addresses
|
||||
- miners
|
||||
- maybe xpubs
|
||||
- charts
|
||||
- improve some names and colors
|
||||
- remove `sum` series when it's a duplicate of the `base` (in subsidy for example)
|
||||
- selected unit sometimes changes when going back end forth
|
||||
- add support for custom charts
|
||||
- price scale format depends on unit, hide digits for sats for example (if/when possible)
|
||||
- table
|
||||
- pagination
|
||||
- exports (.json, .csv,…)
|
||||
- search
|
||||
- datasets add legend, and keywords ?
|
||||
- height/address/txid
|
||||
- api
|
||||
- add api page with interactivity
|
||||
- global
|
||||
- **fix navigation/history**
|
||||
- move share button to footer ?
|
||||
- Use `ichart.createPane()` in wrapper
|
||||
- improve behavior when local storage is unavailable
|
||||
- by having a global state
|
||||
- __global__
|
||||
- check `TODO`s in codebase
|
||||
|
Before Width: | Height: | Size: 340 KiB |
|
Before Width: | Height: | Size: 94 KiB |
|
Before Width: | Height: | Size: 78 KiB |
|
Before Width: | Height: | Size: 133 KiB |
|
Before Width: | Height: | Size: 263 KiB |
|
Before Width: | Height: | Size: 208 KiB |
|
Before Width: | Height: | Size: 386 KiB |
|
Before Width: | Height: | Size: 496 KiB |
|
Before Width: | Height: | Size: 564 KiB |
|
Before Width: | Height: | Size: 592 KiB |
|
Before Width: | Height: | Size: 453 KiB |
|
Before Width: | Height: | Size: 526 KiB |
@@ -1,2 +0,0 @@
|
||||
profile.json.gz
|
||||
flamegraph.svg
|
||||
@@ -2,57 +2,77 @@
|
||||
name = "brk"
|
||||
description.workspace = true
|
||||
license.workspace = true
|
||||
readme.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
edition.workspace = true
|
||||
version.workspace = true
|
||||
build = "build.rs"
|
||||
|
||||
[features]
|
||||
full = [
|
||||
"bencher",
|
||||
"binder",
|
||||
"bundler",
|
||||
"core",
|
||||
"client",
|
||||
"computer",
|
||||
"exit",
|
||||
"error",
|
||||
"fetcher",
|
||||
"cohort",
|
||||
"indexer",
|
||||
"iterator",
|
||||
"logger",
|
||||
"mcp",
|
||||
"parser",
|
||||
"interface",
|
||||
"mempool",
|
||||
"query",
|
||||
"reader",
|
||||
"rpc",
|
||||
"server",
|
||||
"store",
|
||||
"vec",
|
||||
"traversable",
|
||||
"types",
|
||||
]
|
||||
bencher = ["brk_bencher"]
|
||||
binder = ["brk_bindgen"]
|
||||
bundler = ["brk_bundler"]
|
||||
core = ["brk_core"]
|
||||
client = ["brk_client"]
|
||||
computer = ["brk_computer"]
|
||||
exit = ["brk_exit"]
|
||||
error = ["brk_error"]
|
||||
fetcher = ["brk_fetcher"]
|
||||
cohort = ["brk_cohort"]
|
||||
indexer = ["brk_indexer"]
|
||||
iterator = ["brk_iterator"]
|
||||
logger = ["brk_logger"]
|
||||
mcp = ["brk_mcp"]
|
||||
parser = ["brk_parser"]
|
||||
interface = ["brk_interface"]
|
||||
mempool = ["brk_mempool"]
|
||||
query = ["brk_query"]
|
||||
reader = ["brk_reader"]
|
||||
rpc = ["brk_rpc"]
|
||||
server = ["brk_server"]
|
||||
store = ["brk_store"]
|
||||
vec = ["brk_vec"]
|
||||
traversable = ["brk_traversable"]
|
||||
types = ["brk_types"]
|
||||
|
||||
[dependencies]
|
||||
brk_bencher = { workspace = true, optional = true }
|
||||
brk_bindgen = { workspace = true, optional = true }
|
||||
brk_bundler = { workspace = true, optional = true }
|
||||
brk_cli = { workspace = true }
|
||||
brk_core = { workspace = true, optional = true }
|
||||
brk_client = { workspace = true, optional = true }
|
||||
brk_computer = { workspace = true, optional = true }
|
||||
brk_exit = { workspace = true, optional = true }
|
||||
brk_error = { workspace = true, optional = true }
|
||||
brk_fetcher = { workspace = true, optional = true }
|
||||
brk_cohort = { workspace = true, optional = true }
|
||||
brk_indexer = { workspace = true, optional = true }
|
||||
brk_iterator = { workspace = true, optional = true }
|
||||
brk_logger = { workspace = true, optional = true }
|
||||
brk_mcp = { workspace = true, optional = true }
|
||||
brk_parser = { workspace = true, optional = true }
|
||||
brk_interface = { workspace = true, optional = true }
|
||||
brk_mempool = { workspace = true, optional = true }
|
||||
brk_query = { workspace = true, optional = true }
|
||||
brk_reader = { workspace = true, optional = true }
|
||||
brk_rpc = { workspace = true, optional = true }
|
||||
brk_server = { workspace = true, optional = true }
|
||||
brk_store = { workspace = true, optional = true }
|
||||
brk_vec = { workspace = true, optional = true }
|
||||
brk_traversable = { workspace = true, optional = true }
|
||||
brk_types = { workspace = true, optional = true }
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
all-features = true
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# brk
|
||||
|
||||
Umbrella crate for the Bitcoin Research Kit.
|
||||
|
||||
## What It Enables
|
||||
|
||||
Single dependency to access any BRK component. Enable only what you need via feature flags.
|
||||
|
||||
## Usage
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
brk = { version = "0.x", features = ["query", "types"] }
|
||||
```
|
||||
|
||||
```rust,ignore
|
||||
use brk::query::Query;
|
||||
use brk::types::Height;
|
||||
```
|
||||
|
||||
## Feature Flags
|
||||
|
||||
| Feature | Crate | Description |
|
||||
|---------|-------|-------------|
|
||||
| `bencher` | `brk_bencher` | Benchmarking utilities |
|
||||
| `binder` | `brk_binder` | Client code generation |
|
||||
| `bundler` | `brk_bundler` | JS bundling |
|
||||
| `client` | `brk_client` | Generated Rust API client |
|
||||
| `computer` | `brk_computer` | Metric computation |
|
||||
| `error` | `brk_error` | Error types |
|
||||
| `fetcher` | `brk_fetcher` | Price data fetching |
|
||||
| `cohort` | `brk_cohort` | Cohort filtering |
|
||||
| `indexer` | `brk_indexer` | Blockchain indexing |
|
||||
| `iterator` | `brk_iterator` | Block iteration |
|
||||
| `logger` | `brk_logger` | Logging setup |
|
||||
| `mcp` | `brk_mcp` | MCP server |
|
||||
| `mempool` | `brk_mempool` | Mempool monitoring |
|
||||
| `query` | `brk_query` | Query interface |
|
||||
| `reader` | `brk_reader` | Raw block reading |
|
||||
| `rpc` | `brk_rpc` | Bitcoin RPC client |
|
||||
| `server` | `brk_server` | HTTP API server |
|
||||
| `store` | `brk_store` | Key-value storage |
|
||||
| `traversable` | `brk_traversable` | Data traversal |
|
||||
| `types` | `brk_types` | Domain types |
|
||||
|
||||
Use `full` to enable all features.
|
||||
@@ -0,0 +1,8 @@
|
||||
fn main() {
|
||||
let profile = std::env::var("PROFILE").unwrap_or_default();
|
||||
|
||||
if profile == "release" {
|
||||
println!("cargo:rustc-flag=-C");
|
||||
println!("cargo:rustc-flag=target-cpu=native");
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
fn main() {}
|
||||
@@ -1,2 +1 @@
|
||||
cargo build --profile profiling
|
||||
flamegraph -- ../../target/profiling/brk
|
||||
sudo cargo flamegraph --profile profiling --root
|
||||
|
||||
@@ -1,23 +1,32 @@
|
||||
#![doc = include_str!(concat!("../", env!("CARGO_PKG_README")))]
|
||||
#![doc = include_str!("../README.md")]
|
||||
|
||||
#[cfg(feature = "bencher")]
|
||||
#[doc(inline)]
|
||||
pub use brk_bencher as bencher;
|
||||
|
||||
#[cfg(feature = "binder")]
|
||||
#[doc(inline)]
|
||||
pub use brk_bindgen as binder;
|
||||
|
||||
#[cfg(feature = "bundler")]
|
||||
#[doc(inline)]
|
||||
pub use brk_bundler as bundler;
|
||||
|
||||
#[cfg(feature = "client")]
|
||||
#[doc(inline)]
|
||||
pub use brk_cli as cli;
|
||||
pub use brk_client as client;
|
||||
|
||||
#[cfg(feature = "core")]
|
||||
#[cfg(feature = "cohort")]
|
||||
#[doc(inline)]
|
||||
pub use brk_core as core;
|
||||
pub use brk_cohort as cohort;
|
||||
|
||||
#[cfg(feature = "computer")]
|
||||
#[doc(inline)]
|
||||
pub use brk_computer as computer;
|
||||
|
||||
#[cfg(feature = "exit")]
|
||||
#[cfg(feature = "error")]
|
||||
#[doc(inline)]
|
||||
pub use brk_exit as exit;
|
||||
pub use brk_error as error;
|
||||
|
||||
#[cfg(feature = "fetcher")]
|
||||
#[doc(inline)]
|
||||
@@ -27,6 +36,10 @@ pub use brk_fetcher as fetcher;
|
||||
#[doc(inline)]
|
||||
pub use brk_indexer as indexer;
|
||||
|
||||
#[cfg(feature = "iterator")]
|
||||
#[doc(inline)]
|
||||
pub use brk_iterator as iterator;
|
||||
|
||||
#[cfg(feature = "logger")]
|
||||
#[doc(inline)]
|
||||
pub use brk_logger as logger;
|
||||
@@ -35,13 +48,21 @@ pub use brk_logger as logger;
|
||||
#[doc(inline)]
|
||||
pub use brk_mcp as mcp;
|
||||
|
||||
#[cfg(feature = "parser")]
|
||||
#[cfg(feature = "mempool")]
|
||||
#[doc(inline)]
|
||||
pub use brk_parser as parser;
|
||||
pub use brk_mempool as mempool;
|
||||
|
||||
#[cfg(feature = "interface")]
|
||||
#[cfg(feature = "query")]
|
||||
#[doc(inline)]
|
||||
pub use brk_interface as interface;
|
||||
pub use brk_query as query;
|
||||
|
||||
#[cfg(feature = "reader")]
|
||||
#[doc(inline)]
|
||||
pub use brk_reader as reader;
|
||||
|
||||
#[cfg(feature = "rpc")]
|
||||
#[doc(inline)]
|
||||
pub use brk_rpc as rpc;
|
||||
|
||||
#[cfg(feature = "server")]
|
||||
#[doc(inline)]
|
||||
@@ -51,6 +72,10 @@ pub use brk_server as server;
|
||||
#[doc(inline)]
|
||||
pub use brk_store as store;
|
||||
|
||||
#[cfg(feature = "vec")]
|
||||
#[cfg(feature = "traversable")]
|
||||
#[doc(inline)]
|
||||
pub use brk_vec as vec;
|
||||
pub use brk_traversable as traversable;
|
||||
|
||||
#[cfg(feature = "types")]
|
||||
#[doc(inline)]
|
||||
pub use brk_types as types;
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
use brk_cli::main;
|
||||
@@ -0,0 +1,13 @@
|
||||
[package]
|
||||
name = "brk_alloc"
|
||||
description = "Global allocator and memory utilities for brk"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
libmimalloc-sys = { version = "0.1.44", features = ["extended"] }
|
||||
log = { workspace = true }
|
||||
mimalloc = { version = "0.1.48", features = ["v3"] }
|
||||
@@ -0,0 +1,21 @@
|
||||
//! Global allocator and memory utilities for brk.
|
||||
//!
|
||||
//! This crate sets mimalloc as the global allocator and provides
|
||||
//! utilities for monitoring and managing memory.
|
||||
|
||||
use mimalloc::MiMalloc as Allocator;
|
||||
|
||||
#[global_allocator]
|
||||
static GLOBAL: Allocator = Allocator;
|
||||
|
||||
/// Mimalloc allocator utilities
|
||||
pub struct Mimalloc;
|
||||
|
||||
impl Mimalloc {
|
||||
/// Eagerly free memory back to OS.
|
||||
/// Only call at natural pause points.
|
||||
#[inline]
|
||||
pub fn collect() {
|
||||
unsafe { libmimalloc_sys::mi_collect(true) }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
[package]
|
||||
name = "brk_bencher"
|
||||
description = "A simple benchmarker for testing other crates."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
build = "build.rs"
|
||||
|
||||
[dependencies]
|
||||
brk_error = { workspace = true }
|
||||
brk_logger = { workspace = true }
|
||||
parking_lot = { workspace = true }
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
libproc = "0.14"
|
||||
@@ -0,0 +1,43 @@
|
||||
# brk_bencher
|
||||
|
||||
Resource monitoring for long-running Bitcoin indexing operations.
|
||||
|
||||
## What It Enables
|
||||
|
||||
Track disk usage, memory consumption (current + peak), and I/O throughput during indexing runs. Progress tracking hooks into brk_logger to record processing milestones automatically.
|
||||
|
||||
## Key Features
|
||||
|
||||
- **Multi-metric monitoring**: Disk, memory (RSS + peak), I/O read/write
|
||||
- **Progress tracking**: Integrates with logging to capture block heights as they're processed
|
||||
- **Run comparison**: Outputs timestamped CSVs for comparing multiple runs
|
||||
- **macOS optimized**: Uses libproc for accurate process metrics on macOS
|
||||
- **Non-blocking**: Monitors in background thread with 5-second sample interval
|
||||
|
||||
## Core API
|
||||
|
||||
```rust,ignore
|
||||
let mut bencher = Bencher::from_cargo_env("brk_indexer", &data_path)?;
|
||||
bencher.start()?;
|
||||
|
||||
// ... run indexing ...
|
||||
|
||||
bencher.stop()?;
|
||||
```
|
||||
|
||||
## Output Structure
|
||||
|
||||
```
|
||||
benches/
|
||||
└── brk_indexer/
|
||||
└── 1703001234/
|
||||
├── disk.csv # timestamp_ms, bytes
|
||||
├── memory.csv # timestamp_ms, current, peak
|
||||
├── io.csv # timestamp_ms, read, written
|
||||
└── progress.csv # timestamp_ms, height
|
||||
```
|
||||
|
||||
## Built On
|
||||
|
||||
- `brk_error` for error handling
|
||||
- `brk_logger` for progress hook integration
|
||||
@@ -0,0 +1,8 @@
|
||||
fn main() {
|
||||
let profile = std::env::var("PROFILE").unwrap_or_default();
|
||||
|
||||
if profile == "release" {
|
||||
println!("cargo:rustc-flag=-C");
|
||||
println!("cargo:rustc-flag=target-cpu=native");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs::{self, File},
|
||||
io::{self, Write},
|
||||
os::unix::fs::MetadataExt,
|
||||
path::{Path, PathBuf},
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
pub struct DiskMonitor {
|
||||
cache: HashMap<PathBuf, (u64, SystemTime)>, // path -> (bytes_used, mtime)
|
||||
monitored_path: PathBuf,
|
||||
writer: File,
|
||||
}
|
||||
|
||||
impl DiskMonitor {
|
||||
pub fn new(monitored_path: &Path, csv_path: &Path) -> io::Result<Self> {
|
||||
let mut writer = File::create(csv_path)?;
|
||||
writeln!(writer, "timestamp_ms,disk_usage")?;
|
||||
|
||||
Ok(Self {
|
||||
cache: HashMap::new(),
|
||||
monitored_path: monitored_path.to_path_buf(),
|
||||
writer,
|
||||
})
|
||||
}
|
||||
|
||||
/// Record disk usage at the given timestamp
|
||||
pub fn record(&mut self, elapsed_ms: u128) -> io::Result<()> {
|
||||
if let Ok(bytes) = self.scan_recursive(&self.monitored_path.clone()) {
|
||||
writeln!(self.writer, "{},{}", elapsed_ms, bytes)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn scan_recursive(&mut self, path: &Path) -> io::Result<u64> {
|
||||
let mut total = 0;
|
||||
|
||||
for entry in fs::read_dir(path)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
let metadata = entry.metadata()?;
|
||||
|
||||
if metadata.is_file() {
|
||||
let mtime = metadata.modified()?;
|
||||
|
||||
// Check cache: if mtime unchanged, use cached value
|
||||
if let Some((cached_bytes, cached_mtime)) = self.cache.get(&path)
|
||||
&& *cached_mtime == mtime
|
||||
{
|
||||
total += cached_bytes;
|
||||
continue;
|
||||
}
|
||||
|
||||
// File is new or modified - get actual disk usage
|
||||
let bytes = metadata.blocks() * 512;
|
||||
self.cache.insert(path, (bytes, mtime));
|
||||
total += bytes;
|
||||
} else if metadata.is_dir() {
|
||||
total += self.scan_recursive(&path)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(total)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
use std::{
|
||||
fs::File,
|
||||
io::{self, Write},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use std::fs;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use libproc::pid_rusage::{pidrusage, RUsageInfoV2};
|
||||
|
||||
pub struct IoMonitor {
|
||||
pid: u32,
|
||||
writer: File,
|
||||
}
|
||||
|
||||
impl IoMonitor {
|
||||
pub fn new(pid: u32, csv_path: &Path) -> io::Result<Self> {
|
||||
let mut writer = File::create(csv_path)?;
|
||||
writeln!(writer, "timestamp_ms,bytes_read,bytes_written")?;
|
||||
|
||||
Ok(Self { pid, writer })
|
||||
}
|
||||
|
||||
/// Record I/O usage at the given timestamp
|
||||
pub fn record(&mut self, elapsed_ms: u128) -> io::Result<()> {
|
||||
if let Ok((read, written)) = self.get_io_usage() {
|
||||
writeln!(self.writer, "{},{},{}", elapsed_ms, read, written)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get I/O usage in bytes
|
||||
/// Returns (bytes_read, bytes_written)
|
||||
fn get_io_usage(&self) -> io::Result<(u64, u64)> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
self.get_io_usage_linux()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
self.get_io_usage_macos()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn get_io_usage_linux(&self) -> io::Result<(u64, u64)> {
|
||||
let io_content = fs::read_to_string(format!("/proc/{}/io", self.pid))?;
|
||||
|
||||
let mut read_bytes = None;
|
||||
let mut write_bytes = None;
|
||||
|
||||
for line in io_content.lines() {
|
||||
if line.starts_with("read_bytes:") {
|
||||
if let Some(value_str) = line.split_whitespace().nth(1) {
|
||||
read_bytes = value_str.parse::<u64>().ok();
|
||||
}
|
||||
} else if line.starts_with("write_bytes:") {
|
||||
if let Some(value_str) = line.split_whitespace().nth(1) {
|
||||
write_bytes = value_str.parse::<u64>().ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match (read_bytes, write_bytes) {
|
||||
(Some(r), Some(w)) => Ok((r, w)),
|
||||
_ => Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"Failed to parse I/O stats from /proc/[pid]/io",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn get_io_usage_macos(&self) -> io::Result<(u64, u64)> {
|
||||
match pidrusage::<RUsageInfoV2>(self.pid as i32) {
|
||||
Ok(info) => Ok((info.ri_diskio_bytesread, info.ri_diskio_byteswritten)),
|
||||
Err(_) => Err(io::Error::other("Failed to get process I/O stats")),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
thread::{self, JoinHandle},
|
||||
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use brk_error::{Error, Result};
|
||||
|
||||
mod disk;
|
||||
mod io;
|
||||
mod memory;
|
||||
mod progression;
|
||||
|
||||
use disk::*;
|
||||
use io::*;
|
||||
use memory::*;
|
||||
use parking_lot::Mutex;
|
||||
use progression::*;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Bencher(Arc<BencherInner>);
|
||||
|
||||
struct BencherInner {
|
||||
bench_dir: PathBuf,
|
||||
monitored_path: PathBuf,
|
||||
stop_flag: Arc<AtomicBool>,
|
||||
monitor_thread: Mutex<Option<JoinHandle<Result<()>>>>,
|
||||
progression: Arc<ProgressionMonitor>,
|
||||
}
|
||||
|
||||
impl Bencher {
|
||||
/// Create a new bencher for the given crate name
|
||||
/// Creates directory structure: workspace_root/benches/{crate_name}/{timestamp}/
|
||||
pub fn new(crate_name: &str, workspace_root: &Path, monitored_path: &Path) -> Result<Self> {
|
||||
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
|
||||
|
||||
let bench_dir = workspace_root
|
||||
.join("benches")
|
||||
.join(crate_name)
|
||||
.join(timestamp.to_string());
|
||||
|
||||
fs::create_dir_all(&bench_dir)?;
|
||||
|
||||
let progress_csv = bench_dir.join("progress.csv");
|
||||
let progression = Arc::new(ProgressionMonitor::new(&progress_csv)?);
|
||||
let progression_clone = progression.clone();
|
||||
|
||||
// Register hook with logger
|
||||
brk_logger::register_hook(move |message| {
|
||||
progression_clone.check_and_record(message);
|
||||
})
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::AlreadyExists, e))?;
|
||||
|
||||
Ok(Self(Arc::new(BencherInner {
|
||||
bench_dir,
|
||||
monitored_path: monitored_path.to_path_buf(),
|
||||
stop_flag: Arc::new(AtomicBool::new(false)),
|
||||
progression,
|
||||
monitor_thread: Mutex::new(None),
|
||||
})))
|
||||
}
|
||||
|
||||
/// Create a bencher using CARGO_MANIFEST_DIR to find workspace root
|
||||
pub fn from_cargo_env(crate_name: &str, monitored_path: &Path) -> Result<Self> {
|
||||
let mut current = std::env::current_dir()
|
||||
.map_err(|e| format!("Failed to get current directory: {}", e))
|
||||
.unwrap();
|
||||
|
||||
let workspace_root = loop {
|
||||
let cargo_toml = current.join("Cargo.toml");
|
||||
if cargo_toml.exists() {
|
||||
let contents = std::fs::read_to_string(&cargo_toml)
|
||||
.map_err(|e| format!("Failed to read Cargo.toml: {}", e))
|
||||
.unwrap();
|
||||
if contents.contains("[workspace]") {
|
||||
break current;
|
||||
}
|
||||
}
|
||||
|
||||
current = current
|
||||
.parent()
|
||||
.ok_or(Error::NotFound("Workspace root not found".into()))?
|
||||
.to_path_buf();
|
||||
};
|
||||
|
||||
Self::new(crate_name, &workspace_root, monitored_path)
|
||||
}
|
||||
|
||||
/// Start monitoring disk usage and memory footprint
|
||||
pub fn start(&mut self) -> Result<()> {
|
||||
if self.0.monitor_thread.lock().is_some() {
|
||||
return Err(Error::Internal("Bencher already started"));
|
||||
}
|
||||
|
||||
let stop_flag = self.0.stop_flag.clone();
|
||||
let bench_dir = self.0.bench_dir.clone();
|
||||
let monitored_path = self.0.monitored_path.clone();
|
||||
|
||||
let handle =
|
||||
thread::spawn(move || monitor_resources(&monitored_path, &bench_dir, stop_flag));
|
||||
|
||||
*self.0.monitor_thread.lock() = Some(handle);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop monitoring and wait for the thread to finish
|
||||
pub fn stop(&self) -> Result<()> {
|
||||
self.0.stop_flag.store(true, Ordering::Relaxed);
|
||||
|
||||
if let Some(handle) = self.0.monitor_thread.lock().take() {
|
||||
handle.join().map_err(|_| Error::Internal("Monitor thread panicked"))??;
|
||||
}
|
||||
|
||||
self.0.progression.flush()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Bencher {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.stop();
|
||||
}
|
||||
}
|
||||
|
||||
fn monitor_resources(
|
||||
monitored_path: &Path,
|
||||
bench_dir: &Path,
|
||||
stop_flag: Arc<AtomicBool>,
|
||||
) -> Result<()> {
|
||||
let pid = std::process::id();
|
||||
let start = Instant::now();
|
||||
|
||||
let mut disk_monitor = DiskMonitor::new(monitored_path, &bench_dir.join("disk.csv"))?;
|
||||
let mut memory_monitor = MemoryMonitor::new(pid, &bench_dir.join("memory.csv"))?;
|
||||
let mut io_monitor = IoMonitor::new(pid, &bench_dir.join("io.csv"))?;
|
||||
|
||||
'l: loop {
|
||||
let elapsed_ms = start.elapsed().as_millis();
|
||||
|
||||
disk_monitor.record(elapsed_ms)?;
|
||||
memory_monitor.record(elapsed_ms)?;
|
||||
io_monitor.record(elapsed_ms)?;
|
||||
|
||||
for _ in 0..50 {
|
||||
// 50 * 100ms = 5 seconds
|
||||
if stop_flag.load(Ordering::Relaxed) {
|
||||
break 'l;
|
||||
}
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
use std::{
|
||||
fs::File,
|
||||
io::{self, Write},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use std::fs;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use std::process::Command;
|
||||
|
||||
pub struct MemoryMonitor {
|
||||
pid: u32,
|
||||
writer: File,
|
||||
}
|
||||
|
||||
impl MemoryMonitor {
|
||||
pub fn new(pid: u32, csv_path: &Path) -> io::Result<Self> {
|
||||
let mut writer = File::create(csv_path)?;
|
||||
writeln!(writer, "timestamp_ms,phys_footprint,phys_footprint_peak")?;
|
||||
|
||||
Ok(Self { pid, writer })
|
||||
}
|
||||
|
||||
/// Record memory usage at the given timestamp
|
||||
pub fn record(&mut self, elapsed_ms: u128) -> io::Result<()> {
|
||||
if let Ok((footprint, peak)) = self.get_memory_usage() {
|
||||
writeln!(self.writer, "{},{},{}", elapsed_ms, footprint, peak)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get memory usage in bytes
|
||||
/// Returns (current_bytes, peak_bytes)
|
||||
fn get_memory_usage(&self) -> io::Result<(u64, u64)> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
self.get_memory_usage_linux()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
self.get_memory_usage_macos()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn get_memory_usage_linux(&self) -> io::Result<(u64, u64)> {
|
||||
let status_content = fs::read_to_string(format!("/proc/{}/status", self.pid))?;
|
||||
|
||||
let mut vm_rss = None;
|
||||
let mut vm_hwm = None;
|
||||
|
||||
for line in status_content.lines() {
|
||||
if line.starts_with("VmRSS:") {
|
||||
if let Some(value_str) = line.split_whitespace().nth(1) {
|
||||
if let Ok(kb) = value_str.parse::<u64>() {
|
||||
vm_rss = Some(kb * 1024); // KiB to bytes
|
||||
}
|
||||
}
|
||||
} else if line.starts_with("VmHWM:") {
|
||||
if let Some(value_str) = line.split_whitespace().nth(1) {
|
||||
if let Ok(kb) = value_str.parse::<u64>() {
|
||||
vm_hwm = Some(kb * 1024); // KiB to bytes
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match (vm_rss, vm_hwm) {
|
||||
(Some(rss), Some(hwm)) => Ok((rss, hwm)),
|
||||
_ => Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"Failed to parse memory info from /proc/[pid]/status",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn get_memory_usage_macos(&self) -> io::Result<(u64, u64)> {
|
||||
let output = Command::new("footprint")
|
||||
.args(["-p", &self.pid.to_string()])
|
||||
.output()?;
|
||||
|
||||
let stdout = String::from_utf8(output.stdout).map_err(|_| {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "Invalid UTF-8 from footprint")
|
||||
})?;
|
||||
|
||||
parse_footprint_output(&stdout).ok_or_else(|| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"Failed to parse footprint output",
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn parse_footprint_output(output: &str) -> Option<(u64, u64)> {
|
||||
let mut phys_footprint = None;
|
||||
let mut phys_footprint_peak = None;
|
||||
|
||||
for line in output.lines() {
|
||||
let line = line.trim();
|
||||
|
||||
if line.starts_with("phys_footprint:") {
|
||||
// Format: "phys_footprint: 7072 KB"
|
||||
let parts: Vec<&str> = line.split_whitespace().collect();
|
||||
if parts.len() >= 3 {
|
||||
// parts[0] = "phys_footprint:"
|
||||
// parts[1] = "7072"
|
||||
// parts[2] = "KB"
|
||||
phys_footprint = parse_size_to_bytes(parts[1], parts[2]);
|
||||
}
|
||||
} else if line.starts_with("phys_footprint_peak:") {
|
||||
// Format: "phys_footprint_peak: 15 MB"
|
||||
let parts: Vec<&str> = line.split_whitespace().collect();
|
||||
if parts.len() >= 3 {
|
||||
phys_footprint_peak = parse_size_to_bytes(parts[1], parts[2]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match (phys_footprint, phys_footprint_peak) {
|
||||
(Some(f), Some(p)) => Some((f, p)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn parse_size_to_bytes(value: &str, unit: &str) -> Option<u64> {
|
||||
let value: f64 = value.parse().ok()?;
|
||||
|
||||
let multiplier = match unit.to_uppercase().as_str() {
|
||||
"KB" => 1024.0, // KiB to bytes
|
||||
"MB" => 1024.0 * 1024.0, // MiB to bytes
|
||||
"GB" => 1024.0 * 1024.0 * 1024.0, // GiB to bytes
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
Some((value * multiplier) as u64)
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
use parking_lot::Mutex;
|
||||
use std::{
|
||||
fs,
|
||||
io::{self, BufWriter, Write},
|
||||
path::Path,
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
/// Patterns to match for progress tracking.
|
||||
const PROGRESS_PATTERNS: &[&str] = &[
|
||||
"block ", // "Indexing block 123..."
|
||||
"chain at ", // "Processing chain at 456..."
|
||||
];
|
||||
|
||||
pub struct ProgressionMonitor {
|
||||
csv_file: Mutex<BufWriter<fs::File>>,
|
||||
start_time: Instant,
|
||||
}
|
||||
|
||||
impl ProgressionMonitor {
|
||||
pub fn new(csv_path: &Path) -> io::Result<Self> {
|
||||
let mut csv_file = BufWriter::new(fs::File::create(csv_path)?);
|
||||
writeln!(csv_file, "timestamp_ms,value")?;
|
||||
|
||||
Ok(Self {
|
||||
csv_file: Mutex::new(csv_file),
|
||||
start_time: Instant::now(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Check message for progress patterns and record if found
|
||||
#[inline]
|
||||
pub fn check_and_record(&self, message: &str) {
|
||||
let Some(value) = parse_progress(message) else {
|
||||
return;
|
||||
};
|
||||
|
||||
if value % 10 != 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let elapsed_ms = self.start_time.elapsed().as_millis();
|
||||
let _ = writeln!(self.csv_file.lock(), "{},{}", elapsed_ms, value);
|
||||
}
|
||||
|
||||
pub fn flush(&self) -> io::Result<()> {
|
||||
self.csv_file.lock().flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse progress value from message
|
||||
#[inline]
|
||||
fn parse_progress(message: &str) -> Option<u64> {
|
||||
PROGRESS_PATTERNS
|
||||
.iter()
|
||||
.find_map(|pattern| parse_number_after(message, pattern))
|
||||
}
|
||||
|
||||
/// Extract number immediately following the pattern
|
||||
#[inline]
|
||||
fn parse_number_after(message: &str, pattern: &str) -> Option<u64> {
|
||||
let start = message.find(pattern)?;
|
||||
let after = &message[start + pattern.len()..];
|
||||
|
||||
let end = after
|
||||
.find(|c: char| !c.is_ascii_digit())
|
||||
.unwrap_or(after.len());
|
||||
|
||||
if end == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
after[..end].parse().ok()
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
[package]
|
||||
name = "brk_bencher_visualizer"
|
||||
description = "A generator of charts for brk_bencher"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
publish = false
|
||||
build = "build.rs"
|
||||
|
||||
[dependencies]
|
||||
plotters = "0.3.7"
|
||||
@@ -0,0 +1,34 @@
|
||||
# brk_bencher_visualizer
|
||||
|
||||
SVG chart generation for benchmark visualization.
|
||||
|
||||
## What It Enables
|
||||
|
||||
Turn benchmark CSV data into publication-ready SVG charts showing disk usage, memory (current/peak), progress, and I/O over time. Compare multiple runs side-by-side with automatic color coding.
|
||||
|
||||
## Key Features
|
||||
|
||||
- **Multi-run comparison**: Overlay multiple benchmark runs with distinct colors
|
||||
- **Dual-axis charts**: Memory charts show both current and peak usage (solid vs dashed lines)
|
||||
- **Smart scaling**: Automatic unit conversion for bytes (KB/MB/GB) and time (seconds/minutes/hours)
|
||||
- **Per-run trimming**: Aligns data by progress cutoffs for fair comparison
|
||||
- **Dark theme**: Clean, readable charts with monospace fonts
|
||||
|
||||
## Core API
|
||||
|
||||
```rust,ignore
|
||||
let viz = Visualizer::from_cargo_env()?;
|
||||
viz.generate_all_charts()?; // Process all crates in benches/
|
||||
```
|
||||
|
||||
## Chart Types
|
||||
|
||||
- `disk.svg` - Storage consumption over time
|
||||
- `memory.svg` - Current + peak memory usage
|
||||
- `progress.svg` - Processing progress (e.g., blocks indexed)
|
||||
- `io_read.svg` / `io_write.svg` - I/O throughput
|
||||
|
||||
## Input Format
|
||||
|
||||
Reads CSV files from `benches/<crate>/<run_id>/`:
|
||||
- `disk.csv`, `memory.csv`, `progress.csv`, `io.csv`
|
||||
@@ -0,0 +1,8 @@
|
||||
fn main() {
|
||||
let profile = std::env::var("PROFILE").unwrap_or_default();
|
||||
|
||||
if profile == "release" {
|
||||
println!("cargo:rustc-flag=-C");
|
||||
println!("cargo:rustc-flag=target-cpu=native");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
use crate::data::{DataPoint, DualRun, Result, Run};
|
||||
use crate::format;
|
||||
use plotters::prelude::*;
|
||||
use std::path::Path;
|
||||
|
||||
const FONT: &str = "monospace";
|
||||
const FONT_SIZE: i32 = 20;
|
||||
const FONT_SIZE_BIG: i32 = 30;
|
||||
const SIZE: (u32, u32) = (2000, 1000);
|
||||
const TIME_BUFFER_MS: u64 = 10_000;
|
||||
|
||||
const BG_COLOR: RGBColor = RGBColor(18, 18, 24);
|
||||
const TEXT_COLOR: RGBColor = RGBColor(230, 230, 240);
|
||||
const COLORS: [RGBColor; 6] = [
|
||||
RGBColor(255, 99, 132), // Pink/Red
|
||||
RGBColor(54, 162, 235), // Blue
|
||||
RGBColor(75, 192, 192), // Teal
|
||||
RGBColor(255, 206, 86), // Yellow
|
||||
RGBColor(153, 102, 255), // Purple
|
||||
RGBColor(255, 159, 64), // Orange
|
||||
];
|
||||
|
||||
pub enum YAxisFormat {
|
||||
Bytes,
|
||||
Number,
|
||||
}
|
||||
|
||||
pub struct ChartConfig<'a> {
|
||||
pub output_path: &'a Path,
|
||||
pub title: String,
|
||||
pub y_label: String,
|
||||
pub y_format: YAxisFormat,
|
||||
}
|
||||
|
||||
/// Generate a simple line chart from runs
|
||||
pub fn generate(config: ChartConfig, runs: &[Run]) -> Result<()> {
|
||||
if runs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let max_time_ms = runs.iter().map(|r| r.max_timestamp()).max().unwrap_or(1000) + TIME_BUFFER_MS;
|
||||
let max_time_s = max_time_ms as f64 / 1000.0;
|
||||
let max_value = runs.iter().map(|r| r.max_value()).fold(0.0, f64::max);
|
||||
|
||||
let (time_scaled, time_divisor, time_label) = format::time(max_time_s);
|
||||
let (value_scaled, scale_factor, y_label) = scale_y_axis(max_value, &config.y_label, &config.y_format);
|
||||
let x_labels = label_count(time_scaled);
|
||||
|
||||
let root = SVGBackend::new(config.output_path, SIZE).into_drawing_area();
|
||||
root.fill(&BG_COLOR)?;
|
||||
|
||||
let mut chart = ChartBuilder::on(&root)
|
||||
.caption(&config.title, (FONT, FONT_SIZE_BIG).into_font().color(&TEXT_COLOR))
|
||||
.margin(20)
|
||||
.margin_right(40)
|
||||
.x_label_area_size(50)
|
||||
.margin_left(50)
|
||||
.right_y_label_area_size(75)
|
||||
.build_cartesian_2d(0.0..time_scaled * 1.025, 0.0..value_scaled * 1.1)?;
|
||||
|
||||
configure_mesh(&mut chart, time_label, &y_label, &config.y_format, x_labels)?;
|
||||
|
||||
for (idx, run) in runs.iter().enumerate() {
|
||||
let color = COLORS[idx % COLORS.len()];
|
||||
draw_series(&mut chart, &run.data, &run.id, color, time_divisor, scale_factor)?;
|
||||
}
|
||||
|
||||
configure_legend(&mut chart)?;
|
||||
root.present()?;
|
||||
println!("Generated: {}", config.output_path.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Generate a chart with dual series per run (e.g., current + peak memory)
|
||||
pub fn generate_dual(
|
||||
config: ChartConfig,
|
||||
runs: &[DualRun],
|
||||
primary_suffix: &str,
|
||||
secondary_suffix: &str,
|
||||
) -> Result<()> {
|
||||
if runs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let max_time_ms = runs
|
||||
.iter()
|
||||
.flat_map(|r| r.primary.iter().chain(r.secondary.iter()))
|
||||
.map(|d| d.timestamp_ms)
|
||||
.max()
|
||||
.unwrap_or(1000)
|
||||
+ TIME_BUFFER_MS;
|
||||
let max_time_s = max_time_ms as f64 / 1000.0;
|
||||
let max_value = runs.iter().map(|r| r.max_value()).fold(0.0, f64::max);
|
||||
|
||||
let (time_scaled, time_divisor, time_label) = format::time(max_time_s);
|
||||
let (value_scaled, scale_factor, y_label) = scale_y_axis(max_value, &config.y_label, &config.y_format);
|
||||
let x_labels = label_count(time_scaled);
|
||||
|
||||
let root = SVGBackend::new(config.output_path, SIZE).into_drawing_area();
|
||||
root.fill(&BG_COLOR)?;
|
||||
|
||||
let mut chart = ChartBuilder::on(&root)
|
||||
.caption(&config.title, (FONT, FONT_SIZE_BIG).into_font().color(&TEXT_COLOR))
|
||||
.margin(20)
|
||||
.margin_right(40)
|
||||
.x_label_area_size(50)
|
||||
.margin_left(50)
|
||||
.right_y_label_area_size(75)
|
||||
.build_cartesian_2d(0.0..time_scaled * 1.025, 0.0..value_scaled * 1.1)?;
|
||||
|
||||
configure_mesh(&mut chart, time_label, &y_label, &config.y_format, x_labels)?;
|
||||
|
||||
for (idx, run) in runs.iter().enumerate() {
|
||||
let color = COLORS[idx % COLORS.len()];
|
||||
|
||||
// Primary series (solid)
|
||||
draw_series(
|
||||
&mut chart,
|
||||
&run.primary,
|
||||
&format!("{} {}", run.id, primary_suffix),
|
||||
color,
|
||||
time_divisor,
|
||||
scale_factor,
|
||||
)?;
|
||||
|
||||
// Secondary series (dashed)
|
||||
draw_dashed_series(
|
||||
&mut chart,
|
||||
&run.secondary,
|
||||
&format!("{} {}", run.id, secondary_suffix),
|
||||
color.mix(0.5),
|
||||
time_divisor,
|
||||
scale_factor,
|
||||
)?;
|
||||
}
|
||||
|
||||
configure_legend(&mut chart)?;
|
||||
root.present()?;
|
||||
println!("Generated: {}", config.output_path.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn scale_y_axis(max_value: f64, base_label: &str, y_format: &YAxisFormat) -> (f64, f64, String) {
|
||||
match y_format {
|
||||
YAxisFormat::Bytes => {
|
||||
let (scaled, unit) = format::bytes(max_value);
|
||||
let factor = max_value / scaled;
|
||||
(scaled, factor, format!("{} ({})", base_label, unit))
|
||||
}
|
||||
YAxisFormat::Number => (max_value, 1.0, base_label.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Calculate appropriate label count to avoid duplicates when rounding to integers
|
||||
fn label_count(max_value: f64) -> usize {
|
||||
let max_int = max_value.ceil() as usize;
|
||||
// Don't exceed the range, cap at 12 for readability
|
||||
max_int.clamp(2, 12)
|
||||
}
|
||||
|
||||
type Chart<'a, 'b> = ChartContext<
|
||||
'a,
|
||||
SVGBackend<'b>,
|
||||
Cartesian2d<plotters::coord::types::RangedCoordf64, plotters::coord::types::RangedCoordf64>,
|
||||
>;
|
||||
|
||||
fn configure_mesh(chart: &mut Chart, x_label: &str, y_label: &str, y_format: &YAxisFormat, x_labels: usize) -> Result<()> {
|
||||
let y_formatter: Box<dyn Fn(&f64) -> String> = match y_format {
|
||||
YAxisFormat::Bytes => Box::new(|y: &f64| {
|
||||
if y.fract() == 0.0 {
|
||||
format!("{:.0}", y)
|
||||
} else {
|
||||
format!("{:.1}", y)
|
||||
}
|
||||
}),
|
||||
YAxisFormat::Number => Box::new(|y: &f64| format::axis_number(*y)),
|
||||
};
|
||||
|
||||
chart
|
||||
.configure_mesh()
|
||||
.disable_mesh()
|
||||
.x_desc(x_label)
|
||||
.y_desc(y_label)
|
||||
.x_label_formatter(&|x| format!("{:.0}", x))
|
||||
.y_label_formatter(&y_formatter)
|
||||
.x_labels(x_labels)
|
||||
.y_labels(10)
|
||||
.x_label_style((FONT, FONT_SIZE).into_font().color(&TEXT_COLOR.mix(0.7)))
|
||||
.y_label_style((FONT, FONT_SIZE).into_font().color(&TEXT_COLOR.mix(0.7)))
|
||||
.axis_style(TEXT_COLOR.mix(0.3))
|
||||
.draw()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn draw_series(
|
||||
chart: &mut Chart,
|
||||
data: &[DataPoint],
|
||||
label: &str,
|
||||
color: RGBColor,
|
||||
time_divisor: f64,
|
||||
scale_factor: f64,
|
||||
) -> Result<()> {
|
||||
let points = data
|
||||
.iter()
|
||||
.map(|d| (d.timestamp_ms as f64 / 1000.0 / time_divisor, d.value / scale_factor));
|
||||
|
||||
chart
|
||||
.draw_series(LineSeries::new(points, color.stroke_width(1)))?
|
||||
.label(label)
|
||||
.legend(move |(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], color.stroke_width(1)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn draw_dashed_series(
|
||||
chart: &mut Chart,
|
||||
data: &[DataPoint],
|
||||
label: &str,
|
||||
color: RGBAColor,
|
||||
time_divisor: f64,
|
||||
scale_factor: f64,
|
||||
) -> Result<()> {
|
||||
let points: Vec<_> = data
|
||||
.iter()
|
||||
.map(|d| (d.timestamp_ms as f64 / 1000.0 / time_divisor, d.value / scale_factor))
|
||||
.collect();
|
||||
|
||||
// Draw dashed line by skipping every other segment
|
||||
chart
|
||||
.draw_series(
|
||||
points
|
||||
.windows(2)
|
||||
.enumerate()
|
||||
.filter(|(i, _)| i % 2 == 0)
|
||||
.map(|(_, w)| PathElement::new(vec![w[0], w[1]], color.stroke_width(2))),
|
||||
)?
|
||||
.label(label)
|
||||
.legend(move |(x, y)| PathElement::new(vec![(x, y), (x + 10, y), (x + 20, y)], color.stroke_width(2)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn configure_legend<'a>(chart: &mut Chart<'a, 'a>) -> Result<()> {
|
||||
chart
|
||||
.configure_series_labels()
|
||||
.position(SeriesLabelPosition::UpperLeft)
|
||||
.label_font((FONT, FONT_SIZE).into_font().color(&TEXT_COLOR.mix(0.9)))
|
||||
.background_style(BG_COLOR.mix(0.98))
|
||||
.border_style(BG_COLOR)
|
||||
.margin(10)
|
||||
.draw()?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
use std::{collections::HashMap, fs, path::Path};
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DataPoint {
|
||||
pub timestamp_ms: u64,
|
||||
pub value: f64,
|
||||
}
|
||||
|
||||
/// Per-run cutoff timestamps for fair comparison
|
||||
pub struct Cutoffs {
|
||||
by_id: HashMap<String, u64>,
|
||||
default: u64,
|
||||
}
|
||||
|
||||
impl Cutoffs {
|
||||
/// Calculate cutoffs from progress runs.
|
||||
/// Finds the common max progress, then returns when each run reached it.
|
||||
pub fn from_progress(progress_runs: &[Run]) -> Self {
|
||||
const TIME_BUFFER_MS: u64 = 10_000;
|
||||
|
||||
if progress_runs.is_empty() {
|
||||
return Self {
|
||||
by_id: HashMap::new(),
|
||||
default: u64::MAX,
|
||||
};
|
||||
}
|
||||
|
||||
// Find the minimum of max progress values (the common point all runs reached)
|
||||
let common_progress = progress_runs
|
||||
.iter()
|
||||
.map(|r| r.max_value())
|
||||
.fold(f64::MAX, f64::min);
|
||||
|
||||
let by_id: HashMap<_, _> = progress_runs
|
||||
.iter()
|
||||
.map(|run| {
|
||||
let cutoff = run
|
||||
.data
|
||||
.iter()
|
||||
.find(|d| d.value >= common_progress)
|
||||
.map(|d| d.timestamp_ms)
|
||||
.unwrap_or_else(|| run.max_timestamp())
|
||||
.saturating_add(TIME_BUFFER_MS);
|
||||
(run.id.clone(), cutoff)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let default = by_id.values().copied().max().unwrap_or(u64::MAX);
|
||||
|
||||
Self { by_id, default }
|
||||
}
|
||||
|
||||
pub fn get(&self, id: &str) -> u64 {
|
||||
self.by_id.get(id).copied().unwrap_or(self.default)
|
||||
}
|
||||
|
||||
pub fn trim_runs(&self, runs: &[Run]) -> Vec<Run> {
|
||||
runs.iter().map(|r| r.trimmed(self.get(&r.id))).collect()
|
||||
}
|
||||
|
||||
pub fn trim_dual_runs(&self, runs: &[DualRun]) -> Vec<DualRun> {
|
||||
runs.iter().map(|r| r.trimmed(self.get(&r.id))).collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Run {
|
||||
pub id: String,
|
||||
pub data: Vec<DataPoint>,
|
||||
}
|
||||
|
||||
impl Run {
|
||||
pub fn max_timestamp(&self) -> u64 {
|
||||
self.data.iter().map(|d| d.timestamp_ms).max().unwrap_or(0)
|
||||
}
|
||||
|
||||
pub fn max_value(&self) -> f64 {
|
||||
self.data.iter().map(|d| d.value).fold(0.0, f64::max)
|
||||
}
|
||||
|
||||
pub fn trimmed(&self, max_timestamp_ms: u64) -> Self {
|
||||
Self {
|
||||
id: self.id.clone(),
|
||||
data: self
|
||||
.data
|
||||
.iter()
|
||||
.filter(|d| d.timestamp_ms <= max_timestamp_ms)
|
||||
.cloned()
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Two data series from a single run (e.g., memory footprint + peak, or io read + write)
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DualRun {
|
||||
pub id: String,
|
||||
pub primary: Vec<DataPoint>,
|
||||
pub secondary: Vec<DataPoint>,
|
||||
}
|
||||
|
||||
impl DualRun {
|
||||
pub fn trimmed(&self, max_timestamp_ms: u64) -> Self {
|
||||
Self {
|
||||
id: self.id.clone(),
|
||||
primary: self
|
||||
.primary
|
||||
.iter()
|
||||
.filter(|d| d.timestamp_ms <= max_timestamp_ms)
|
||||
.cloned()
|
||||
.collect(),
|
||||
secondary: self
|
||||
.secondary
|
||||
.iter()
|
||||
.filter(|d| d.timestamp_ms <= max_timestamp_ms)
|
||||
.cloned()
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn max_value(&self) -> f64 {
|
||||
self.primary
|
||||
.iter()
|
||||
.chain(self.secondary.iter())
|
||||
.map(|d| d.value)
|
||||
.fold(0.0, f64::max)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn read_runs(crate_path: &Path, filename: &str) -> Result<Vec<Run>> {
|
||||
let mut runs = Vec::new();
|
||||
|
||||
for entry in fs::read_dir(crate_path)? {
|
||||
let run_path = entry?.path();
|
||||
if !run_path.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let run_id = run_path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.ok_or("Invalid run ID")?
|
||||
.to_string();
|
||||
|
||||
// Skip underscore-prefixed or numeric-only directories
|
||||
if run_id.starts_with('_') || run_id.chars().all(|c| c.is_ascii_digit()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let csv_path = run_path.join(filename);
|
||||
if csv_path.exists()
|
||||
&& let Ok(data) = read_csv(&csv_path)
|
||||
{
|
||||
runs.push(Run { id: run_id, data });
|
||||
}
|
||||
}
|
||||
|
||||
Ok(runs)
|
||||
}
|
||||
|
||||
pub fn read_dual_runs(crate_path: &Path, filename: &str) -> Result<Vec<DualRun>> {
|
||||
let mut runs = Vec::new();
|
||||
|
||||
for entry in fs::read_dir(crate_path)? {
|
||||
let run_path = entry?.path();
|
||||
if !run_path.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let run_id = run_path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.ok_or("Invalid run ID")?
|
||||
.to_string();
|
||||
|
||||
if run_id.starts_with('_') || run_id.chars().all(|c| c.is_ascii_digit()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let csv_path = run_path.join(filename);
|
||||
if csv_path.exists()
|
||||
&& let Ok((primary, secondary)) = read_dual_csv(&csv_path)
|
||||
{
|
||||
runs.push(DualRun {
|
||||
id: run_id,
|
||||
primary,
|
||||
secondary,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(runs)
|
||||
}
|
||||
|
||||
fn read_csv(path: &Path) -> Result<Vec<DataPoint>> {
|
||||
let content = fs::read_to_string(path)?;
|
||||
let data = content
|
||||
.lines()
|
||||
.skip(1) // header
|
||||
.filter_map(|line| {
|
||||
let mut parts = line.split(',');
|
||||
let timestamp_ms = parts.next()?.parse().ok()?;
|
||||
let value = parts.next()?.parse().ok()?;
|
||||
Some(DataPoint {
|
||||
timestamp_ms,
|
||||
value,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
fn read_dual_csv(path: &Path) -> Result<(Vec<DataPoint>, Vec<DataPoint>)> {
|
||||
let content = fs::read_to_string(path)?;
|
||||
let mut primary = Vec::new();
|
||||
let mut secondary = Vec::new();
|
||||
|
||||
for line in content.lines().skip(1) {
|
||||
let mut parts = line.split(',');
|
||||
if let (Some(ts), Some(v1), Some(v2)) = (parts.next(), parts.next(), parts.next())
|
||||
&& let (Ok(timestamp_ms), Ok(val1), Ok(val2)) =
|
||||
(ts.parse(), v1.parse::<f64>(), v2.parse::<f64>())
|
||||
{
|
||||
primary.push(DataPoint {
|
||||
timestamp_ms,
|
||||
value: val1,
|
||||
});
|
||||
secondary.push(DataPoint {
|
||||
timestamp_ms,
|
||||
value: val2,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok((primary, secondary))
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
const KIB: f64 = 1024.0;
|
||||
const MIB: f64 = KIB * 1024.0;
|
||||
const GIB: f64 = MIB * 1024.0;
|
||||
|
||||
const MINUTE: f64 = 60.0;
|
||||
const HOUR: f64 = 3600.0;
|
||||
|
||||
/// Returns (scaled_value, unit_suffix)
|
||||
pub fn bytes(bytes: f64) -> (f64, &'static str) {
|
||||
if bytes >= GIB {
|
||||
(bytes / GIB, "GiB")
|
||||
} else if bytes >= MIB {
|
||||
(bytes / MIB, "MiB")
|
||||
} else if bytes >= KIB {
|
||||
(bytes / KIB, "KiB")
|
||||
} else {
|
||||
(bytes, "bytes")
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns (scaled_value, divisor, axis_label)
|
||||
pub fn time(seconds: f64) -> (f64, f64, &'static str) {
|
||||
if seconds >= HOUR * 2.0 {
|
||||
(seconds / HOUR, HOUR, "Time (h)")
|
||||
} else if seconds >= MINUTE * 2.0 {
|
||||
(seconds / MINUTE, MINUTE, "Time (min)")
|
||||
} else {
|
||||
(seconds, 1.0, "Time (s)")
|
||||
}
|
||||
}
|
||||
|
||||
pub fn axis_number(value: f64) -> String {
|
||||
if value >= 1000.0 {
|
||||
let k = value / 1000.0;
|
||||
if k.fract() == 0.0 || k >= 100.0 {
|
||||
format!("{:.0}k", k)
|
||||
} else if k >= 10.0 {
|
||||
format!("{:.1}k", k)
|
||||
} else {
|
||||
format!("{:.2}k", k)
|
||||
}
|
||||
} else {
|
||||
format!("{:.0}", value)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
mod chart;
|
||||
mod data;
|
||||
mod format;
|
||||
|
||||
use data::{read_dual_runs, read_runs, Cutoffs, DualRun, Result, Run};
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
pub struct Visualizer {
|
||||
workspace_root: PathBuf,
|
||||
}
|
||||
|
||||
impl Visualizer {
|
||||
pub fn new(workspace_root: impl AsRef<Path>) -> Self {
|
||||
Self {
|
||||
workspace_root: workspace_root.as_ref().to_path_buf(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_cargo_env() -> Result<Self> {
|
||||
let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.parent()
|
||||
.and_then(|p| p.parent())
|
||||
.ok_or("Failed to find workspace root")?
|
||||
.to_path_buf();
|
||||
Ok(Self { workspace_root })
|
||||
}
|
||||
|
||||
pub fn generate_all_charts(&self) -> Result<()> {
|
||||
let benches_dir = self.workspace_root.join("benches");
|
||||
if !benches_dir.exists() {
|
||||
return Err("Benches directory does not exist".into());
|
||||
}
|
||||
|
||||
for entry in fs::read_dir(&benches_dir)? {
|
||||
let path = entry?.path();
|
||||
if path.is_dir() {
|
||||
let crate_name = path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.ok_or("Invalid crate name")?;
|
||||
|
||||
println!("Generating charts for crate: {}", crate_name);
|
||||
self.generate_crate_charts(&path, crate_name)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn generate_crate_charts(&self, crate_path: &Path, crate_name: &str) -> Result<()> {
|
||||
let disk_runs = read_runs(crate_path, "disk.csv")?;
|
||||
let memory_runs = read_dual_runs(crate_path, "memory.csv")?;
|
||||
let progress_runs = read_runs(crate_path, "progress.csv")?;
|
||||
let io_runs = read_dual_runs(crate_path, "io.csv")?;
|
||||
|
||||
// Combined charts (all runs)
|
||||
self.generate_combined_charts(crate_path, crate_name, &disk_runs, &memory_runs, &progress_runs, &io_runs)?;
|
||||
|
||||
// Individual charts (one per run)
|
||||
self.generate_individual_charts(crate_path, crate_name, &disk_runs, &memory_runs, &progress_runs, &io_runs)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn generate_combined_charts(
|
||||
&self,
|
||||
crate_path: &Path,
|
||||
crate_name: &str,
|
||||
disk_runs: &[Run],
|
||||
memory_runs: &[DualRun],
|
||||
progress_runs: &[Run],
|
||||
io_runs: &[DualRun],
|
||||
) -> Result<()> {
|
||||
let cutoffs = Cutoffs::from_progress(progress_runs);
|
||||
|
||||
// Trim data to per-run cutoffs for fair comparison
|
||||
let disk_trimmed = cutoffs.trim_runs(disk_runs);
|
||||
let memory_trimmed = cutoffs.trim_dual_runs(memory_runs);
|
||||
let io_trimmed = cutoffs.trim_dual_runs(io_runs);
|
||||
|
||||
if !disk_trimmed.is_empty() {
|
||||
chart::generate(
|
||||
chart::ChartConfig {
|
||||
output_path: &crate_path.join("disk.svg"),
|
||||
title: format!("{} — Disk Usage", crate_name),
|
||||
y_label: "Disk Usage".to_string(),
|
||||
y_format: chart::YAxisFormat::Bytes,
|
||||
},
|
||||
&disk_trimmed,
|
||||
)?;
|
||||
}
|
||||
|
||||
if !memory_trimmed.is_empty() {
|
||||
chart::generate_dual(
|
||||
chart::ChartConfig {
|
||||
output_path: &crate_path.join("memory.svg"),
|
||||
title: format!("{} — Memory", crate_name),
|
||||
y_label: "Memory".to_string(),
|
||||
y_format: chart::YAxisFormat::Bytes,
|
||||
},
|
||||
&memory_trimmed,
|
||||
"(current)",
|
||||
"(peak)",
|
||||
)?;
|
||||
}
|
||||
|
||||
if !progress_runs.is_empty() {
|
||||
let progress_trimmed = cutoffs.trim_runs(progress_runs);
|
||||
chart::generate(
|
||||
chart::ChartConfig {
|
||||
output_path: &crate_path.join("progress.svg"),
|
||||
title: format!("{} — Progress", crate_name),
|
||||
y_label: "Progress".to_string(),
|
||||
y_format: chart::YAxisFormat::Number,
|
||||
},
|
||||
&progress_trimmed,
|
||||
)?;
|
||||
}
|
||||
|
||||
if !io_trimmed.is_empty() {
|
||||
// I/O Read (primary column)
|
||||
let io_read: Vec<_> = io_trimmed
|
||||
.iter()
|
||||
.map(|r| Run {
|
||||
id: r.id.clone(),
|
||||
data: r.primary.clone(),
|
||||
})
|
||||
.collect();
|
||||
chart::generate(
|
||||
chart::ChartConfig {
|
||||
output_path: &crate_path.join("io_read.svg"),
|
||||
title: format!("{} — I/O Read", crate_name),
|
||||
y_label: "Bytes Read".to_string(),
|
||||
y_format: chart::YAxisFormat::Bytes,
|
||||
},
|
||||
&io_read,
|
||||
)?;
|
||||
|
||||
// I/O Write (secondary column)
|
||||
let io_write: Vec<_> = io_trimmed
|
||||
.iter()
|
||||
.map(|r| Run {
|
||||
id: r.id.clone(),
|
||||
data: r.secondary.clone(),
|
||||
})
|
||||
.collect();
|
||||
chart::generate(
|
||||
chart::ChartConfig {
|
||||
output_path: &crate_path.join("io_write.svg"),
|
||||
title: format!("{} — I/O Write", crate_name),
|
||||
y_label: "Bytes Written".to_string(),
|
||||
y_format: chart::YAxisFormat::Bytes,
|
||||
},
|
||||
&io_write,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn generate_individual_charts(
|
||||
&self,
|
||||
crate_path: &Path,
|
||||
crate_name: &str,
|
||||
disk_runs: &[Run],
|
||||
memory_runs: &[DualRun],
|
||||
progress_runs: &[Run],
|
||||
io_runs: &[DualRun],
|
||||
) -> Result<()> {
|
||||
for run in disk_runs {
|
||||
let run_path = crate_path.join(&run.id);
|
||||
chart::generate(
|
||||
chart::ChartConfig {
|
||||
output_path: &run_path.join("disk.svg"),
|
||||
title: format!("{} — Disk Usage", crate_name),
|
||||
y_label: "Disk Usage".to_string(),
|
||||
y_format: chart::YAxisFormat::Bytes,
|
||||
},
|
||||
std::slice::from_ref(run),
|
||||
)?;
|
||||
}
|
||||
|
||||
for run in memory_runs {
|
||||
let run_path = crate_path.join(&run.id);
|
||||
chart::generate_dual(
|
||||
chart::ChartConfig {
|
||||
output_path: &run_path.join("memory.svg"),
|
||||
title: format!("{} — Memory", crate_name),
|
||||
y_label: "Memory".to_string(),
|
||||
y_format: chart::YAxisFormat::Bytes,
|
||||
},
|
||||
std::slice::from_ref(run),
|
||||
"(current)",
|
||||
"(peak)",
|
||||
)?;
|
||||
}
|
||||
|
||||
for run in progress_runs {
|
||||
let run_path = crate_path.join(&run.id);
|
||||
chart::generate(
|
||||
chart::ChartConfig {
|
||||
output_path: &run_path.join("progress.svg"),
|
||||
title: format!("{} — Progress", crate_name),
|
||||
y_label: "Progress".to_string(),
|
||||
y_format: chart::YAxisFormat::Number,
|
||||
},
|
||||
std::slice::from_ref(run),
|
||||
)?;
|
||||
}
|
||||
|
||||
for run in io_runs {
|
||||
let run_path = crate_path.join(&run.id);
|
||||
|
||||
let read_run = Run {
|
||||
id: run.id.clone(),
|
||||
data: run.primary.clone(),
|
||||
};
|
||||
chart::generate(
|
||||
chart::ChartConfig {
|
||||
output_path: &run_path.join("io_read.svg"),
|
||||
title: format!("{} — I/O Read", crate_name),
|
||||
y_label: "Bytes Read".to_string(),
|
||||
y_format: chart::YAxisFormat::Bytes,
|
||||
},
|
||||
std::slice::from_ref(&read_run),
|
||||
)?;
|
||||
|
||||
let write_run = Run {
|
||||
id: run.id.clone(),
|
||||
data: run.secondary.clone(),
|
||||
};
|
||||
chart::generate(
|
||||
chart::ChartConfig {
|
||||
output_path: &run_path.join("io_write.svg"),
|
||||
title: format!("{} — I/O Write", crate_name),
|
||||
y_label: "Bytes Written".to_string(),
|
||||
y_format: chart::YAxisFormat::Bytes,
|
||||
},
|
||||
std::slice::from_ref(&write_run),
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
use brk_bencher_visualizer::Visualizer;
|
||||
|
||||
fn main() {
|
||||
let v = Visualizer::from_cargo_env().unwrap();
|
||||
v.generate_all_charts().unwrap();
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
clients/
|
||||
@@ -0,0 +1,19 @@
|
||||
[package]
|
||||
name = "brk_bindgen"
|
||||
description = "A trait-based generator of client bindings for multiple languages"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
build = "build.rs"
|
||||
|
||||
[dependencies]
|
||||
brk_cohort = { workspace = true }
|
||||
brk_query = { workspace = true }
|
||||
brk_types = { workspace = true }
|
||||
oas3 = "0.20"
|
||||
schemars = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
vecdb = { workspace = true }
|
||||
@@ -0,0 +1,296 @@
|
||||
# brk_bindgen Design Document
|
||||
|
||||
## Goal
|
||||
|
||||
Generate typed API clients for **Rust, JavaScript, and Python** with:
|
||||
- **Discoverability**: Full IDE autocomplete for 20k+ metrics
|
||||
- **Ease of use**: Fluent API with `.fetch()` on each metric node
|
||||
|
||||
## Current State
|
||||
|
||||
### What's Working ✅
|
||||
|
||||
1. **JS + JSDoc generator**: Generates `client.js` with full JSDoc type annotations
|
||||
2. **Python generator**: Generates `client.py` with type hints and httpx
|
||||
3. **Rust generator**: Generates `client.rs` with strong typing and reqwest
|
||||
4. **schemars integration**: JSON schemas embedded in `MetricLeafWithSchema` for type info
|
||||
5. **Tree navigation**: `client.tree.blocks.difficulty.fetch()` pattern
|
||||
6. **OpenAPI integration**: All GET endpoints generate typed methods
|
||||
7. **Server integration**: brk_server calls brk_bindgen on startup (when clients/ dir exists)
|
||||
|
||||
### Generated Output
|
||||
|
||||
When `crates/brk_bindgen/clients/` directory exists, running the server generates:
|
||||
|
||||
```
|
||||
crates/brk_bindgen/clients/
|
||||
├── javascript/
|
||||
│ └── client.js # JS + JSDoc with tree + API methods
|
||||
├── python/
|
||||
│ └── client.py # Python with type hints + httpx
|
||||
└── rust/
|
||||
└── client.rs # Rust with reqwest + strong typing
|
||||
```
|
||||
|
||||
## Target Architecture
|
||||
|
||||
### Input Sources
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Input Sources │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ 1. OpenAPI spec (from aide) - endpoint definitions │
|
||||
│ 2. brk_query catalog - metric tree structure │
|
||||
│ 3. brk_types - Rust types for responses (Rust client only) │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Output: Fluent Client
|
||||
|
||||
```javascript
|
||||
// JavaScript (with JSDoc for IDE support)
|
||||
const client = new BrkClient("http://localhost:3000");
|
||||
const data = await client.tree.supply.active.by_date.fetch();
|
||||
// ^^^^ autocomplete all the way down
|
||||
```
|
||||
|
||||
```python
|
||||
# Python
|
||||
client = BrkClient("http://localhost:3000")
|
||||
data = client.tree.supply.active.by_date.fetch()
|
||||
```
|
||||
|
||||
```rust
|
||||
// Rust
|
||||
let client = BrkClient::new("http://localhost:3000")?;
|
||||
let data = client.tree().supply.active.by_date.fetch()?;
|
||||
```
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Smart Metric Nodes
|
||||
|
||||
Each tree leaf becomes a "smart node" holding a client reference:
|
||||
|
||||
```javascript
|
||||
// JavaScript + JSDoc
|
||||
/**
|
||||
* Metric node with fetch capability
|
||||
* @template T
|
||||
*/
|
||||
class MetricNode {
|
||||
constructor(client, path) {
|
||||
this._client = client;
|
||||
this._path = path;
|
||||
}
|
||||
|
||||
async fetch() {
|
||||
return this._client.get(this._path);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```python
|
||||
# Python
|
||||
class MetricNode(Generic[T]):
|
||||
def __init__(self, client: BrkClientBase, path: str):
|
||||
self._client = client
|
||||
self._path = path
|
||||
|
||||
def fetch(self) -> T:
|
||||
return self._client.get(self._path)
|
||||
```
|
||||
|
||||
```rust
|
||||
// Rust
|
||||
pub struct MetricNode<'a, T> {
|
||||
client: &'a BrkClientBase,
|
||||
path: &'static str,
|
||||
_marker: PhantomData<T>,
|
||||
}
|
||||
|
||||
impl<'a, T: DeserializeOwned> MetricNode<'a, T> {
|
||||
pub fn fetch(&self) -> Result<T> {
|
||||
self.client.get(self.path)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Pattern Reuse
|
||||
|
||||
To avoid 20k+ individual types, reuse structural patterns:
|
||||
|
||||
```rust
|
||||
// Shared pattern for metrics with same index groupings
|
||||
struct ByDateHeightMonth<T> {
|
||||
by_date: MetricNode<T>,
|
||||
by_height: MetricNode<T>,
|
||||
by_month: MetricNode<T>,
|
||||
}
|
||||
|
||||
// Composed into full tree
|
||||
struct Supply {
|
||||
active: ByDateHeightMonth<Vec<f64>>,
|
||||
total: ByDateHeightMonth<Vec<f64>>,
|
||||
}
|
||||
```
|
||||
|
||||
## Type Discovery Solution ✅ IMPLEMENTED
|
||||
|
||||
### The Problem
|
||||
|
||||
Type information was erased at runtime because metrics are stored as `&dyn AnyExportableVec` trait objects.
|
||||
|
||||
### The Solution
|
||||
|
||||
Use `std::any::type_name::<T>()` with caching to extract short type names.
|
||||
|
||||
#### Implementation (vecdb)
|
||||
|
||||
Added `short_type_name<T>()` helper and `value_type_to_string()` to `AnyVec` trait.
|
||||
|
||||
### Result
|
||||
|
||||
`brk_query` now exposes:
|
||||
|
||||
```rust
|
||||
for (metric_name, index_to_vec) in &vecs.metric_to_index_to_vec {
|
||||
for (index, vec) in index_to_vec {
|
||||
println!("{} @ {} -> {}",
|
||||
metric_name, // "difficulty"
|
||||
vec.index_type_to_string(), // "Height"
|
||||
vec.value_type_to_string(), // "StoredF64"
|
||||
);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## TreeNode Enhancement ✅ IMPLEMENTED
|
||||
|
||||
Changed `TreeNode::Leaf(String)` to `TreeNode::Leaf(MetricLeafWithSchema)` where:
|
||||
|
||||
```rust
|
||||
#[derive(Debug, Clone, Serialize, JsonSchema)]
|
||||
pub struct MetricLeafWithSchema {
|
||||
#[serde(flatten)]
|
||||
pub leaf: MetricLeaf,
|
||||
#[serde(skip)]
|
||||
pub schema: serde_json::Value, // JSON Schema from schemars
|
||||
}
|
||||
```
|
||||
|
||||
## OpenAPI Integration ✅ IMPLEMENTED
|
||||
|
||||
### Flow
|
||||
|
||||
1. brk_server creates OpenAPI spec via aide
|
||||
2. On startup, serializes spec to JSON string
|
||||
3. Passes JSON to `brk_bindgen::generate_clients()`
|
||||
4. brk_bindgen parses with `oas3` crate (supports OpenAPI 3.1)
|
||||
5. Generates typed methods for all GET endpoints
|
||||
|
||||
### Why oas3?
|
||||
|
||||
aide generates OpenAPI 3.1 specs. The `openapiv3` crate only supports 3.0.x.
|
||||
The `oas3` crate supports OpenAPI 3.1.x parsing.
|
||||
|
||||
## Tasks
|
||||
|
||||
### Phase 0: Type Infrastructure ✅ COMPLETE
|
||||
|
||||
- [x] vecdb: Add `short_type_name<T>()` and `value_type_to_string()`
|
||||
- [x] vecdb: Add optional `schemars` feature with `AnySchemaVec` trait
|
||||
- [x] brk_types: Enhance `TreeNode::Leaf` to include `MetricLeafWithSchema`
|
||||
- [x] brk_traversable: Update all `to_tree_node()` with schemars integration
|
||||
- [x] brk_bindgen: Set up generator module structure
|
||||
|
||||
### Phase 1: JavaScript Client ✅ COMPLETE
|
||||
|
||||
- [x] Define `MetricNode` class with JSDoc generics
|
||||
- [x] Define `BrkClient` with base HTTP functionality
|
||||
- [x] Generate `client.js` with full JSDoc type annotations
|
||||
- [x] Tree navigation: `client.tree.category.metric.fetch()`
|
||||
- [x] API methods from OpenAPI endpoints
|
||||
|
||||
### Phase 2: OpenAPI Integration ✅ COMPLETE
|
||||
|
||||
- [x] Add `oas3` crate dependency (OpenAPI 3.1 support)
|
||||
- [x] brk_server passes OpenAPI JSON to brk_bindgen on startup
|
||||
- [x] Parse OpenAPI spec and extract endpoint definitions
|
||||
- [x] Generate typed methods for each GET endpoint
|
||||
|
||||
### Phase 3: Python Client ✅ COMPLETE
|
||||
|
||||
- [x] Define `MetricNode` class with type hints
|
||||
- [x] Define `BrkClient` with httpx
|
||||
- [x] Generate typed methods from OpenAPI
|
||||
- [x] Generate tree navigation
|
||||
|
||||
### Phase 4: Rust Client ✅ COMPLETE
|
||||
|
||||
- [x] Define `MetricNode<T>` struct with lifetimes
|
||||
- [x] Define `BrkClient` with reqwest (blocking)
|
||||
- [x] Generate tree navigation with proper lifetimes
|
||||
- [x] Generate typed methods from OpenAPI
|
||||
|
||||
### Phase 5: Polish
|
||||
|
||||
- [x] Switch from `openapiv3` to `oas3` crate
|
||||
- [ ] Error types per language
|
||||
- [ ] Documentation generation
|
||||
- [ ] Tests
|
||||
- [ ] Example usage in each language
|
||||
- [ ] Async Rust client variant
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
crates/brk_bindgen/
|
||||
├── src/
|
||||
│ ├── lib.rs
|
||||
│ ├── js.rs # JS constants generation (existing)
|
||||
│ └── generator/
|
||||
│ ├── mod.rs # generate_clients() entry point
|
||||
│ ├── types.rs # ClientMetadata, MetricInfo, IndexPattern
|
||||
│ ├── openapi.rs # OpenAPI 3.1 spec parsing (oas3)
|
||||
│ ├── javascript.rs # JavaScript + JSDoc client ✅
|
||||
│ ├── python.rs # Python client ✅
|
||||
│ └── rust.rs # Rust client ✅
|
||||
├── clients/ # Generated output (gitignored)
|
||||
│ ├── javascript/
|
||||
│ ├── python/
|
||||
│ └── rust/
|
||||
├── Cargo.toml
|
||||
├── README.md
|
||||
└── DESIGN.md
|
||||
|
||||
crates/brk_server/
|
||||
└── src/
|
||||
├── lib.rs # Calls brk_bindgen::generate_clients() on startup
|
||||
└── api/
|
||||
└── openapi.rs # create_openapi() for aide
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
brk_query = { workspace = true }
|
||||
brk_types = { workspace = true }
|
||||
oas3 = "0.20" # OpenAPI 3.1 spec parsing
|
||||
schemars = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
To generate clients:
|
||||
|
||||
```bash
|
||||
# Create the output directory
|
||||
mkdir -p crates/brk_bindgen/clients
|
||||
|
||||
# Run the server (generates clients on startup)
|
||||
cargo run -p brk_server
|
||||
```
|
||||
@@ -0,0 +1,46 @@
|
||||
# brk_bindgen
|
||||
|
||||
Code generation for BRK client libraries.
|
||||
|
||||
## What It Enables
|
||||
|
||||
Generate typed client libraries for Rust, JavaScript/TypeScript, and Python from the OpenAPI specification. Keeps frontend code in sync with available metrics and API endpoints without manual maintenance.
|
||||
|
||||
## Key Features
|
||||
|
||||
- **Multi-language**: Generates Rust, JavaScript, and Python clients
|
||||
- **OpenAPI-driven**: Extracts endpoints and schemas from the OpenAPI spec
|
||||
- **Metric catalog**: Includes all metric IDs and their supported indexes
|
||||
- **Type definitions**: Generates types/interfaces from JSON Schema
|
||||
- **Selective output**: Generate only the languages you need
|
||||
|
||||
## Core API
|
||||
|
||||
```rust,ignore
|
||||
use brk_bindgen::{generate_clients, ClientOutputPaths};
|
||||
|
||||
let paths = ClientOutputPaths::new()
|
||||
.rust("crates/brk_client/src/lib.rs")
|
||||
.javascript("modules/brk-client/index.js")
|
||||
.python("packages/brk_client/brk_client/__init__.py");
|
||||
|
||||
generate_clients(&vecs, &openapi_json, &paths)?;
|
||||
```
|
||||
|
||||
## Generated Clients
|
||||
|
||||
| Language | Contents |
|
||||
|----------|----------|
|
||||
| Rust | Typed API client using `brk_types`, metric catalog |
|
||||
| JavaScript | ES module with JSDoc types, metric catalog, fetch helpers |
|
||||
| Python | Typed client with dataclasses, metric catalog |
|
||||
|
||||
Each client includes:
|
||||
- All REST API endpoints as typed functions
|
||||
- Complete metric catalog with index information
|
||||
- Type definitions for request/response schemas
|
||||
|
||||
## Built On
|
||||
|
||||
- `brk_query` for metric enumeration
|
||||
- `brk_types` for type schemas
|
||||
@@ -0,0 +1,8 @@
|
||||
fn main() {
|
||||
let profile = std::env::var("PROFILE").unwrap_or_default();
|
||||
|
||||
if profile == "release" {
|
||||
println!("cargo:rustc-flag=-C");
|
||||
println!("cargo:rustc-flag=target-cpu=native");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
//! Analysis module for name deconstruction and pattern detection.
|
||||
//!
|
||||
//! This module implements bottom-up analysis of vec names to detect
|
||||
//! common denominators (prefixes/suffixes) and field positions.
|
||||
|
||||
mod names;
|
||||
mod patterns;
|
||||
mod positions;
|
||||
mod tree;
|
||||
|
||||
pub use names::*;
|
||||
pub use patterns::*;
|
||||
pub use positions::*;
|
||||
pub use tree::*;
|
||||
@@ -0,0 +1,451 @@
|
||||
//! Vec name deconstruction and reconstruction logic.
|
||||
//!
|
||||
//! This module analyzes vec names bottom-up to detect common denominators
|
||||
//! (prefixes or suffixes) and field positions for pattern instances.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::FieldNamePosition;
|
||||
|
||||
/// Common denominator found across children's effective names.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CommonDenominator {
|
||||
/// Children share this prefix. Fields append their unique suffix.
|
||||
/// Example: children are ["addrs_0sats", "addrs_1sats"], common = "addrs_"
|
||||
Prefix(String),
|
||||
/// Children share this suffix. Fields prepend their unique prefix.
|
||||
/// Example: children are ["cumulative_supply", "net_supply"], common = "_supply"
|
||||
Suffix(String),
|
||||
/// No common part found. Fields use Identity (field = base).
|
||||
None,
|
||||
}
|
||||
|
||||
/// Result of analyzing a pattern level.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PatternAnalysis {
|
||||
/// The common prefix/suffix found across all children.
|
||||
pub common: CommonDenominator,
|
||||
/// What's left after stripping the common part (passed to parent).
|
||||
pub base: String,
|
||||
/// How each field modifies the accumulated name.
|
||||
pub field_positions: HashMap<String, FieldNamePosition>,
|
||||
}
|
||||
|
||||
/// Analyze a pattern level using child effective names.
|
||||
///
|
||||
/// This is the core algorithm that detects common prefix/suffix and
|
||||
/// determines field positions for each child.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `child_names` - Vec of (field_name, effective_name) pairs
|
||||
/// where effective_name is either:
|
||||
/// - For leaves: the leaf's vec name
|
||||
/// - For branches: the base returned by analyzing that branch
|
||||
pub fn analyze_pattern_level(child_names: &[(String, String)]) -> PatternAnalysis {
|
||||
if child_names.is_empty() {
|
||||
return PatternAnalysis {
|
||||
common: CommonDenominator::None,
|
||||
base: String::new(),
|
||||
field_positions: HashMap::new(),
|
||||
};
|
||||
}
|
||||
|
||||
if child_names.len() == 1 {
|
||||
let (field_name, effective) = &child_names[0];
|
||||
let mut positions = HashMap::new();
|
||||
|
||||
// Try suffix match: effective ends with "_fieldname"
|
||||
let suffix_pattern = format!("_{}", field_name);
|
||||
if let Some(base) = effective.strip_suffix(&suffix_pattern) {
|
||||
positions.insert(
|
||||
field_name.clone(),
|
||||
FieldNamePosition::Append(suffix_pattern),
|
||||
);
|
||||
return PatternAnalysis {
|
||||
common: CommonDenominator::None,
|
||||
base: base.to_string(),
|
||||
field_positions: positions,
|
||||
};
|
||||
}
|
||||
|
||||
// Try prefix match: effective starts with "fieldname_"
|
||||
let prefix_pattern = format!("{}_", field_name);
|
||||
if let Some(base) = effective.strip_prefix(&prefix_pattern) {
|
||||
positions.insert(
|
||||
field_name.clone(),
|
||||
FieldNamePosition::Prepend(prefix_pattern),
|
||||
);
|
||||
return PatternAnalysis {
|
||||
common: CommonDenominator::None,
|
||||
base: base.to_string(),
|
||||
field_positions: positions,
|
||||
};
|
||||
}
|
||||
|
||||
// Field equals effective OR field doesn't appear → Identity
|
||||
// Root-level instances where field == effective are handled by
|
||||
// passing empty `acc` and conditional position expressions
|
||||
positions.insert(field_name.clone(), FieldNamePosition::Identity);
|
||||
return PatternAnalysis {
|
||||
common: CommonDenominator::None,
|
||||
base: effective.clone(),
|
||||
field_positions: positions,
|
||||
};
|
||||
}
|
||||
|
||||
let effective_names: Vec<&str> = child_names.iter().map(|(_, n)| n.as_str()).collect();
|
||||
|
||||
// Try to find common prefix first
|
||||
if let Some(prefix) = find_common_prefix(&effective_names)
|
||||
&& !prefix.is_empty()
|
||||
{
|
||||
let base = prefix.trim_end_matches('_').to_string();
|
||||
let mut positions = HashMap::new();
|
||||
for (field_name, effective) in child_names {
|
||||
// If effective equals the base (prefix without underscore), use Identity
|
||||
if effective == &base {
|
||||
positions.insert(field_name.clone(), FieldNamePosition::Identity);
|
||||
} else if let Some(suffix) = effective.strip_prefix(&prefix) {
|
||||
// Normal case: effective has the full prefix
|
||||
let suffix_with_underscore = if suffix.starts_with('_') {
|
||||
suffix.to_string()
|
||||
} else {
|
||||
format!("_{}", suffix)
|
||||
};
|
||||
positions.insert(
|
||||
field_name.clone(),
|
||||
FieldNamePosition::Append(suffix_with_underscore),
|
||||
);
|
||||
} else {
|
||||
// Fallback: use Identity if strip_prefix fails unexpectedly
|
||||
positions.insert(field_name.clone(), FieldNamePosition::Identity);
|
||||
}
|
||||
}
|
||||
return PatternAnalysis {
|
||||
common: CommonDenominator::Prefix(prefix),
|
||||
base,
|
||||
field_positions: positions,
|
||||
};
|
||||
}
|
||||
|
||||
// Try to find common suffix
|
||||
if let Some(suffix) = find_common_suffix(&effective_names)
|
||||
&& !suffix.is_empty()
|
||||
{
|
||||
let mut positions = HashMap::new();
|
||||
for (field_name, effective) in child_names {
|
||||
let prefix = effective
|
||||
.strip_suffix(&suffix)
|
||||
.unwrap_or(effective)
|
||||
.to_string();
|
||||
let prefix_with_underscore = if prefix.ends_with('_') {
|
||||
prefix
|
||||
} else {
|
||||
format!("{}_", prefix)
|
||||
};
|
||||
positions.insert(
|
||||
field_name.clone(),
|
||||
FieldNamePosition::Prepend(prefix_with_underscore),
|
||||
);
|
||||
}
|
||||
let base = suffix.trim_start_matches('_').to_string();
|
||||
return PatternAnalysis {
|
||||
common: CommonDenominator::Suffix(suffix),
|
||||
base,
|
||||
field_positions: positions,
|
||||
};
|
||||
}
|
||||
|
||||
// No common part - use Identity for all fields
|
||||
let mut positions = HashMap::new();
|
||||
for (field_name, _) in child_names {
|
||||
positions.insert(field_name.clone(), FieldNamePosition::Identity);
|
||||
}
|
||||
|
||||
// Use the first name as base (they're all independent)
|
||||
let base = child_names
|
||||
.first()
|
||||
.map(|(_, n)| n.clone())
|
||||
.unwrap_or_default();
|
||||
|
||||
PatternAnalysis {
|
||||
common: CommonDenominator::None,
|
||||
base,
|
||||
field_positions: positions,
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the longest common prefix among all strings.
|
||||
/// The prefix must end at an underscore boundary for semantic coherence.
|
||||
fn find_common_prefix(names: &[&str]) -> Option<String> {
|
||||
if names.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let first = names[0];
|
||||
if first.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Find character-by-character common prefix
|
||||
let mut prefix_len = 0;
|
||||
for (i, ch) in first.chars().enumerate() {
|
||||
if names.iter().all(|n| n.chars().nth(i) == Some(ch)) {
|
||||
prefix_len = i + 1;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if prefix_len == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let raw_prefix = &first[..prefix_len];
|
||||
|
||||
// If raw_prefix exactly matches one of the names, it's a complete metric name.
|
||||
// In this case, return it with trailing underscore to preserve the full name.
|
||||
if names.contains(&raw_prefix) {
|
||||
return Some(format!("{}_", raw_prefix));
|
||||
}
|
||||
|
||||
// Find the last underscore position to get a clean boundary
|
||||
// Prefer ending at an underscore for semantic coherence
|
||||
if let Some(last_underscore) = raw_prefix.rfind('_')
|
||||
&& last_underscore > 0
|
||||
{
|
||||
let clean_prefix = &first[..=last_underscore];
|
||||
// Verify this still works for all names
|
||||
if names.iter().all(|n| n.starts_with(clean_prefix)) {
|
||||
return Some(clean_prefix.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// If no underscore boundary works, the full prefix must end at an underscore
|
||||
if raw_prefix.ends_with('_') {
|
||||
return Some(raw_prefix.to_string());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Find the longest common suffix among all strings.
|
||||
/// The suffix must start at an underscore boundary for semantic coherence.
|
||||
fn find_common_suffix(names: &[&str]) -> Option<String> {
|
||||
if names.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let first = names[0];
|
||||
if first.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Find character-by-character common suffix (from the end)
|
||||
let first_chars: Vec<char> = first.chars().collect();
|
||||
let mut suffix_len = 0;
|
||||
|
||||
for i in 0..first_chars.len() {
|
||||
let idx_from_end = first_chars.len() - 1 - i;
|
||||
let ch = first_chars[idx_from_end];
|
||||
|
||||
let all_match = names.iter().all(|n| {
|
||||
let n_chars: Vec<char> = n.chars().collect();
|
||||
if i >= n_chars.len() {
|
||||
return false;
|
||||
}
|
||||
n_chars[n_chars.len() - 1 - i] == ch
|
||||
});
|
||||
|
||||
if all_match {
|
||||
suffix_len = i + 1;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if suffix_len == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let raw_suffix = &first[first.len() - suffix_len..];
|
||||
|
||||
// Find the first underscore position to get a clean boundary
|
||||
if let Some(first_underscore) = raw_suffix.find('_')
|
||||
&& first_underscore < raw_suffix.len() - 1
|
||||
{
|
||||
let clean_suffix = &raw_suffix[first_underscore..];
|
||||
// Verify this still works for all names
|
||||
if names.iter().all(|n| n.ends_with(clean_suffix)) {
|
||||
return Some(clean_suffix.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// If no underscore boundary works, the full suffix must start with underscore
|
||||
if raw_suffix.starts_with('_') {
|
||||
return Some(raw_suffix.to_string());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_common_prefix() {
|
||||
let names = vec!["addrs_0sats", "addrs_1sats", "addrs_2sats"];
|
||||
assert_eq!(find_common_prefix(&names), Some("addrs_".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_common_suffix() {
|
||||
let names = vec!["cumulative_supply", "net_supply", "total_supply"];
|
||||
assert_eq!(find_common_suffix(&names), Some("_supply".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_common() {
|
||||
let names = vec!["foo", "bar", "baz"];
|
||||
assert_eq!(find_common_prefix(&names), None);
|
||||
assert_eq!(find_common_suffix(&names), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_pattern_level_prefix() {
|
||||
let children = vec![
|
||||
("_0sats".to_string(), "addrs_0sats".to_string()),
|
||||
("_1sats".to_string(), "addrs_1sats".to_string()),
|
||||
];
|
||||
let analysis = analyze_pattern_level(&children);
|
||||
|
||||
assert!(matches!(analysis.common, CommonDenominator::Prefix(_)));
|
||||
assert_eq!(analysis.base, "addrs");
|
||||
assert!(matches!(
|
||||
analysis.field_positions.get("_0sats"),
|
||||
Some(FieldNamePosition::Append(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_pattern_level_suffix() {
|
||||
let children = vec![
|
||||
("cumulative".to_string(), "cumulative_supply".to_string()),
|
||||
("net".to_string(), "net_supply".to_string()),
|
||||
];
|
||||
let analysis = analyze_pattern_level(&children);
|
||||
|
||||
assert!(matches!(analysis.common, CommonDenominator::Suffix(_)));
|
||||
assert_eq!(analysis.base, "supply");
|
||||
assert!(matches!(
|
||||
analysis.field_positions.get("cumulative"),
|
||||
Some(FieldNamePosition::Prepend(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_child_suffix() {
|
||||
// Field "count" appears as suffix "_count" in "activity_count"
|
||||
let children = vec![("count".to_string(), "activity_count".to_string())];
|
||||
let analysis = analyze_pattern_level(&children);
|
||||
|
||||
assert!(matches!(analysis.common, CommonDenominator::None));
|
||||
assert_eq!(analysis.base, "activity");
|
||||
assert_eq!(
|
||||
analysis.field_positions.get("count"),
|
||||
Some(&FieldNamePosition::Append("_count".to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_child_prefix() {
|
||||
// Field "cumulative" appears as prefix "cumulative_" in "cumulative_supply"
|
||||
let children = vec![("cumulative".to_string(), "cumulative_supply".to_string())];
|
||||
let analysis = analyze_pattern_level(&children);
|
||||
|
||||
assert!(matches!(analysis.common, CommonDenominator::None));
|
||||
assert_eq!(analysis.base, "supply");
|
||||
assert_eq!(
|
||||
analysis.field_positions.get("cumulative"),
|
||||
Some(&FieldNamePosition::Prepend("cumulative_".to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_child_identity_equal() {
|
||||
// Field "supply" equals effective "supply" → Identity
|
||||
// (root-level handling is done via empty acc and conditional expressions)
|
||||
let children = vec![("supply".to_string(), "supply".to_string())];
|
||||
let analysis = analyze_pattern_level(&children);
|
||||
|
||||
assert!(matches!(analysis.common, CommonDenominator::None));
|
||||
assert_eq!(analysis.base, "supply");
|
||||
assert_eq!(
|
||||
analysis.field_positions.get("supply"),
|
||||
Some(&FieldNamePosition::Identity)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_child_identity_structural() {
|
||||
// Field "x" doesn't appear in "a_b" - it's structural grouping
|
||||
let children = vec![("x".to_string(), "a_b".to_string())];
|
||||
let analysis = analyze_pattern_level(&children);
|
||||
|
||||
assert!(matches!(analysis.common, CommonDenominator::None));
|
||||
assert_eq!(analysis.base, "a_b"); // passes through unchanged
|
||||
assert_eq!(
|
||||
analysis.field_positions.get("x"),
|
||||
Some(&FieldNamePosition::Identity)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_common_prefix_exact_match() {
|
||||
// When one name exactly matches the common prefix, preserve the full name
|
||||
// This fixes the realized_loss vs realized_count bug
|
||||
let names = vec!["realized_loss", "realized_loss_cumulative"];
|
||||
assert_eq!(
|
||||
find_common_prefix(&names),
|
||||
Some("realized_loss_".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_common_prefix_exact_match_multiple() {
|
||||
// Multiple children with same base name
|
||||
let names = vec!["realized_loss", "realized_loss", "realized_loss_cumulative"];
|
||||
assert_eq!(
|
||||
find_common_prefix(&names),
|
||||
Some("realized_loss_".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_analyze_pattern_level_full_base() {
|
||||
// When names are like [realized_loss, realized_loss_cumulative],
|
||||
// base should be "realized_loss" not "realized"
|
||||
let children = vec![
|
||||
("sum".to_string(), "realized_loss".to_string()),
|
||||
(
|
||||
"cumulative".to_string(),
|
||||
"realized_loss_cumulative".to_string(),
|
||||
),
|
||||
];
|
||||
let analysis = analyze_pattern_level(&children);
|
||||
|
||||
assert!(matches!(analysis.common, CommonDenominator::Prefix(_)));
|
||||
assert_eq!(analysis.base, "realized_loss");
|
||||
// sum effective equals base, so position is Identity
|
||||
assert_eq!(
|
||||
analysis.field_positions.get("sum"),
|
||||
Some(&FieldNamePosition::Identity)
|
||||
);
|
||||
// cumulative has suffix "_cumulative" after the base
|
||||
assert_eq!(
|
||||
analysis.field_positions.get("cumulative"),
|
||||
Some(&FieldNamePosition::Append("_cumulative".to_string()))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
//! Structural pattern detection using bottom-up analysis.
|
||||
//!
|
||||
//! This module detects repeating tree structures and analyzes them
|
||||
//! using the bottom-up name deconstruction algorithm.
|
||||
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
|
||||
use brk_types::TreeNode;
|
||||
|
||||
use super::analyze_all_field_positions;
|
||||
use crate::{PatternField, StructuralPattern, schema_to_json_type, to_pascal_case};
|
||||
|
||||
/// Context for pattern detection, holding all intermediate state.
|
||||
struct PatternContext {
|
||||
/// Maps field signatures to pattern names
|
||||
signature_to_pattern: HashMap<Vec<PatternField>, String>,
|
||||
/// Counts how many times each signature appears
|
||||
signature_counts: HashMap<Vec<PatternField>, usize>,
|
||||
/// Maps normalized signatures to pattern names (for naming consistency)
|
||||
normalized_to_name: HashMap<Vec<PatternField>, String>,
|
||||
/// Counts pattern name usage (for unique naming)
|
||||
name_counts: HashMap<String, usize>,
|
||||
/// Maps signatures to their child field lists
|
||||
signature_to_child_fields: HashMap<Vec<PatternField>, Vec<Vec<PatternField>>>,
|
||||
}
|
||||
|
||||
impl PatternContext {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
signature_to_pattern: HashMap::new(),
|
||||
signature_counts: HashMap::new(),
|
||||
normalized_to_name: HashMap::new(),
|
||||
name_counts: HashMap::new(),
|
||||
signature_to_child_fields: HashMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect structural patterns in the tree using a bottom-up approach.
|
||||
///
|
||||
/// Returns (patterns, concrete_to_pattern, concrete_to_type_param).
|
||||
pub fn detect_structural_patterns(
|
||||
tree: &TreeNode,
|
||||
) -> (
|
||||
Vec<StructuralPattern>,
|
||||
HashMap<Vec<PatternField>, String>,
|
||||
HashMap<Vec<PatternField>, String>,
|
||||
) {
|
||||
let mut ctx = PatternContext::new();
|
||||
resolve_branch_patterns(tree, "root", &mut ctx);
|
||||
|
||||
let (generic_patterns, generic_mappings, type_mappings) =
|
||||
detect_generic_patterns(&ctx.signature_to_pattern);
|
||||
|
||||
let mut patterns: Vec<StructuralPattern> = ctx.signature_to_pattern
|
||||
.iter()
|
||||
.filter(|(sig, _)| {
|
||||
ctx.signature_counts.get(*sig).copied().unwrap_or(0) >= 2
|
||||
&& !generic_mappings.contains_key(*sig)
|
||||
})
|
||||
.map(|(fields, name)| {
|
||||
let child_fields_list = ctx.signature_to_child_fields.get(fields);
|
||||
let fields_with_type_params = fields
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, f)| {
|
||||
let type_param = child_fields_list
|
||||
.and_then(|list| list.get(i))
|
||||
.and_then(|cf| type_mappings.get(cf).cloned());
|
||||
PatternField {
|
||||
type_param,
|
||||
..f.clone()
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
StructuralPattern {
|
||||
name: name.clone(),
|
||||
fields: fields_with_type_params,
|
||||
field_positions: HashMap::new(),
|
||||
is_generic: false,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
patterns.extend(generic_patterns);
|
||||
|
||||
let mut pattern_lookup: HashMap<Vec<PatternField>, String> = HashMap::new();
|
||||
for (sig, name) in &ctx.signature_to_pattern {
|
||||
if ctx.signature_counts.get(sig).copied().unwrap_or(0) >= 2 {
|
||||
pattern_lookup.insert(sig.clone(), name.clone());
|
||||
}
|
||||
}
|
||||
pattern_lookup.extend(generic_mappings.clone());
|
||||
|
||||
let concrete_to_pattern = pattern_lookup.clone();
|
||||
|
||||
// Use the new bottom-up field position analysis
|
||||
analyze_all_field_positions(tree, &mut patterns, &pattern_lookup);
|
||||
|
||||
patterns.sort_by(|a, b| b.fields.len().cmp(&a.fields.len()));
|
||||
(patterns, concrete_to_pattern, type_mappings)
|
||||
}
|
||||
|
||||
/// Detect generic patterns by grouping signatures by their normalized form.
|
||||
fn detect_generic_patterns(
|
||||
signature_to_pattern: &HashMap<Vec<PatternField>, String>,
|
||||
) -> (
|
||||
Vec<StructuralPattern>,
|
||||
HashMap<Vec<PatternField>, String>,
|
||||
HashMap<Vec<PatternField>, String>,
|
||||
) {
|
||||
let mut normalized_groups: HashMap<
|
||||
Vec<PatternField>,
|
||||
Vec<(Vec<PatternField>, String, String)>,
|
||||
> = HashMap::new();
|
||||
|
||||
for (fields, name) in signature_to_pattern {
|
||||
if let Some((normalized, extracted_type)) = normalize_fields_for_generic(fields) {
|
||||
normalized_groups
|
||||
.entry(normalized)
|
||||
.or_default()
|
||||
.push((fields.clone(), name.clone(), extracted_type));
|
||||
}
|
||||
}
|
||||
|
||||
let mut patterns = Vec::new();
|
||||
let mut pattern_mappings: HashMap<Vec<PatternField>, String> = HashMap::new();
|
||||
let mut type_mappings: HashMap<Vec<PatternField>, String> = HashMap::new();
|
||||
|
||||
for (normalized_fields, group) in normalized_groups {
|
||||
if group.len() >= 2 {
|
||||
let generic_name = group[0].1.clone();
|
||||
for (concrete_fields, _, extracted_type) in &group {
|
||||
pattern_mappings.insert(concrete_fields.clone(), generic_name.clone());
|
||||
type_mappings.insert(concrete_fields.clone(), extracted_type.clone());
|
||||
}
|
||||
patterns.push(StructuralPattern {
|
||||
name: generic_name,
|
||||
fields: normalized_fields,
|
||||
field_positions: HashMap::new(),
|
||||
is_generic: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
(patterns, pattern_mappings, type_mappings)
|
||||
}
|
||||
|
||||
/// Normalize fields by replacing concrete value types with "T".
|
||||
fn normalize_fields_for_generic(fields: &[PatternField]) -> Option<(Vec<PatternField>, String)> {
|
||||
let leaf_types: Vec<&str> = fields
|
||||
.iter()
|
||||
.filter(|f| f.is_leaf())
|
||||
.map(|f| f.rust_type.as_str())
|
||||
.collect();
|
||||
|
||||
if leaf_types.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let first_type = leaf_types[0];
|
||||
if !leaf_types.iter().all(|t| *t == first_type) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let normalized = fields
|
||||
.iter()
|
||||
.map(|f| {
|
||||
if f.is_branch() {
|
||||
f.clone()
|
||||
} else {
|
||||
PatternField {
|
||||
name: f.name.clone(),
|
||||
rust_type: "T".to_string(),
|
||||
json_type: "T".to_string(),
|
||||
indexes: f.indexes.clone(),
|
||||
type_param: None,
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Some((normalized, crate::extract_inner_type(first_type)))
|
||||
}
|
||||
|
||||
/// Recursively resolve branch patterns bottom-up.
|
||||
fn resolve_branch_patterns(
|
||||
node: &TreeNode,
|
||||
field_name: &str,
|
||||
ctx: &mut PatternContext,
|
||||
) -> Option<(String, Vec<PatternField>)> {
|
||||
let TreeNode::Branch(children) = node else {
|
||||
return None;
|
||||
};
|
||||
|
||||
let mut fields: Vec<PatternField> = Vec::new();
|
||||
let mut child_fields_vec: Vec<Vec<PatternField>> = Vec::new();
|
||||
|
||||
for (child_name, child_node) in children {
|
||||
let (rust_type, json_type, indexes, child_fields) = match child_node {
|
||||
TreeNode::Leaf(leaf) => (
|
||||
leaf.value_type().to_string(),
|
||||
schema_to_json_type(&leaf.schema),
|
||||
leaf.indexes().clone(),
|
||||
Vec::new(),
|
||||
),
|
||||
TreeNode::Branch(_) => {
|
||||
let (pattern_name, child_pattern_fields) =
|
||||
resolve_branch_patterns(child_node, child_name, ctx)
|
||||
.unwrap_or_else(|| ("Unknown".to_string(), Vec::new()));
|
||||
(
|
||||
pattern_name.clone(),
|
||||
pattern_name,
|
||||
BTreeSet::new(),
|
||||
child_pattern_fields,
|
||||
)
|
||||
}
|
||||
};
|
||||
fields.push(PatternField {
|
||||
name: child_name.clone(),
|
||||
rust_type,
|
||||
json_type,
|
||||
indexes,
|
||||
type_param: None,
|
||||
});
|
||||
child_fields_vec.push(child_fields);
|
||||
}
|
||||
|
||||
fields.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
*ctx.signature_counts.entry(fields.clone()).or_insert(0) += 1;
|
||||
|
||||
ctx.signature_to_child_fields
|
||||
.entry(fields.clone())
|
||||
.or_insert(child_fields_vec);
|
||||
|
||||
let pattern_name = if let Some(existing) = ctx.signature_to_pattern.get(&fields) {
|
||||
existing.clone()
|
||||
} else {
|
||||
let normalized = normalize_fields_for_naming(&fields);
|
||||
let name = ctx
|
||||
.normalized_to_name
|
||||
.entry(normalized)
|
||||
.or_insert_with(|| generate_pattern_name(field_name, &mut ctx.name_counts))
|
||||
.clone();
|
||||
ctx.signature_to_pattern.insert(fields.clone(), name.clone());
|
||||
name
|
||||
};
|
||||
|
||||
Some((pattern_name, fields))
|
||||
}
|
||||
|
||||
/// Normalize fields for naming (same structure = same name).
|
||||
fn normalize_fields_for_naming(fields: &[PatternField]) -> Vec<PatternField> {
|
||||
fields
|
||||
.iter()
|
||||
.map(|f| {
|
||||
if f.is_branch() {
|
||||
f.clone()
|
||||
} else {
|
||||
PatternField {
|
||||
name: f.name.clone(),
|
||||
rust_type: "_".to_string(),
|
||||
json_type: "_".to_string(),
|
||||
indexes: f.indexes.clone(),
|
||||
type_param: None,
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Generate a unique pattern name.
|
||||
fn generate_pattern_name(field_name: &str, name_counts: &mut HashMap<String, usize>) -> String {
|
||||
let pascal = to_pascal_case(field_name);
|
||||
let sanitized = if pascal.chars().next().is_some_and(|c| c.is_ascii_digit()) {
|
||||
format!("_{}", pascal)
|
||||
} else {
|
||||
pascal
|
||||
};
|
||||
|
||||
let base_name = format!("{}Pattern", sanitized);
|
||||
let count = name_counts.entry(base_name.clone()).or_insert(0);
|
||||
*count += 1;
|
||||
|
||||
if *count == 1 {
|
||||
base_name
|
||||
} else {
|
||||
format!("{}{}", base_name, count)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
//! Field position detection for pattern instances.
|
||||
//!
|
||||
//! This module bridges the name analysis with pattern field positions,
|
||||
//! processing patterns bottom-up to determine how each field modifies
|
||||
//! the accumulated metric name.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use brk_types::TreeNode;
|
||||
|
||||
use super::{analyze_pattern_level, get_node_fields};
|
||||
use crate::{FieldNamePosition, PatternField, StructuralPattern};
|
||||
|
||||
/// Analyze field positions for all patterns using bottom-up tree traversal.
|
||||
///
|
||||
/// This is the main entry point for field position detection. It processes
|
||||
/// the tree bottom-up, analyzing each pattern instance and aggregating
|
||||
/// the positions across all instances.
|
||||
pub fn analyze_all_field_positions(
|
||||
tree: &TreeNode,
|
||||
patterns: &mut [StructuralPattern],
|
||||
pattern_lookup: &HashMap<Vec<PatternField>, String>,
|
||||
) {
|
||||
let mut all_positions: HashMap<String, HashMap<String, Vec<FieldNamePosition>>> =
|
||||
HashMap::new();
|
||||
|
||||
// Collect positions from all instances bottom-up
|
||||
collect_positions_bottom_up(tree, pattern_lookup, &mut all_positions);
|
||||
|
||||
// Merge positions into patterns
|
||||
for pattern in patterns.iter_mut() {
|
||||
if let Some(field_positions) = all_positions.get(&pattern.name) {
|
||||
pattern.field_positions = merge_field_positions(field_positions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Recursively collect field positions bottom-up.
|
||||
/// Returns the effective base for this node (used by parent level).
|
||||
fn collect_positions_bottom_up(
|
||||
node: &TreeNode,
|
||||
pattern_lookup: &HashMap<Vec<PatternField>, String>,
|
||||
all_positions: &mut HashMap<String, HashMap<String, Vec<FieldNamePosition>>>,
|
||||
) -> Option<String> {
|
||||
match node {
|
||||
TreeNode::Leaf(leaf) => {
|
||||
// Leaves return their vec name as the effective base
|
||||
Some(leaf.name().to_string())
|
||||
}
|
||||
TreeNode::Branch(children) => {
|
||||
// First, process all children recursively (bottom-up)
|
||||
let mut child_bases: HashMap<String, String> = HashMap::new();
|
||||
for (field_name, child_node) in children {
|
||||
if let Some(base) = collect_positions_bottom_up(child_node, pattern_lookup, all_positions) {
|
||||
child_bases.insert(field_name.clone(), base);
|
||||
}
|
||||
}
|
||||
|
||||
// Build child names for this level's analysis
|
||||
let child_names: Vec<(String, String)> = children
|
||||
.keys()
|
||||
.filter_map(|field_name| {
|
||||
child_bases
|
||||
.get(field_name)
|
||||
.map(|base| (field_name.clone(), base.clone()))
|
||||
})
|
||||
.collect();
|
||||
|
||||
if child_names.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Analyze this level
|
||||
let analysis = analyze_pattern_level(&child_names);
|
||||
|
||||
// Get the pattern name for this node (if any)
|
||||
let fields = get_node_fields(children, pattern_lookup);
|
||||
if let Some(pattern_name) = pattern_lookup.get(&fields) {
|
||||
// Record field positions for this pattern instance
|
||||
for (field_name, position) in &analysis.field_positions {
|
||||
all_positions
|
||||
.entry(pattern_name.clone())
|
||||
.or_default()
|
||||
.entry(field_name.clone())
|
||||
.or_default()
|
||||
.push(position.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Return our base for the parent level
|
||||
Some(analysis.base)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Merge multiple observed positions for each field into a single position.
|
||||
/// Uses the first non-Identity position found, as Identity from root-level
|
||||
/// instances is now handled by passing empty `acc`.
|
||||
fn merge_field_positions(
|
||||
field_positions: &HashMap<String, Vec<FieldNamePosition>>,
|
||||
) -> HashMap<String, FieldNamePosition> {
|
||||
field_positions
|
||||
.iter()
|
||||
.filter_map(|(field_name, positions)| {
|
||||
if positions.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Prefer Append/Prepend over Identity, as Identity at root-level
|
||||
// is handled by empty acc and conditional position expressions
|
||||
let preferred = positions
|
||||
.iter()
|
||||
.find(|p| !matches!(p, FieldNamePosition::Identity))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| positions[0].clone());
|
||||
|
||||
Some((field_name.clone(), preferred))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
//! Tree traversal helpers for pattern analysis.
|
||||
//!
|
||||
//! This module provides utilities for working with the TreeNode structure,
|
||||
//! including leaf name extraction and index pattern detection.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
|
||||
use brk_types::{Index, TreeNode};
|
||||
|
||||
use crate::{IndexSetPattern, PatternField, child_type_name, schema_to_json_type};
|
||||
|
||||
/// Get the first leaf name from a tree node.
|
||||
pub fn get_first_leaf_name(node: &TreeNode) -> Option<String> {
|
||||
match node {
|
||||
TreeNode::Leaf(leaf) => Some(leaf.name().to_string()),
|
||||
TreeNode::Branch(children) => children.values().find_map(get_first_leaf_name),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get all leaf names from a tree node.
|
||||
pub fn get_all_leaf_names(node: &TreeNode) -> Vec<String> {
|
||||
match node {
|
||||
TreeNode::Leaf(leaf) => vec![leaf.name().to_string()],
|
||||
TreeNode::Branch(children) => children.values().flat_map(get_all_leaf_names).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the field signature for a branch node's children.
|
||||
pub fn get_node_fields(
|
||||
children: &BTreeMap<String, TreeNode>,
|
||||
pattern_lookup: &HashMap<Vec<PatternField>, String>,
|
||||
) -> Vec<PatternField> {
|
||||
let mut fields: Vec<PatternField> = children
|
||||
.iter()
|
||||
.map(|(name, node)| {
|
||||
let (rust_type, json_type, indexes) = match node {
|
||||
TreeNode::Leaf(leaf) => (
|
||||
leaf.value_type().to_string(),
|
||||
schema_to_json_type(&leaf.schema),
|
||||
leaf.indexes().clone(),
|
||||
),
|
||||
TreeNode::Branch(grandchildren) => {
|
||||
let child_fields = get_node_fields(grandchildren, pattern_lookup);
|
||||
let pattern_name = pattern_lookup
|
||||
.get(&child_fields)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "Unknown".to_string());
|
||||
(pattern_name.clone(), pattern_name, BTreeSet::new())
|
||||
}
|
||||
};
|
||||
PatternField {
|
||||
name: name.clone(),
|
||||
rust_type,
|
||||
json_type,
|
||||
indexes,
|
||||
type_param: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
fields.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
fields
|
||||
}
|
||||
|
||||
/// Detect index patterns (sets of indexes that appear together on metrics).
|
||||
pub fn detect_index_patterns(tree: &TreeNode) -> (BTreeSet<Index>, Vec<IndexSetPattern>) {
|
||||
let mut used_indexes: BTreeSet<Index> = BTreeSet::new();
|
||||
let mut unique_index_sets: BTreeSet<BTreeSet<Index>> = BTreeSet::new();
|
||||
|
||||
collect_indexes_from_tree(tree, &mut used_indexes, &mut unique_index_sets);
|
||||
|
||||
// Sort by count (descending) then by first index name for deterministic ordering
|
||||
let mut sorted_sets: Vec<_> = unique_index_sets
|
||||
.into_iter()
|
||||
.filter(|indexes| !indexes.is_empty())
|
||||
.collect();
|
||||
sorted_sets.sort_by(|a, b| {
|
||||
b.len()
|
||||
.cmp(&a.len())
|
||||
.then_with(|| a.iter().next().cmp(&b.iter().next()))
|
||||
});
|
||||
|
||||
// Assign unique sequential names
|
||||
let patterns: Vec<IndexSetPattern> = sorted_sets
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, indexes)| IndexSetPattern {
|
||||
name: format!("MetricPattern{}", i + 1),
|
||||
indexes,
|
||||
})
|
||||
.collect();
|
||||
|
||||
(used_indexes, patterns)
|
||||
}
|
||||
|
||||
fn collect_indexes_from_tree(
|
||||
node: &TreeNode,
|
||||
used_indexes: &mut BTreeSet<Index>,
|
||||
unique_index_sets: &mut BTreeSet<BTreeSet<Index>>,
|
||||
) {
|
||||
match node {
|
||||
TreeNode::Leaf(leaf) => {
|
||||
used_indexes.extend(leaf.indexes().iter().cloned());
|
||||
unique_index_sets.insert(leaf.indexes().clone());
|
||||
}
|
||||
TreeNode::Branch(children) => {
|
||||
for child in children.values() {
|
||||
collect_indexes_from_tree(child, used_indexes, unique_index_sets);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the metric base for a pattern instance by analyzing all leaf descendants.
|
||||
///
|
||||
/// For root-level instances (no common prefix/suffix among leaves), returns empty string.
|
||||
/// For cohort-level instances, returns the common prefix or suffix among all leaves.
|
||||
pub fn get_pattern_instance_base(node: &TreeNode) -> String {
|
||||
let leaf_names = get_all_leaf_names(node);
|
||||
if leaf_names.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
// First try to find a common prefix
|
||||
let common_prefix = find_common_prefix_at_underscore(&leaf_names);
|
||||
if !common_prefix.is_empty() {
|
||||
return common_prefix.trim_end_matches('_').to_string();
|
||||
}
|
||||
|
||||
// If no common prefix, try to find a common suffix
|
||||
let common_suffix = find_common_suffix_at_underscore(&leaf_names);
|
||||
if !common_suffix.is_empty() {
|
||||
return common_suffix.trim_start_matches('_').to_string();
|
||||
}
|
||||
|
||||
String::new()
|
||||
}
|
||||
|
||||
/// Find the longest common prefix at an underscore boundary.
|
||||
fn find_common_prefix_at_underscore(names: &[String]) -> String {
|
||||
if names.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let first = &names[0];
|
||||
if first.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
// Find character-by-character common prefix
|
||||
let mut prefix_len = 0;
|
||||
for (i, ch) in first.chars().enumerate() {
|
||||
if names.iter().all(|n| n.chars().nth(i) == Some(ch)) {
|
||||
prefix_len = i + 1;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if prefix_len == 0 {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let raw_prefix = &first[..prefix_len];
|
||||
|
||||
// If raw_prefix exactly matches a leaf name, it's a complete metric name.
|
||||
// In this case, return it with trailing underscore (will be trimmed by caller).
|
||||
if names.iter().any(|n| n == raw_prefix) {
|
||||
return format!("{}_", raw_prefix);
|
||||
}
|
||||
|
||||
// Find the last underscore position to get a clean boundary
|
||||
if let Some(last_underscore) = raw_prefix.rfind('_')
|
||||
&& last_underscore > 0
|
||||
{
|
||||
let clean_prefix = &first[..=last_underscore];
|
||||
// Verify this still works for all names
|
||||
if names.iter().all(|n| n.starts_with(clean_prefix)) {
|
||||
return clean_prefix.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
// If no underscore boundary works, check if full prefix ends at underscore
|
||||
if raw_prefix.ends_with('_') {
|
||||
return raw_prefix.to_string();
|
||||
}
|
||||
|
||||
String::new()
|
||||
}
|
||||
|
||||
/// Find the longest common suffix at an underscore boundary.
|
||||
fn find_common_suffix_at_underscore(names: &[String]) -> String {
|
||||
// Reverse strings, find common prefix, reverse result
|
||||
let reversed: Vec<String> = names.iter().map(|s| s.chars().rev().collect()).collect();
|
||||
let prefix = find_common_prefix_at_underscore(&reversed);
|
||||
prefix.chars().rev().collect()
|
||||
}
|
||||
|
||||
/// Infer the accumulated name for a child node based on a descendant leaf name.
|
||||
pub fn infer_accumulated_name(parent_acc: &str, field_name: &str, descendant_leaf: &str) -> String {
|
||||
if let Some(pos) = descendant_leaf.find(field_name) {
|
||||
if pos == 0 {
|
||||
return field_name.to_string();
|
||||
}
|
||||
if pos > 0 && descendant_leaf.chars().nth(pos - 1) == Some('_') {
|
||||
return if parent_acc.is_empty() {
|
||||
field_name.to_string()
|
||||
} else {
|
||||
format!("{}_{}", parent_acc, field_name)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if parent_acc.is_empty() {
|
||||
field_name.to_string()
|
||||
} else {
|
||||
format!("{}_{}", parent_acc, field_name)
|
||||
}
|
||||
}
|
||||
|
||||
/// Get fields with child field information for generic pattern lookup.
|
||||
pub fn get_fields_with_child_info(
|
||||
children: &BTreeMap<String, TreeNode>,
|
||||
parent_name: &str,
|
||||
pattern_lookup: &HashMap<Vec<PatternField>, String>,
|
||||
) -> Vec<(PatternField, Option<Vec<PatternField>>)> {
|
||||
children
|
||||
.iter()
|
||||
.map(|(name, node)| {
|
||||
let (rust_type, json_type, indexes, child_fields) = match node {
|
||||
TreeNode::Leaf(leaf) => (
|
||||
leaf.value_type().to_string(),
|
||||
schema_to_json_type(&leaf.schema),
|
||||
leaf.indexes().clone(),
|
||||
None,
|
||||
),
|
||||
TreeNode::Branch(grandchildren) => {
|
||||
let child_fields = get_node_fields(grandchildren, pattern_lookup);
|
||||
let pattern_name = pattern_lookup
|
||||
.get(&child_fields)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| child_type_name(parent_name, name));
|
||||
(
|
||||
pattern_name.clone(),
|
||||
pattern_name,
|
||||
BTreeSet::new(),
|
||||
Some(child_fields),
|
||||
)
|
||||
}
|
||||
};
|
||||
(
|
||||
PatternField {
|
||||
name: name.clone(),
|
||||
rust_type,
|
||||
json_type,
|
||||
indexes,
|
||||
type_param: None,
|
||||
},
|
||||
child_fields,
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
//! JavaScript language syntax implementation.
|
||||
|
||||
use crate::{FieldNamePosition, GenericSyntax, LanguageSyntax, to_camel_case, to_pascal_case};
|
||||
|
||||
/// JavaScript-specific code generation syntax.
|
||||
pub struct JavaScriptSyntax;
|
||||
|
||||
impl LanguageSyntax for JavaScriptSyntax {
|
||||
fn field_name(&self, name: &str) -> String {
|
||||
to_camel_case(name)
|
||||
}
|
||||
|
||||
fn path_expr(&self, base_var: &str, suffix: &str) -> String {
|
||||
// Convert base_var to camelCase for JavaScript
|
||||
let var_name = to_camel_case(base_var);
|
||||
format!("`${{{}}}{}`", var_name, suffix)
|
||||
}
|
||||
|
||||
fn position_expr(&self, pos: &FieldNamePosition, base_var: &str) -> String {
|
||||
// Convert base_var to camelCase for JavaScript
|
||||
let var_name = to_camel_case(base_var);
|
||||
match pos {
|
||||
FieldNamePosition::Append(s) => {
|
||||
// Use helper _m(acc, suffix) to build metric name
|
||||
// e.g., _m(acc, "cap") produces: acc ? `${acc}_cap` : 'cap'
|
||||
if let Some(suffix) = s.strip_prefix('_') {
|
||||
format!("_m({}, '{}')", var_name, suffix)
|
||||
} else {
|
||||
format!("`${{{}}}{}`", var_name, s)
|
||||
}
|
||||
}
|
||||
FieldNamePosition::Prepend(s) => {
|
||||
// Handle empty acc case for prepend
|
||||
if let Some(prefix) = s.strip_suffix('_') {
|
||||
format!(
|
||||
"({} ? `{}${{{}}}` : '{}')",
|
||||
var_name, s, var_name, prefix
|
||||
)
|
||||
} else {
|
||||
format!("`{}${{{}}}`", s, var_name)
|
||||
}
|
||||
}
|
||||
FieldNamePosition::Identity => var_name,
|
||||
FieldNamePosition::SetBase(s) => format!("'{}'", s),
|
||||
}
|
||||
}
|
||||
|
||||
fn constructor(&self, type_name: &str, path_expr: &str) -> String {
|
||||
format!("create{}(client, {})", type_name, path_expr)
|
||||
}
|
||||
|
||||
fn field_init(&self, indent: &str, name: &str, _type_ann: &str, value: &str) -> String {
|
||||
// JavaScript uses object literal syntax; type is in JSDoc, not in assignment
|
||||
format!("{}{}: {},", indent, name, value)
|
||||
}
|
||||
|
||||
fn generic_syntax(&self) -> GenericSyntax {
|
||||
GenericSyntax::JAVASCRIPT
|
||||
}
|
||||
|
||||
fn struct_header(&self, name: &str, generic_params: &str, doc: Option<&str>) -> String {
|
||||
let mut result = String::new();
|
||||
if let Some(doc) = doc {
|
||||
result.push_str(&format!("/** {} */\n", doc));
|
||||
}
|
||||
// JavaScript uses factory functions that return object literals
|
||||
result.push_str(&format!(
|
||||
"function create{}{}(client, basePath) {{\n return {{\n",
|
||||
name, generic_params
|
||||
));
|
||||
result
|
||||
}
|
||||
|
||||
fn struct_footer(&self) -> String {
|
||||
" };\n}\n".to_string()
|
||||
}
|
||||
|
||||
fn constructor_header(&self, _params: &str) -> String {
|
||||
// JavaScript factory functions don't have a separate constructor
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn constructor_footer(&self) -> String {
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn field_declaration(&self, indent: &str, _name: &str, type_ann: &str) -> String {
|
||||
// JSDoc property declaration
|
||||
format!("{}/** @type {{{}}} */\n", indent, type_ann)
|
||||
}
|
||||
|
||||
fn index_field_name(&self, index_name: &str) -> String {
|
||||
format!("by{}", to_pascal_case(index_name))
|
||||
}
|
||||
|
||||
fn string_literal(&self, value: &str) -> String {
|
||||
format!("'{}'", value)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
//! Language-specific syntax backends.
|
||||
//!
|
||||
//! This module contains implementations of the `LanguageSyntax` trait
|
||||
//! for each supported target language.
|
||||
|
||||
mod javascript;
|
||||
mod python;
|
||||
mod rust;
|
||||
|
||||
pub use javascript::JavaScriptSyntax;
|
||||
pub use python::PythonSyntax;
|
||||
pub use rust::RustSyntax;
|
||||
@@ -0,0 +1,89 @@
|
||||
//! Python language syntax implementation.
|
||||
|
||||
use crate::{FieldNamePosition, GenericSyntax, LanguageSyntax, escape_python_keyword, to_snake_case};
|
||||
|
||||
/// Python-specific code generation syntax.
|
||||
pub struct PythonSyntax;
|
||||
|
||||
impl LanguageSyntax for PythonSyntax {
|
||||
fn field_name(&self, name: &str) -> String {
|
||||
escape_python_keyword(&to_snake_case(name))
|
||||
}
|
||||
|
||||
fn path_expr(&self, base_var: &str, suffix: &str) -> String {
|
||||
format!("f'{{{{{}}}}}{}'", base_var, suffix)
|
||||
}
|
||||
|
||||
fn position_expr(&self, pos: &FieldNamePosition, base_var: &str) -> String {
|
||||
match pos {
|
||||
FieldNamePosition::Append(s) => {
|
||||
// Use helper _m(acc, suffix) to build metric name
|
||||
if let Some(suffix) = s.strip_prefix('_') {
|
||||
format!("_m({}, '{}')", base_var, suffix)
|
||||
} else {
|
||||
format!("f'{{{{{}}}}}{}'", base_var, s)
|
||||
}
|
||||
}
|
||||
FieldNamePosition::Prepend(s) => {
|
||||
// Handle empty acc case for prepend
|
||||
if let Some(prefix) = s.strip_suffix('_') {
|
||||
format!(
|
||||
"(f'{s}{{{{{base_var}}}}}' if {base_var} else '{prefix}')",
|
||||
s = s,
|
||||
base_var = base_var,
|
||||
prefix = prefix
|
||||
)
|
||||
} else {
|
||||
format!("f'{}{{{{{}}}}}'", s, base_var)
|
||||
}
|
||||
}
|
||||
FieldNamePosition::Identity => base_var.to_string(),
|
||||
FieldNamePosition::SetBase(s) => format!("'{}'", s),
|
||||
}
|
||||
}
|
||||
|
||||
fn constructor(&self, type_name: &str, path_expr: &str) -> String {
|
||||
format!("{}(client, {})", type_name, path_expr)
|
||||
}
|
||||
|
||||
fn field_init(&self, indent: &str, name: &str, type_ann: &str, value: &str) -> String {
|
||||
format!("{}self.{}: {} = {}", indent, name, type_ann, value)
|
||||
}
|
||||
|
||||
fn generic_syntax(&self) -> GenericSyntax {
|
||||
GenericSyntax::PYTHON
|
||||
}
|
||||
|
||||
fn struct_header(&self, name: &str, generic_params: &str, doc: Option<&str>) -> String {
|
||||
let mut result = format!("class {}{}:\n", name, generic_params);
|
||||
if let Some(doc) = doc {
|
||||
result.push_str(&format!(" \"\"\"{}\"\"\"\n", doc));
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn struct_footer(&self) -> String {
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn constructor_header(&self, params: &str) -> String {
|
||||
format!(" def __init__(self{}) -> None:\n", params)
|
||||
}
|
||||
|
||||
fn constructor_footer(&self) -> String {
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn field_declaration(&self, _indent: &str, _name: &str, _type_ann: &str) -> String {
|
||||
// Python uses __init__ for field declarations, so this is a no-op
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn index_field_name(&self, index_name: &str) -> String {
|
||||
format!("by_{}", to_snake_case(index_name))
|
||||
}
|
||||
|
||||
fn string_literal(&self, value: &str) -> String {
|
||||
format!("'{}'", value)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
//! Rust language syntax implementation.
|
||||
|
||||
use crate::{FieldNamePosition, GenericSyntax, LanguageSyntax, to_snake_case};
|
||||
|
||||
/// Rust-specific code generation syntax.
|
||||
pub struct RustSyntax;
|
||||
|
||||
impl LanguageSyntax for RustSyntax {
|
||||
fn field_name(&self, name: &str) -> String {
|
||||
to_snake_case(name)
|
||||
}
|
||||
|
||||
fn path_expr(&self, base_var: &str, suffix: &str) -> String {
|
||||
format!("format!(\"{{{}}}{}\")", base_var, suffix)
|
||||
}
|
||||
|
||||
fn position_expr(&self, pos: &FieldNamePosition, _base_var: &str) -> String {
|
||||
match pos {
|
||||
FieldNamePosition::Append(s) => {
|
||||
// Use helper _m(&acc, suffix) to build metric name
|
||||
if let Some(suffix) = s.strip_prefix('_') {
|
||||
format!("_m(&acc, \"{}\")", suffix)
|
||||
} else {
|
||||
format!("format!(\"{{acc}}{}\")", s)
|
||||
}
|
||||
}
|
||||
FieldNamePosition::Prepend(s) => {
|
||||
// Handle empty acc case for prepend
|
||||
if let Some(prefix) = s.strip_suffix('_') {
|
||||
format!(
|
||||
"if acc.is_empty() {{ \"{prefix}\".to_string() }} else {{ format!(\"{s}{{acc}}\") }}",
|
||||
prefix = prefix,
|
||||
s = s
|
||||
)
|
||||
} else {
|
||||
format!("format!(\"{}{{acc}}\")", s)
|
||||
}
|
||||
}
|
||||
FieldNamePosition::Identity => "acc.clone()".to_string(),
|
||||
FieldNamePosition::SetBase(base) => format!("\"{}\".to_string()", base),
|
||||
}
|
||||
}
|
||||
|
||||
fn constructor(&self, type_name: &str, path_expr: &str) -> String {
|
||||
format!("{}::new(client.clone(), {})", type_name, path_expr)
|
||||
}
|
||||
|
||||
fn field_init(&self, indent: &str, name: &str, _type_ann: &str, value: &str) -> String {
|
||||
// Rust struct initialization; type is in struct definition, not in init
|
||||
format!("{}{}: {},", indent, name, value)
|
||||
}
|
||||
|
||||
fn generic_syntax(&self) -> GenericSyntax {
|
||||
GenericSyntax::RUST
|
||||
}
|
||||
|
||||
fn struct_header(&self, name: &str, generic_params: &str, doc: Option<&str>) -> String {
|
||||
let mut result = String::new();
|
||||
if let Some(doc) = doc {
|
||||
result.push_str(&format!("/// {}\n", doc));
|
||||
}
|
||||
result.push_str(&format!("pub struct {}{} {{\n", name, generic_params));
|
||||
result
|
||||
}
|
||||
|
||||
fn struct_footer(&self) -> String {
|
||||
"}\n".to_string()
|
||||
}
|
||||
|
||||
fn constructor_header(&self, params: &str) -> String {
|
||||
format!(" pub fn new({}) -> Self {{\n Self {{\n", params)
|
||||
}
|
||||
|
||||
fn constructor_footer(&self) -> String {
|
||||
" }\n }\n".to_string()
|
||||
}
|
||||
|
||||
fn field_declaration(&self, indent: &str, name: &str, type_ann: &str) -> String {
|
||||
format!("{}pub {}: {},\n", indent, name, type_ann)
|
||||
}
|
||||
|
||||
fn index_field_name(&self, index_name: &str) -> String {
|
||||
format!("by_{}", to_snake_case(index_name))
|
||||
}
|
||||
|
||||
fn string_literal(&self, value: &str) -> String {
|
||||
format!("\"{}\".to_string()", value)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
//! Shared field generation logic.
|
||||
//!
|
||||
//! This module contains the core field generation logic that is shared
|
||||
//! across all language backends. The `LanguageSyntax` trait is used to
|
||||
//! abstract over language-specific formatting.
|
||||
|
||||
use std::fmt::Write;
|
||||
|
||||
use crate::{ClientMetadata, LanguageSyntax, PatternField, StructuralPattern};
|
||||
|
||||
/// Create a path suffix from a name.
|
||||
/// Adds `_` prefix only if the name doesn't already start with `_`.
|
||||
fn path_suffix(name: &str) -> String {
|
||||
if name.starts_with('_') {
|
||||
name.to_string()
|
||||
} else {
|
||||
format!("_{}", name)
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate a parameterized field using the language syntax.
|
||||
///
|
||||
/// This is used for pattern instances where fields use an accumulated
|
||||
/// metric name that's built up through the tree traversal.
|
||||
pub fn generate_parameterized_field<S: LanguageSyntax>(
|
||||
output: &mut String,
|
||||
syntax: &S,
|
||||
field: &PatternField,
|
||||
pattern: &StructuralPattern,
|
||||
metadata: &ClientMetadata,
|
||||
indent: &str,
|
||||
) {
|
||||
let field_name = syntax.field_name(&field.name);
|
||||
let type_ann = metadata.field_type_annotation(field, pattern.is_generic, None, syntax.generic_syntax());
|
||||
|
||||
// Compute path expression from field position
|
||||
let path_expr = pattern
|
||||
.get_field_position(&field.name)
|
||||
.map(|pos| syntax.position_expr(pos, "acc"))
|
||||
.unwrap_or_else(|| syntax.path_expr("acc", &path_suffix(&field.name)));
|
||||
|
||||
let value = if metadata.is_pattern_type(&field.rust_type) {
|
||||
syntax.constructor(&field.rust_type, &path_expr)
|
||||
} else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) {
|
||||
syntax.constructor(&accessor.name, &path_expr)
|
||||
} else {
|
||||
panic!(
|
||||
"Field '{}' has no matching pattern or index accessor. All metrics must be indexed.",
|
||||
field.name
|
||||
)
|
||||
};
|
||||
|
||||
writeln!(output, "{}", syntax.field_init(indent, &field_name, &type_ann, &value)).unwrap();
|
||||
}
|
||||
|
||||
/// Generate a tree-path field using the language syntax.
|
||||
///
|
||||
/// This is the fallback for non-parameterizable patterns where fields
|
||||
/// use a base path that's extended with the field name.
|
||||
pub fn generate_tree_path_field<S: LanguageSyntax>(
|
||||
output: &mut String,
|
||||
syntax: &S,
|
||||
field: &PatternField,
|
||||
metadata: &ClientMetadata,
|
||||
indent: &str,
|
||||
) {
|
||||
let field_name = syntax.field_name(&field.name);
|
||||
let type_ann = metadata.field_type_annotation(field, false, None, syntax.generic_syntax());
|
||||
let path_expr = syntax.path_expr("base_path", &path_suffix(&field.name));
|
||||
|
||||
let value = if metadata.is_pattern_type(&field.rust_type) {
|
||||
syntax.constructor(&field.rust_type, &path_expr)
|
||||
} else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) {
|
||||
syntax.constructor(&accessor.name, &path_expr)
|
||||
} else {
|
||||
panic!(
|
||||
"Field '{}' has no matching pattern or index accessor. All metrics must be indexed.",
|
||||
field.name
|
||||
)
|
||||
};
|
||||
|
||||
writeln!(output, "{}", syntax.field_init(indent, &field_name, &type_ann, &value)).unwrap();
|
||||
}
|
||||
|
||||
/// Generate a tree node field with a specific child node for pattern instance base detection.
|
||||
///
|
||||
/// This is used when generating tree nodes where we need to detect the pattern instance
|
||||
/// base from descendant leaf names.
|
||||
pub fn generate_tree_node_field<S: LanguageSyntax>(
|
||||
output: &mut String,
|
||||
syntax: &S,
|
||||
field: &PatternField,
|
||||
metadata: &ClientMetadata,
|
||||
indent: &str,
|
||||
child_name: &str,
|
||||
pattern_base: Option<&str>,
|
||||
) {
|
||||
let field_name = syntax.field_name(&field.name);
|
||||
let type_ann = metadata.field_type_annotation(field, false, None, syntax.generic_syntax());
|
||||
|
||||
let value = if metadata.is_pattern_type(&field.rust_type) {
|
||||
// Check if this pattern is parameterizable
|
||||
let pattern = metadata.find_pattern(&field.rust_type);
|
||||
let is_parameterizable = pattern.is_some_and(|p| p.is_parameterizable());
|
||||
|
||||
if is_parameterizable {
|
||||
if let Some(base) = pattern_base {
|
||||
// Use the detected metric base
|
||||
let path = syntax.string_literal(base);
|
||||
syntax.constructor(&field.rust_type, &path)
|
||||
} else {
|
||||
// Fallback to tree path
|
||||
let path_expr = syntax.path_expr("base_path", &path_suffix(child_name));
|
||||
syntax.constructor(&field.rust_type, &path_expr)
|
||||
}
|
||||
} else {
|
||||
let path_expr = syntax.path_expr("base_path", &path_suffix(child_name));
|
||||
syntax.constructor(&field.rust_type, &path_expr)
|
||||
}
|
||||
} else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) {
|
||||
let path_expr = syntax.path_expr("base_path", &path_suffix(child_name));
|
||||
syntax.constructor(&accessor.name, &path_expr)
|
||||
} else if field.is_branch() {
|
||||
// Non-pattern branch - instantiate the nested struct
|
||||
let path_expr = syntax.path_expr("base_path", &path_suffix(child_name));
|
||||
syntax.constructor(&field.rust_type, &path_expr)
|
||||
} else {
|
||||
// All metrics must be indexed
|
||||
panic!(
|
||||
"Field '{}' is a leaf with no index accessor. All metrics must be indexed.",
|
||||
field.name
|
||||
)
|
||||
};
|
||||
|
||||
writeln!(output, "{}", syntax.field_init(indent, &field_name, &type_ann, &value)).unwrap();
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
//! Shared code generation logic.
|
||||
//!
|
||||
//! This module contains generation functions that are parameterized by
|
||||
//! the `LanguageSyntax` trait, allowing them to work across all supported
|
||||
//! language backends.
|
||||
|
||||
mod fields;
|
||||
|
||||
pub use fields::*;
|
||||
@@ -0,0 +1,112 @@
|
||||
//! JavaScript API method generation.
|
||||
|
||||
use std::fmt::Write;
|
||||
|
||||
use crate::{Endpoint, Parameter, to_camel_case};
|
||||
|
||||
/// Generate API methods for the BrkClient class.
|
||||
pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
|
||||
for endpoint in endpoints {
|
||||
if !endpoint.should_generate() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let method_name = endpoint_to_method_name(endpoint);
|
||||
let return_type = endpoint.response_type.as_deref().unwrap_or("*");
|
||||
|
||||
writeln!(output, " /**").unwrap();
|
||||
if let Some(summary) = &endpoint.summary {
|
||||
writeln!(output, " * {}", summary).unwrap();
|
||||
}
|
||||
if let Some(desc) = &endpoint.description
|
||||
&& endpoint.summary.as_ref() != Some(desc)
|
||||
{
|
||||
writeln!(output, " * @description {}", desc).unwrap();
|
||||
}
|
||||
|
||||
for param in &endpoint.path_params {
|
||||
let desc = param.description.as_deref().unwrap_or("");
|
||||
writeln!(
|
||||
output,
|
||||
" * @param {{{}}} {} {}",
|
||||
param.param_type, param.name, desc
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
for param in &endpoint.query_params {
|
||||
let optional = if param.required { "" } else { "=" };
|
||||
let desc = param.description.as_deref().unwrap_or("");
|
||||
writeln!(
|
||||
output,
|
||||
" * @param {{{}{}}} [{}] {}",
|
||||
param.param_type, optional, param.name, desc
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
writeln!(output, " * @returns {{Promise<{}>}}", return_type).unwrap();
|
||||
writeln!(output, " */").unwrap();
|
||||
|
||||
let params = build_method_params(endpoint);
|
||||
writeln!(output, " async {}({}) {{", method_name, params).unwrap();
|
||||
|
||||
let path = build_path_template(&endpoint.path, &endpoint.path_params);
|
||||
|
||||
if endpoint.query_params.is_empty() {
|
||||
writeln!(output, " return this.get(`{}`);", path).unwrap();
|
||||
} else {
|
||||
writeln!(output, " const params = new URLSearchParams();").unwrap();
|
||||
for param in &endpoint.query_params {
|
||||
if param.required {
|
||||
writeln!(
|
||||
output,
|
||||
" params.set('{}', String({}));",
|
||||
param.name, param.name
|
||||
)
|
||||
.unwrap();
|
||||
} else {
|
||||
writeln!(
|
||||
output,
|
||||
" if ({} !== undefined) params.set('{}', String({}));",
|
||||
param.name, param.name, param.name
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
writeln!(output, " const query = params.toString();").unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" return this.get(`{}${{query ? '?' + query : ''}}`);",
|
||||
path
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
writeln!(output, " }}\n").unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn endpoint_to_method_name(endpoint: &Endpoint) -> String {
|
||||
to_camel_case(&endpoint.operation_name())
|
||||
}
|
||||
|
||||
fn build_method_params(endpoint: &Endpoint) -> String {
|
||||
let mut params = Vec::new();
|
||||
for param in &endpoint.path_params {
|
||||
params.push(param.name.clone());
|
||||
}
|
||||
for param in &endpoint.query_params {
|
||||
params.push(param.name.clone());
|
||||
}
|
||||
params.join(", ")
|
||||
}
|
||||
|
||||
fn build_path_template(path: &str, path_params: &[Parameter]) -> String {
|
||||
let mut result = path.to_string();
|
||||
for param in path_params {
|
||||
let placeholder = format!("{{{}}}", param.name);
|
||||
let interpolation = format!("${{{}}}", param.name);
|
||||
result = result.replace(&placeholder, &interpolation);
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
//! JavaScript base client and pattern factory generation.
|
||||
|
||||
use std::fmt::Write;
|
||||
|
||||
use brk_cohort::{
|
||||
AGE_RANGE_NAMES, AMOUNT_RANGE_NAMES, EPOCH_NAMES, GE_AMOUNT_NAMES, LT_AMOUNT_NAMES,
|
||||
MAX_AGE_NAMES, MIN_AGE_NAMES, SPENDABLE_TYPE_NAMES, TERM_NAMES, YEAR_NAMES,
|
||||
};
|
||||
use brk_types::{Index, PoolSlug, pools};
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
ClientMetadata, GenericSyntax, IndexSetPattern, JavaScriptSyntax, PatternField,
|
||||
StructuralPattern, VERSION, generate_parameterized_field, generate_tree_path_field,
|
||||
to_camel_case,
|
||||
};
|
||||
|
||||
/// Generate the base BrkClient class with HTTP functionality.
|
||||
pub fn generate_base_client(output: &mut String) {
|
||||
writeln!(
|
||||
output,
|
||||
r#"/**
|
||||
* @typedef {{Object}} BrkClientOptions
|
||||
* @property {{string}} baseUrl - Base URL for the API
|
||||
* @property {{number}} [timeout] - Request timeout in milliseconds
|
||||
*/
|
||||
|
||||
const _isBrowser = typeof window !== 'undefined' && 'caches' in window;
|
||||
const _runIdle = (/** @type {{VoidFunction}} */ fn) => (globalThis.requestIdleCallback ?? setTimeout)(fn);
|
||||
|
||||
/** @type {{Promise<Cache | null>}} */
|
||||
const _cachePromise = _isBrowser
|
||||
? caches.open('__BRK_CLIENT__').catch(() => null)
|
||||
: Promise.resolve(null);
|
||||
|
||||
/**
|
||||
* Custom error class for BRK client errors
|
||||
*/
|
||||
class BrkError extends Error {{
|
||||
/**
|
||||
* @param {{string}} message
|
||||
* @param {{number}} [status]
|
||||
*/
|
||||
constructor(message, status) {{
|
||||
super(message);
|
||||
this.name = 'BrkError';
|
||||
this.status = status;
|
||||
}}
|
||||
}}
|
||||
|
||||
/**
|
||||
* @template T
|
||||
* @typedef {{Object}} Endpoint
|
||||
* @property {{(onUpdate?: (value: T[]) => void) => Promise<T[]>}} get - Fetch all data points
|
||||
* @property {{(from?: number, to?: number, onUpdate?: (value: T[]) => void) => Promise<T[]>}} range - Fetch data in range
|
||||
* @property {{string}} path - The endpoint path
|
||||
*/
|
||||
|
||||
/**
|
||||
* @template T
|
||||
* @typedef {{Object}} MetricPattern
|
||||
* @property {{string}} name - The metric name
|
||||
* @property {{Partial<Record<Index, Endpoint<T>>>}} by - Index endpoints (lazy getters)
|
||||
* @property {{() => Index[]}} indexes - Get the list of available indexes
|
||||
* @property {{(index: Index) => Endpoint<T>|undefined}} get - Get an endpoint for a specific index
|
||||
*/
|
||||
|
||||
/** @typedef {{MetricPattern<unknown>}} AnyMetricPattern */
|
||||
|
||||
/**
|
||||
* Create an endpoint for a metric index.
|
||||
* @template T
|
||||
* @param {{BrkClientBase}} client
|
||||
* @param {{string}} name - The metric vec name
|
||||
* @param {{Index}} index - The index name
|
||||
* @returns {{Endpoint<T>}}
|
||||
*/
|
||||
function _endpoint(client, name, index) {{
|
||||
const p = `/api/metric/${{name}}/${{index}}`;
|
||||
return {{
|
||||
get: (onUpdate) => client.get(p, onUpdate),
|
||||
range: (from, to, onUpdate) => {{
|
||||
const params = new URLSearchParams();
|
||||
if (from !== undefined) params.set('from', String(from));
|
||||
if (to !== undefined) params.set('to', String(to));
|
||||
const query = params.toString();
|
||||
return client.get(query ? `${{p}}?${{query}}` : p, onUpdate);
|
||||
}},
|
||||
get path() {{ return p; }},
|
||||
}};
|
||||
}}
|
||||
|
||||
/**
|
||||
* Base HTTP client for making requests with caching support
|
||||
*/
|
||||
class BrkClientBase {{
|
||||
/**
|
||||
* @param {{BrkClientOptions|string}} options
|
||||
*/
|
||||
constructor(options) {{
|
||||
const isString = typeof options === 'string';
|
||||
this.baseUrl = isString ? options : options.baseUrl;
|
||||
this.timeout = isString ? 5000 : (options.timeout ?? 5000);
|
||||
}}
|
||||
|
||||
/**
|
||||
* Make a GET request with stale-while-revalidate caching
|
||||
* @template T
|
||||
* @param {{string}} path
|
||||
* @param {{(value: T) => void}} [onUpdate] - Called when data is available
|
||||
* @returns {{Promise<T>}}
|
||||
*/
|
||||
async get(path, onUpdate) {{
|
||||
const base = this.baseUrl.endsWith('/') ? this.baseUrl.slice(0, -1) : this.baseUrl;
|
||||
const url = `${{base}}${{path}}`;
|
||||
const cache = await _cachePromise;
|
||||
const cachedRes = await cache?.match(url);
|
||||
const cachedJson = cachedRes ? await cachedRes.json() : null;
|
||||
|
||||
if (cachedJson) onUpdate?.(cachedJson);
|
||||
if (!globalThis.navigator?.onLine) {{
|
||||
if (cachedJson) return cachedJson;
|
||||
throw new BrkError('Offline and no cached data available');
|
||||
}}
|
||||
|
||||
try {{
|
||||
const res = await fetch(url, {{ signal: AbortSignal.timeout(this.timeout) }});
|
||||
if (!res.ok) throw new BrkError(`HTTP ${{res.status}}`, res.status);
|
||||
if (cachedRes?.headers.get('ETag') === res.headers.get('ETag')) return cachedJson;
|
||||
|
||||
const cloned = res.clone();
|
||||
const json = await res.json();
|
||||
onUpdate?.(json);
|
||||
if (cache) _runIdle(() => cache.put(url, cloned));
|
||||
return json;
|
||||
}} catch (e) {{
|
||||
if (cachedJson) return cachedJson;
|
||||
throw e;
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
/**
|
||||
* Build metric name with optional prefix.
|
||||
* @param {{string}} acc - Accumulated prefix
|
||||
* @param {{string}} s - Metric suffix
|
||||
* @returns {{string}}
|
||||
*/
|
||||
const _m = (acc, s) => acc ? `${{acc}}_${{s}}` : s;
|
||||
|
||||
"#
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Generate static constants for the BrkClient class.
|
||||
pub fn generate_static_constants(output: &mut String) {
|
||||
fn instance_const<T: Serialize>(output: &mut String, name: &str, value: &T) {
|
||||
write_static_const(output, name, &serde_json::to_string_pretty(value).unwrap());
|
||||
}
|
||||
|
||||
fn instance_const_raw(output: &mut String, name: &str, value: &str) {
|
||||
writeln!(output, " {} = {};\n", name, value).unwrap();
|
||||
}
|
||||
|
||||
instance_const_raw(output, "VERSION", &format!("\"v{}\"", VERSION));
|
||||
|
||||
let indexes = Index::all();
|
||||
let indexes_json: Vec<&'static str> = indexes.iter().map(|i| i.serialize_long()).collect();
|
||||
instance_const(output, "INDEXES", &indexes_json);
|
||||
|
||||
let pools = pools();
|
||||
let mut sorted_pools: Vec<_> = pools.iter().collect();
|
||||
sorted_pools.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase()));
|
||||
let pool_map: std::collections::BTreeMap<PoolSlug, &'static str> =
|
||||
sorted_pools.iter().map(|p| (p.slug(), p.name)).collect();
|
||||
instance_const(output, "POOL_ID_TO_POOL_NAME", &pool_map);
|
||||
|
||||
fn instance_const_camel<T: Serialize>(output: &mut String, name: &str, value: &T) {
|
||||
let json_value: Value = serde_json::to_value(value).unwrap();
|
||||
let camel_value = camel_case_top_level_keys(json_value);
|
||||
write_static_const(output, name, &serde_json::to_string_pretty(&camel_value).unwrap());
|
||||
}
|
||||
|
||||
instance_const_camel(output, "TERM_NAMES", &TERM_NAMES);
|
||||
instance_const_camel(output, "EPOCH_NAMES", &EPOCH_NAMES);
|
||||
instance_const_camel(output, "YEAR_NAMES", &YEAR_NAMES);
|
||||
instance_const_camel(output, "SPENDABLE_TYPE_NAMES", &SPENDABLE_TYPE_NAMES);
|
||||
instance_const_camel(output, "AGE_RANGE_NAMES", &AGE_RANGE_NAMES);
|
||||
instance_const_camel(output, "MAX_AGE_NAMES", &MAX_AGE_NAMES);
|
||||
instance_const_camel(output, "MIN_AGE_NAMES", &MIN_AGE_NAMES);
|
||||
instance_const_camel(output, "AMOUNT_RANGE_NAMES", &AMOUNT_RANGE_NAMES);
|
||||
instance_const_camel(output, "GE_AMOUNT_NAMES", &GE_AMOUNT_NAMES);
|
||||
instance_const_camel(output, "LT_AMOUNT_NAMES", <_AMOUNT_NAMES);
|
||||
}
|
||||
|
||||
fn camel_case_top_level_keys(value: Value) -> Value {
|
||||
match value {
|
||||
Value::Object(map) => {
|
||||
let new_map: serde_json::Map<String, Value> = map
|
||||
.into_iter()
|
||||
.map(|(k, v)| (to_camel_case(&k), v))
|
||||
.collect();
|
||||
Value::Object(new_map)
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn indent_json_const(json: &str) -> String {
|
||||
json.lines()
|
||||
.enumerate()
|
||||
.map(|(i, line)| if i == 0 { line.to_string() } else { format!(" {}", line) })
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
fn write_static_const(output: &mut String, name: &str, json: &str) {
|
||||
writeln!(output, " {} = /** @type {{const}} */ ({});\n", name, indent_json_const(json)).unwrap();
|
||||
}
|
||||
|
||||
/// Generate index accessor factory functions.
|
||||
pub fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) {
|
||||
if patterns.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
writeln!(output, "// Index accessor factory functions\n").unwrap();
|
||||
|
||||
for pattern in patterns {
|
||||
let by_fields: Vec<String> = pattern
|
||||
.indexes
|
||||
.iter()
|
||||
.map(|idx| format!("{}: Endpoint<T>", idx.serialize_long()))
|
||||
.collect();
|
||||
let by_type = format!("{{ {} }}", by_fields.join(", "));
|
||||
|
||||
writeln!(output, "/**").unwrap();
|
||||
writeln!(output, " * @template T").unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" * @typedef {{{{ name: string, by: {}, indexes: () => Index[], get: (index: Index) => Endpoint<T>|undefined }}}} {}",
|
||||
by_type, pattern.name
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output, " */\n").unwrap();
|
||||
|
||||
writeln!(output, "/**").unwrap();
|
||||
writeln!(output, " * Create a {} accessor", pattern.name).unwrap();
|
||||
writeln!(output, " * @template T").unwrap();
|
||||
writeln!(output, " * @param {{BrkClientBase}} client").unwrap();
|
||||
writeln!(output, " * @param {{string}} name - The metric vec name").unwrap();
|
||||
writeln!(output, " * @returns {{{}<T>}}", pattern.name).unwrap();
|
||||
writeln!(output, " */").unwrap();
|
||||
writeln!(output, "function create{}(client, name) {{", pattern.name).unwrap();
|
||||
writeln!(output, " return {{").unwrap();
|
||||
writeln!(output, " name,").unwrap();
|
||||
writeln!(output, " by: {{").unwrap();
|
||||
|
||||
for (i, index) in pattern.indexes.iter().enumerate() {
|
||||
let index_name = index.serialize_long();
|
||||
let comma = if i < pattern.indexes.len() - 1 { "," } else { "" };
|
||||
writeln!(
|
||||
output,
|
||||
" get {}() {{ return _endpoint(client, name, '{}'); }}{}",
|
||||
index_name, index_name, comma
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
writeln!(output, " }},").unwrap();
|
||||
writeln!(output, " indexes() {{").unwrap();
|
||||
|
||||
write!(output, " return [").unwrap();
|
||||
for (i, index) in pattern.indexes.iter().enumerate() {
|
||||
if i > 0 {
|
||||
write!(output, ", ").unwrap();
|
||||
}
|
||||
write!(output, "'{}'", index.serialize_long()).unwrap();
|
||||
}
|
||||
writeln!(output, "];").unwrap();
|
||||
|
||||
writeln!(output, " }},").unwrap();
|
||||
writeln!(output, " get(index) {{").unwrap();
|
||||
writeln!(output, " if (this.indexes().includes(index)) {{").unwrap();
|
||||
writeln!(output, " return _endpoint(client, name, index);").unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output, " }};").unwrap();
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate structural pattern factory functions.
|
||||
pub fn generate_structural_patterns(
|
||||
output: &mut String,
|
||||
patterns: &[StructuralPattern],
|
||||
metadata: &ClientMetadata,
|
||||
) {
|
||||
if patterns.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
writeln!(output, "// Reusable structural pattern factories\n").unwrap();
|
||||
|
||||
for pattern in patterns {
|
||||
let is_parameterizable = pattern.is_parameterizable();
|
||||
|
||||
writeln!(output, "/**").unwrap();
|
||||
if pattern.is_generic {
|
||||
writeln!(output, " * @template T").unwrap();
|
||||
}
|
||||
writeln!(output, " * @typedef {{Object}} {}", pattern.name).unwrap();
|
||||
for field in &pattern.fields {
|
||||
let js_type = field_type_annotation(field, metadata, pattern.is_generic);
|
||||
writeln!(
|
||||
output,
|
||||
" * @property {{{}}} {}",
|
||||
js_type,
|
||||
to_camel_case(&field.name)
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
writeln!(output, " */\n").unwrap();
|
||||
|
||||
writeln!(output, "/**").unwrap();
|
||||
writeln!(output, " * Create a {} pattern node", pattern.name).unwrap();
|
||||
if pattern.is_generic {
|
||||
writeln!(output, " * @template T").unwrap();
|
||||
}
|
||||
writeln!(output, " * @param {{BrkClientBase}} client").unwrap();
|
||||
if is_parameterizable {
|
||||
writeln!(output, " * @param {{string}} acc - Accumulated metric name").unwrap();
|
||||
} else {
|
||||
writeln!(output, " * @param {{string}} basePath").unwrap();
|
||||
}
|
||||
let return_type = if pattern.is_generic {
|
||||
format!("{}<T>", pattern.name)
|
||||
} else {
|
||||
pattern.name.clone()
|
||||
};
|
||||
writeln!(output, " * @returns {{{}}}", return_type).unwrap();
|
||||
writeln!(output, " */").unwrap();
|
||||
|
||||
let param_name = if is_parameterizable { "acc" } else { "basePath" };
|
||||
writeln!(output, "function create{}(client, {}) {{", pattern.name, param_name).unwrap();
|
||||
writeln!(output, " return {{").unwrap();
|
||||
|
||||
let syntax = JavaScriptSyntax;
|
||||
for field in &pattern.fields {
|
||||
if is_parameterizable {
|
||||
generate_parameterized_field(output, &syntax, field, pattern, metadata, " ");
|
||||
} else {
|
||||
generate_tree_path_field(output, &syntax, field, metadata, " ");
|
||||
}
|
||||
}
|
||||
|
||||
writeln!(output, " }};").unwrap();
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn field_type_annotation(field: &PatternField, metadata: &ClientMetadata, is_generic: bool) -> String {
|
||||
metadata.field_type_annotation(field, is_generic, None, GenericSyntax::JAVASCRIPT)
|
||||
}
|
||||
|
||||
/// Get field type with specific generic value type.
|
||||
pub fn field_type_with_generic(
|
||||
field: &PatternField,
|
||||
metadata: &ClientMetadata,
|
||||
is_generic: bool,
|
||||
generic_value_type: Option<&str>,
|
||||
) -> String {
|
||||
metadata.field_type_annotation(field, is_generic, generic_value_type, GenericSyntax::JAVASCRIPT)
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
//! JavaScript client generation.
|
||||
//!
|
||||
//! This module generates a JavaScript + JSDoc client for the BRK API.
|
||||
|
||||
mod api;
|
||||
mod client;
|
||||
mod tree;
|
||||
mod types;
|
||||
|
||||
use std::{fmt::Write, fs, io, path::Path};
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use crate::{ClientMetadata, Endpoint, TypeSchemas, VERSION};
|
||||
|
||||
/// Generate JavaScript + JSDoc client from metadata and OpenAPI endpoints.
|
||||
///
|
||||
/// `output_path` is the full path to the output file (e.g., "modules/brk-client/index.js").
|
||||
pub fn generate_javascript_client(
|
||||
metadata: &ClientMetadata,
|
||||
endpoints: &[Endpoint],
|
||||
schemas: &TypeSchemas,
|
||||
output_path: &Path,
|
||||
) -> io::Result<()> {
|
||||
let mut output = String::new();
|
||||
|
||||
writeln!(output, "// Auto-generated BRK JavaScript client").unwrap();
|
||||
writeln!(output, "// Do not edit manually\n").unwrap();
|
||||
|
||||
types::generate_type_definitions(&mut output, schemas);
|
||||
client::generate_base_client(&mut output);
|
||||
client::generate_index_accessors(&mut output, &metadata.index_set_patterns);
|
||||
client::generate_structural_patterns(&mut output, &metadata.structural_patterns, metadata);
|
||||
tree::generate_tree_typedefs(&mut output, &metadata.catalog, metadata);
|
||||
tree::generate_main_client(&mut output, &metadata.catalog, metadata, endpoints);
|
||||
|
||||
fs::write(output_path, output)?;
|
||||
|
||||
// Update package.json version if it exists in the same directory
|
||||
if let Some(parent) = output_path.parent() {
|
||||
let package_json_path = parent.join("package.json");
|
||||
if package_json_path.exists() {
|
||||
update_package_json_version(&package_json_path)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn update_package_json_version(package_json_path: &Path) -> io::Result<()> {
|
||||
let content = fs::read_to_string(package_json_path)?;
|
||||
let mut package: serde_json::Value = serde_json::from_str(&content)
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
|
||||
|
||||
if let Some(obj) = package.as_object_mut() {
|
||||
obj.insert("version".to_string(), json!(VERSION));
|
||||
}
|
||||
|
||||
let updated = serde_json::to_string_pretty(&package)
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
|
||||
|
||||
fs::write(package_json_path, updated + "\n")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
//! JavaScript tree structure generation.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::fmt::Write;
|
||||
|
||||
use brk_types::TreeNode;
|
||||
|
||||
use crate::{
|
||||
ClientMetadata, Endpoint, PatternField, child_type_name, get_fields_with_child_info,
|
||||
get_first_leaf_name, get_node_fields, get_pattern_instance_base, infer_accumulated_name,
|
||||
to_camel_case,
|
||||
};
|
||||
|
||||
use super::api::generate_api_methods;
|
||||
use super::client::{field_type_with_generic, generate_static_constants};
|
||||
|
||||
/// Generate JSDoc typedefs for the catalog tree.
|
||||
pub fn generate_tree_typedefs(output: &mut String, catalog: &TreeNode, metadata: &ClientMetadata) {
|
||||
writeln!(output, "// Catalog tree typedefs\n").unwrap();
|
||||
|
||||
let pattern_lookup = metadata.pattern_lookup();
|
||||
let mut generated = HashSet::new();
|
||||
generate_tree_typedef(
|
||||
output,
|
||||
"CatalogTree",
|
||||
catalog,
|
||||
&pattern_lookup,
|
||||
metadata,
|
||||
&mut generated,
|
||||
);
|
||||
}
|
||||
|
||||
fn generate_tree_typedef(
|
||||
output: &mut String,
|
||||
name: &str,
|
||||
node: &TreeNode,
|
||||
pattern_lookup: &std::collections::HashMap<Vec<PatternField>, String>,
|
||||
metadata: &ClientMetadata,
|
||||
generated: &mut HashSet<String>,
|
||||
) {
|
||||
let TreeNode::Branch(children) = node else {
|
||||
return;
|
||||
};
|
||||
|
||||
let fields_with_child_info = get_fields_with_child_info(children, name, pattern_lookup);
|
||||
let fields: Vec<PatternField> = fields_with_child_info
|
||||
.iter()
|
||||
.map(|(f, _)| f.clone())
|
||||
.collect();
|
||||
|
||||
if pattern_lookup.contains_key(&fields)
|
||||
&& pattern_lookup.get(&fields) != Some(&name.to_string())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if generated.contains(name) {
|
||||
return;
|
||||
}
|
||||
generated.insert(name.to_string());
|
||||
|
||||
writeln!(output, "/**").unwrap();
|
||||
writeln!(output, " * @typedef {{Object}} {}", name).unwrap();
|
||||
|
||||
for (field, child_fields) in &fields_with_child_info {
|
||||
let generic_value_type = child_fields
|
||||
.as_ref()
|
||||
.and_then(|cf| metadata.get_type_param(cf))
|
||||
.map(String::as_str);
|
||||
let js_type = field_type_with_generic(field, metadata, false, generic_value_type);
|
||||
writeln!(
|
||||
output,
|
||||
" * @property {{{}}} {}",
|
||||
js_type,
|
||||
to_camel_case(&field.name)
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
writeln!(output, " */\n").unwrap();
|
||||
|
||||
for (child_name, child_node) in children {
|
||||
if let TreeNode::Branch(grandchildren) = child_node {
|
||||
let child_fields = get_node_fields(grandchildren, pattern_lookup);
|
||||
if !pattern_lookup.contains_key(&child_fields) {
|
||||
let child_type = child_type_name(name, child_name);
|
||||
generate_tree_typedef(
|
||||
output,
|
||||
&child_type,
|
||||
child_node,
|
||||
pattern_lookup,
|
||||
metadata,
|
||||
generated,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate the main BrkClient class.
|
||||
pub fn generate_main_client(
|
||||
output: &mut String,
|
||||
catalog: &TreeNode,
|
||||
metadata: &ClientMetadata,
|
||||
endpoints: &[Endpoint],
|
||||
) {
|
||||
let pattern_lookup = metadata.pattern_lookup();
|
||||
|
||||
writeln!(output, "/**").unwrap();
|
||||
writeln!(output, " * Main BRK client with catalog tree and API methods").unwrap();
|
||||
writeln!(output, " * @extends BrkClientBase").unwrap();
|
||||
writeln!(output, " */").unwrap();
|
||||
writeln!(output, "class BrkClient extends BrkClientBase {{").unwrap();
|
||||
|
||||
generate_static_constants(output);
|
||||
|
||||
writeln!(output, " /**").unwrap();
|
||||
writeln!(output, " * @param {{BrkClientOptions|string}} options").unwrap();
|
||||
writeln!(output, " */").unwrap();
|
||||
writeln!(output, " constructor(options) {{").unwrap();
|
||||
writeln!(output, " super(options);").unwrap();
|
||||
writeln!(output, " /** @type {{CatalogTree}} */").unwrap();
|
||||
writeln!(output, " this.tree = this._buildTree('');").unwrap();
|
||||
writeln!(output, " }}\n").unwrap();
|
||||
|
||||
writeln!(output, " /**").unwrap();
|
||||
writeln!(output, " * @private").unwrap();
|
||||
writeln!(output, " * @param {{string}} basePath").unwrap();
|
||||
writeln!(output, " * @returns {{CatalogTree}}").unwrap();
|
||||
writeln!(output, " */").unwrap();
|
||||
writeln!(output, " _buildTree(basePath) {{").unwrap();
|
||||
writeln!(output, " return {{").unwrap();
|
||||
generate_tree_initializer(output, catalog, "", 3, &pattern_lookup, metadata);
|
||||
writeln!(output, " }};").unwrap();
|
||||
writeln!(output, " }}\n").unwrap();
|
||||
|
||||
generate_api_methods(output, endpoints);
|
||||
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
|
||||
writeln!(output, "export {{ BrkClient, BrkClientBase, BrkError }};").unwrap();
|
||||
}
|
||||
|
||||
fn generate_tree_initializer(
|
||||
output: &mut String,
|
||||
node: &TreeNode,
|
||||
accumulated_name: &str,
|
||||
indent: usize,
|
||||
pattern_lookup: &std::collections::HashMap<Vec<PatternField>, String>,
|
||||
metadata: &ClientMetadata,
|
||||
) {
|
||||
let indent_str = " ".repeat(indent);
|
||||
|
||||
if let TreeNode::Branch(children) = node {
|
||||
for (i, (child_name, child_node)) in children.iter().enumerate() {
|
||||
let field_name = to_camel_case(child_name);
|
||||
let comma = if i < children.len() - 1 { "," } else { "" };
|
||||
|
||||
match child_node {
|
||||
TreeNode::Leaf(leaf) => {
|
||||
let accessor = metadata
|
||||
.find_index_set_pattern(leaf.indexes())
|
||||
.unwrap_or_else(|| {
|
||||
panic!(
|
||||
"Metric '{}' has no matching index pattern. All metrics must be indexed.",
|
||||
leaf.name()
|
||||
)
|
||||
});
|
||||
writeln!(
|
||||
output,
|
||||
"{}{}: create{}(this, '{}'){}",
|
||||
indent_str, field_name, accessor.name, leaf.name(), comma
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
TreeNode::Branch(grandchildren) => {
|
||||
let child_fields = get_node_fields(grandchildren, pattern_lookup);
|
||||
if let Some(pattern_name) = pattern_lookup.get(&child_fields) {
|
||||
let pattern = metadata
|
||||
.structural_patterns
|
||||
.iter()
|
||||
.find(|p| &p.name == pattern_name);
|
||||
let is_parameterizable =
|
||||
pattern.map(|p| p.is_parameterizable()).unwrap_or(false);
|
||||
|
||||
let arg = if is_parameterizable {
|
||||
get_pattern_instance_base(child_node)
|
||||
} else if accumulated_name.is_empty() {
|
||||
format!("/{}", child_name)
|
||||
} else {
|
||||
format!("{}/{}", accumulated_name, child_name)
|
||||
};
|
||||
|
||||
writeln!(
|
||||
output,
|
||||
"{}{}: create{}(this, '{}'){}",
|
||||
indent_str, field_name, pattern_name, arg, comma
|
||||
)
|
||||
.unwrap();
|
||||
} else {
|
||||
let child_acc =
|
||||
infer_child_accumulated_name(child_node, accumulated_name, child_name);
|
||||
writeln!(output, "{}{}: {{", indent_str, field_name).unwrap();
|
||||
generate_tree_initializer(
|
||||
output,
|
||||
child_node,
|
||||
&child_acc,
|
||||
indent + 1,
|
||||
pattern_lookup,
|
||||
metadata,
|
||||
);
|
||||
writeln!(output, "{}}}{}", indent_str, comma).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn infer_child_accumulated_name(node: &TreeNode, parent_acc: &str, field_name: &str) -> String {
|
||||
let leaf_name = get_first_leaf_name(node).unwrap_or_default();
|
||||
infer_accumulated_name(parent_acc, field_name, &leaf_name)
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
//! JavaScript type definitions generation.
|
||||
|
||||
use std::fmt::Write;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{TypeSchemas, ref_to_type_name, to_camel_case};
|
||||
|
||||
/// Generate JSDoc type definitions from OpenAPI schemas.
|
||||
pub fn generate_type_definitions(output: &mut String, schemas: &TypeSchemas) {
|
||||
if schemas.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
writeln!(output, "// Type definitions\n").unwrap();
|
||||
|
||||
for (name, schema) in schemas {
|
||||
let js_type = schema_to_js_type(schema, Some(name));
|
||||
|
||||
if is_primitive_alias(schema) {
|
||||
writeln!(output, "/** @typedef {{{}}} {} */", js_type, name).unwrap();
|
||||
} else if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) {
|
||||
writeln!(output, "/**").unwrap();
|
||||
writeln!(output, " * @typedef {{Object}} {}", name).unwrap();
|
||||
for (prop_name, prop_schema) in props {
|
||||
let prop_type = schema_to_js_type(prop_schema, Some(name));
|
||||
let required = schema
|
||||
.get("required")
|
||||
.and_then(|r| r.as_array())
|
||||
.map(|arr| arr.iter().any(|v| v.as_str() == Some(prop_name)))
|
||||
.unwrap_or(false);
|
||||
let optional = if required { "" } else { "=" };
|
||||
let safe_name = to_camel_case(prop_name);
|
||||
writeln!(
|
||||
output,
|
||||
" * @property {{{}{}}} {}",
|
||||
prop_type, optional, safe_name
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
writeln!(output, " */").unwrap();
|
||||
} else {
|
||||
writeln!(output, "/** @typedef {{{}}} {} */", js_type, name).unwrap();
|
||||
}
|
||||
}
|
||||
writeln!(output).unwrap();
|
||||
}
|
||||
|
||||
fn is_primitive_alias(schema: &Value) -> bool {
|
||||
schema.get("properties").is_none()
|
||||
&& schema.get("items").is_none()
|
||||
&& schema.get("anyOf").is_none()
|
||||
&& schema.get("oneOf").is_none()
|
||||
&& schema.get("enum").is_none()
|
||||
}
|
||||
|
||||
fn json_type_to_js(ty: &str, schema: &Value, current_type: Option<&str>) -> String {
|
||||
match ty {
|
||||
"integer" | "number" => "number".to_string(),
|
||||
"boolean" => "boolean".to_string(),
|
||||
"string" => "string".to_string(),
|
||||
"null" => "null".to_string(),
|
||||
"array" => {
|
||||
let item_type = schema
|
||||
.get("items")
|
||||
.map(|s| schema_to_js_type(s, current_type))
|
||||
.unwrap_or_else(|| "*".to_string());
|
||||
format!("{}[]", item_type)
|
||||
}
|
||||
"object" => {
|
||||
if let Some(add_props) = schema.get("additionalProperties") {
|
||||
let value_type = schema_to_js_type(add_props, current_type);
|
||||
return format!("{{ [key: string]: {} }}", value_type);
|
||||
}
|
||||
"Object".to_string()
|
||||
}
|
||||
_ => "*".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a JSON schema to a JavaScript type string.
|
||||
pub fn schema_to_js_type(schema: &Value, current_type: Option<&str>) -> String {
|
||||
if let Some(all_of) = schema.get("allOf").and_then(|v| v.as_array()) {
|
||||
for item in all_of {
|
||||
let resolved = schema_to_js_type(item, current_type);
|
||||
if resolved != "*" {
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref_path) = schema.get("$ref").and_then(|r| r.as_str()) {
|
||||
return ref_to_type_name(ref_path).unwrap_or("*").to_string();
|
||||
}
|
||||
|
||||
if let Some(enum_values) = schema.get("enum").and_then(|e| e.as_array()) {
|
||||
let literals: Vec<String> = enum_values
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.map(|s| format!("\"{}\"", s))
|
||||
.collect();
|
||||
if !literals.is_empty() {
|
||||
return format!("({})", literals.join("|"));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ty) = schema.get("type") {
|
||||
if let Some(type_array) = ty.as_array() {
|
||||
let types: Vec<String> = type_array
|
||||
.iter()
|
||||
.filter_map(|t| t.as_str())
|
||||
.filter(|t| *t != "null")
|
||||
.map(|t| json_type_to_js(t, schema, current_type))
|
||||
.collect();
|
||||
let has_null = type_array.iter().any(|t| t.as_str() == Some("null"));
|
||||
|
||||
if types.len() == 1 {
|
||||
let base_type = &types[0];
|
||||
return if has_null {
|
||||
format!("?{}", base_type)
|
||||
} else {
|
||||
base_type.clone()
|
||||
};
|
||||
} else if !types.is_empty() {
|
||||
let union = format!("({})", types.join("|"));
|
||||
return if has_null {
|
||||
format!("?{}", union)
|
||||
} else {
|
||||
union
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ty_str) = ty.as_str() {
|
||||
return json_type_to_js(ty_str, schema, current_type);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(variants) = schema
|
||||
.get("anyOf")
|
||||
.or_else(|| schema.get("oneOf"))
|
||||
.and_then(|v| v.as_array())
|
||||
{
|
||||
let types: Vec<String> = variants
|
||||
.iter()
|
||||
.map(|v| schema_to_js_type(v, current_type))
|
||||
.collect();
|
||||
let filtered: Vec<_> = types.iter().filter(|t| *t != "*").collect();
|
||||
if !filtered.is_empty() {
|
||||
return format!(
|
||||
"({})",
|
||||
filtered
|
||||
.iter()
|
||||
.map(|s| s.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join("|")
|
||||
);
|
||||
}
|
||||
return format!("({})", types.join("|"));
|
||||
}
|
||||
|
||||
if let Some(format) = schema.get("format").and_then(|f| f.as_str()) {
|
||||
return match format {
|
||||
"int32" | "int64" => "number".to_string(),
|
||||
"float" | "double" => "number".to_string(),
|
||||
"date" | "date-time" => "string".to_string(),
|
||||
_ => "*".to_string(),
|
||||
};
|
||||
}
|
||||
|
||||
"*".to_string()
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
//! Code generators for client libraries.
|
||||
//!
|
||||
//! Each language has its own submodule with focused files:
|
||||
//! - `types.rs` - Type definitions
|
||||
//! - `client.rs` - Base client and pattern factories
|
||||
//! - `tree.rs` - Tree structure generation
|
||||
//! - `api.rs` - API method generation
|
||||
//! - `mod.rs` - Entry point
|
||||
|
||||
pub mod javascript;
|
||||
pub mod python;
|
||||
pub mod rust;
|
||||
|
||||
pub use javascript::generate_javascript_client;
|
||||
pub use python::generate_python_client;
|
||||
pub use rust::generate_rust_client;
|
||||
@@ -0,0 +1,151 @@
|
||||
//! Python API method generation.
|
||||
|
||||
use std::fmt::Write;
|
||||
|
||||
use crate::{Endpoint, Parameter, escape_python_keyword, to_snake_case};
|
||||
|
||||
use super::client::generate_class_constants;
|
||||
use super::types::js_type_to_python;
|
||||
|
||||
/// Generate the main client class
|
||||
pub fn generate_main_client(output: &mut String, endpoints: &[Endpoint]) {
|
||||
writeln!(output, "class BrkClient(BrkClientBase):").unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" \"\"\"Main BRK client with catalog tree and API methods.\"\"\""
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output).unwrap();
|
||||
|
||||
// Generate class-level constants
|
||||
generate_class_constants(output);
|
||||
|
||||
writeln!(
|
||||
output,
|
||||
" def __init__(self, base_url: str = 'http://localhost:3000', timeout: float = 30.0):"
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output, " super().__init__(base_url, timeout)").unwrap();
|
||||
writeln!(output, " self.tree = CatalogTree(self)").unwrap();
|
||||
writeln!(output).unwrap();
|
||||
|
||||
// Generate API methods
|
||||
generate_api_methods(output, endpoints);
|
||||
}
|
||||
|
||||
/// Generate API methods from OpenAPI endpoints
|
||||
pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
|
||||
for endpoint in endpoints {
|
||||
if !endpoint.should_generate() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let method_name = endpoint_to_method_name(endpoint);
|
||||
let return_type = endpoint
|
||||
.response_type
|
||||
.as_deref()
|
||||
.map(js_type_to_python)
|
||||
.unwrap_or_else(|| "Any".to_string());
|
||||
|
||||
// Build method signature
|
||||
let params = build_method_params(endpoint);
|
||||
writeln!(
|
||||
output,
|
||||
" def {}(self{}) -> {}:",
|
||||
method_name, params, return_type
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Docstring
|
||||
match (&endpoint.summary, &endpoint.description) {
|
||||
(Some(summary), Some(desc)) if summary != desc => {
|
||||
writeln!(output, " \"\"\"{}.", summary.trim_end_matches('.')).unwrap();
|
||||
writeln!(output).unwrap();
|
||||
writeln!(output, " {}\"\"\"", desc).unwrap();
|
||||
}
|
||||
(Some(summary), _) => {
|
||||
writeln!(output, " \"\"\"{}\"\"\"", summary).unwrap();
|
||||
}
|
||||
(None, Some(desc)) => {
|
||||
writeln!(output, " \"\"\"{}\"\"\"", desc).unwrap();
|
||||
}
|
||||
(None, None) => {}
|
||||
}
|
||||
|
||||
// Build path
|
||||
let path = build_path_template(&endpoint.path, &endpoint.path_params);
|
||||
|
||||
if endpoint.query_params.is_empty() {
|
||||
if endpoint.path_params.is_empty() {
|
||||
writeln!(output, " return self.get('{}')", path).unwrap();
|
||||
} else {
|
||||
writeln!(output, " return self.get(f'{}')", path).unwrap();
|
||||
}
|
||||
} else {
|
||||
writeln!(output, " params = []").unwrap();
|
||||
for param in &endpoint.query_params {
|
||||
// Use safe name for Python variable, original name for API query parameter
|
||||
let safe_name = escape_python_keyword(¶m.name);
|
||||
if param.required {
|
||||
writeln!(
|
||||
output,
|
||||
" params.append(f'{}={{{}}}')",
|
||||
param.name, safe_name
|
||||
)
|
||||
.unwrap();
|
||||
} else {
|
||||
writeln!(
|
||||
output,
|
||||
" if {} is not None: params.append(f'{}={{{}}}')",
|
||||
safe_name, param.name, safe_name
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
writeln!(output, " query = '&'.join(params)").unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" return self.get(f'{}{{\"?\" + query if query else \"\"}}')",
|
||||
path
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
writeln!(output).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn endpoint_to_method_name(endpoint: &Endpoint) -> String {
|
||||
to_snake_case(&endpoint.operation_name())
|
||||
}
|
||||
|
||||
fn build_method_params(endpoint: &Endpoint) -> String {
|
||||
let mut params = Vec::new();
|
||||
for param in &endpoint.path_params {
|
||||
let safe_name = escape_python_keyword(¶m.name);
|
||||
let py_type = js_type_to_python(¶m.param_type);
|
||||
params.push(format!(", {}: {}", safe_name, py_type));
|
||||
}
|
||||
for param in &endpoint.query_params {
|
||||
let safe_name = escape_python_keyword(¶m.name);
|
||||
let py_type = js_type_to_python(¶m.param_type);
|
||||
if param.required {
|
||||
params.push(format!(", {}: {}", safe_name, py_type));
|
||||
} else {
|
||||
params.push(format!(", {}: Optional[{}] = None", safe_name, py_type));
|
||||
}
|
||||
}
|
||||
params.join("")
|
||||
}
|
||||
|
||||
fn build_path_template(path: &str, path_params: &[Parameter]) -> String {
|
||||
let mut result = path.to_string();
|
||||
for param in path_params {
|
||||
let placeholder = format!("{{{}}}", param.name);
|
||||
// Use escaped name for Python variable interpolation in f-string
|
||||
let safe_name = escape_python_keyword(¶m.name);
|
||||
let interpolation = format!("{{{}}}", safe_name);
|
||||
result = result.replace(&placeholder, &interpolation);
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
//! Python base client and pattern factory generation.
|
||||
|
||||
use std::fmt::Write;
|
||||
|
||||
use brk_cohort::{
|
||||
AGE_RANGE_NAMES, AMOUNT_RANGE_NAMES, EPOCH_NAMES, GE_AMOUNT_NAMES, LT_AMOUNT_NAMES,
|
||||
MAX_AGE_NAMES, MIN_AGE_NAMES, SPENDABLE_TYPE_NAMES, TERM_NAMES, YEAR_NAMES,
|
||||
};
|
||||
use brk_types::{pools, Index};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::{
|
||||
ClientMetadata, GenericSyntax, IndexSetPattern, PatternField, PythonSyntax,
|
||||
StructuralPattern, VERSION, generate_parameterized_field, generate_tree_path_field,
|
||||
index_to_field_name,
|
||||
};
|
||||
|
||||
/// Generate class-level constants for the BrkClient class.
|
||||
pub fn generate_class_constants(output: &mut String) {
|
||||
fn class_const<T: Serialize>(output: &mut String, name: &str, value: &T) {
|
||||
let json = serde_json::to_string_pretty(value).unwrap();
|
||||
// Indent all lines for class body
|
||||
let indented = json
|
||||
.lines()
|
||||
.enumerate()
|
||||
.map(|(i, line)| {
|
||||
if i == 0 {
|
||||
format!(" {} = {}", name, line)
|
||||
} else {
|
||||
format!(" {}", line)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
writeln!(output, "{}\n", indented).unwrap();
|
||||
}
|
||||
|
||||
// VERSION
|
||||
writeln!(output, " VERSION = \"v{}\"\n", VERSION).unwrap();
|
||||
|
||||
// INDEXES
|
||||
let indexes = Index::all();
|
||||
let indexes_list: Vec<&str> = indexes.iter().map(|i| i.serialize_long()).collect();
|
||||
class_const(output, "INDEXES", &indexes_list);
|
||||
|
||||
// POOL_ID_TO_POOL_NAME
|
||||
let pools = pools();
|
||||
let mut sorted_pools: Vec<_> = pools.iter().collect();
|
||||
sorted_pools.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase()));
|
||||
let pool_map: std::collections::BTreeMap<String, &str> = sorted_pools
|
||||
.iter()
|
||||
.map(|p| (p.slug().to_string(), p.name))
|
||||
.collect();
|
||||
class_const(output, "POOL_ID_TO_POOL_NAME", &pool_map);
|
||||
|
||||
// Cohort names
|
||||
class_const(output, "TERM_NAMES", &TERM_NAMES);
|
||||
class_const(output, "EPOCH_NAMES", &EPOCH_NAMES);
|
||||
class_const(output, "YEAR_NAMES", &YEAR_NAMES);
|
||||
class_const(output, "SPENDABLE_TYPE_NAMES", &SPENDABLE_TYPE_NAMES);
|
||||
class_const(output, "AGE_RANGE_NAMES", &AGE_RANGE_NAMES);
|
||||
class_const(output, "MAX_AGE_NAMES", &MAX_AGE_NAMES);
|
||||
class_const(output, "MIN_AGE_NAMES", &MIN_AGE_NAMES);
|
||||
class_const(output, "AMOUNT_RANGE_NAMES", &AMOUNT_RANGE_NAMES);
|
||||
class_const(output, "GE_AMOUNT_NAMES", &GE_AMOUNT_NAMES);
|
||||
class_const(output, "LT_AMOUNT_NAMES", <_AMOUNT_NAMES);
|
||||
}
|
||||
|
||||
/// Generate the base BrkClient class with HTTP functionality
|
||||
pub fn generate_base_client(output: &mut String) {
|
||||
writeln!(
|
||||
output,
|
||||
r#"class BrkError(Exception):
|
||||
"""Custom error class for BRK client errors."""
|
||||
|
||||
def __init__(self, message: str, status: Optional[int] = None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
|
||||
|
||||
class BrkClientBase:
|
||||
"""Base HTTP client for making requests."""
|
||||
|
||||
def __init__(self, base_url: str, timeout: float = 30.0):
|
||||
self.base_url = base_url
|
||||
self.timeout = timeout
|
||||
self._client = httpx.Client(timeout=timeout)
|
||||
|
||||
def get(self, path: str) -> Any:
|
||||
"""Make a GET request."""
|
||||
try:
|
||||
base = self.base_url.rstrip('/')
|
||||
response = self._client.get(f"{{base}}{{path}}")
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise BrkError(f"HTTP error: {{e.response.status_code}}", e.response.status_code)
|
||||
except httpx.RequestError as e:
|
||||
raise BrkError(str(e))
|
||||
|
||||
def close(self):
|
||||
"""Close the HTTP client."""
|
||||
self._client.close()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.close()
|
||||
|
||||
|
||||
def _m(acc: str, s: str) -> str:
|
||||
"""Build metric name with optional prefix."""
|
||||
return f"{{acc}}_{{s}}" if acc else s
|
||||
|
||||
"#
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Generate the Endpoint class
|
||||
pub fn generate_endpoint_class(output: &mut String) {
|
||||
writeln!(
|
||||
output,
|
||||
r#"class Endpoint(Generic[T]):
|
||||
"""An endpoint for a specific metric + index combination."""
|
||||
|
||||
def __init__(self, client: BrkClientBase, name: str, index: str):
|
||||
self._client = client
|
||||
self._name = name
|
||||
self._index = index
|
||||
|
||||
def get(self) -> List[T]:
|
||||
"""Fetch all data points for this metric/index."""
|
||||
return self._client.get(self.path())
|
||||
|
||||
def range(self, from_val: Optional[int] = None, to_val: Optional[int] = None) -> List[T]:
|
||||
"""Fetch data points within a range."""
|
||||
params = []
|
||||
if from_val is not None:
|
||||
params.append(f"from={{from_val}}")
|
||||
if to_val is not None:
|
||||
params.append(f"to={{to_val}}")
|
||||
query = "&".join(params)
|
||||
p = self.path()
|
||||
return self._client.get(f"{{p}}?{{query}}" if query else p)
|
||||
|
||||
def path(self) -> str:
|
||||
"""Get the endpoint path."""
|
||||
return f"/api/metric/{{self._name}}/{{self._index}}"
|
||||
|
||||
|
||||
class MetricPattern(Protocol[T]):
|
||||
"""Protocol for metric patterns with different index sets."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""Get the metric name."""
|
||||
...
|
||||
|
||||
def indexes(self) -> List[str]:
|
||||
"""Get the list of available indexes for this metric."""
|
||||
...
|
||||
|
||||
def get(self, index: str) -> Optional[Endpoint[T]]:
|
||||
"""Get an endpoint for a specific index, if supported."""
|
||||
...
|
||||
|
||||
"#
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Generate index accessor classes
|
||||
pub fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) {
|
||||
if patterns.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
writeln!(output, "# Index accessor classes\n").unwrap();
|
||||
|
||||
for pattern in patterns {
|
||||
let by_class_name = format!("_{}By", pattern.name);
|
||||
|
||||
// Generate the By class with lazy endpoint methods
|
||||
writeln!(output, "class {}(Generic[T]):", by_class_name).unwrap();
|
||||
writeln!(output, " \"\"\"Index endpoint methods container.\"\"\"").unwrap();
|
||||
writeln!(output, " ").unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" def __init__(self, client: BrkClientBase, name: str):"
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output, " self._client = client").unwrap();
|
||||
writeln!(output, " self._name = name").unwrap();
|
||||
writeln!(output).unwrap();
|
||||
|
||||
// Generate methods for each index
|
||||
for index in &pattern.indexes {
|
||||
let method_name = index_to_field_name(index);
|
||||
let index_name = index.serialize_long();
|
||||
writeln!(output, " def {}(self) -> Endpoint[T]:", method_name).unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" return Endpoint(self._client, self._name, '{}')",
|
||||
index_name
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output).unwrap();
|
||||
}
|
||||
|
||||
// Generate the main accessor class
|
||||
writeln!(output, "class {}(Generic[T]):", pattern.name).unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" \"\"\"Index accessor for metrics with {} indexes.\"\"\"",
|
||||
pattern.indexes.len()
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output, " ").unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" def __init__(self, client: BrkClientBase, name: str):"
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output, " self._client = client").unwrap();
|
||||
writeln!(output, " self._name = name").unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" self.by: {}[T] = {}(client, name)",
|
||||
by_class_name, by_class_name
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output).unwrap();
|
||||
writeln!(output, " @property").unwrap();
|
||||
writeln!(output, " def name(self) -> str:").unwrap();
|
||||
writeln!(output, " \"\"\"Get the metric name.\"\"\"").unwrap();
|
||||
writeln!(output, " return self._name").unwrap();
|
||||
writeln!(output).unwrap();
|
||||
writeln!(output, " def indexes(self) -> List[str]:").unwrap();
|
||||
writeln!(output, " \"\"\"Get the list of available indexes.\"\"\"").unwrap();
|
||||
write!(output, " return [").unwrap();
|
||||
for (i, index) in pattern.indexes.iter().enumerate() {
|
||||
if i > 0 {
|
||||
write!(output, ", ").unwrap();
|
||||
}
|
||||
write!(output, "'{}'", index.serialize_long()).unwrap();
|
||||
}
|
||||
writeln!(output, "]").unwrap();
|
||||
writeln!(output).unwrap();
|
||||
|
||||
// Generate get(index) method
|
||||
writeln!(output, " def get(self, index: str) -> Optional[Endpoint[T]]:").unwrap();
|
||||
writeln!(output, " \"\"\"Get an endpoint for a specific index, if supported.\"\"\"").unwrap();
|
||||
for (i, index) in pattern.indexes.iter().enumerate() {
|
||||
let method_name = index_to_field_name(index);
|
||||
let index_name = index.serialize_long();
|
||||
if i == 0 {
|
||||
writeln!(output, " if index == '{}': return self.by.{}()", index_name, method_name).unwrap();
|
||||
} else {
|
||||
writeln!(output, " elif index == '{}': return self.by.{}()", index_name, method_name).unwrap();
|
||||
}
|
||||
}
|
||||
writeln!(output, " return None").unwrap();
|
||||
writeln!(output).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate structural pattern classes
|
||||
pub fn generate_structural_patterns(
|
||||
output: &mut String,
|
||||
patterns: &[StructuralPattern],
|
||||
metadata: &ClientMetadata,
|
||||
) {
|
||||
if patterns.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
writeln!(output, "# Reusable structural pattern classes\n").unwrap();
|
||||
|
||||
for pattern in patterns {
|
||||
let is_parameterizable = pattern.is_parameterizable();
|
||||
|
||||
// For generic patterns, inherit from Generic[T]
|
||||
if pattern.is_generic {
|
||||
writeln!(output, "class {}(Generic[T]):", pattern.name).unwrap();
|
||||
} else {
|
||||
writeln!(output, "class {}:", pattern.name).unwrap();
|
||||
}
|
||||
writeln!(
|
||||
output,
|
||||
" \"\"\"Pattern struct for repeated tree structure.\"\"\""
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output, " ").unwrap();
|
||||
|
||||
if is_parameterizable {
|
||||
writeln!(
|
||||
output,
|
||||
" def __init__(self, client: BrkClientBase, acc: str):"
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" \"\"\"Create pattern node with accumulated metric name.\"\"\""
|
||||
)
|
||||
.unwrap();
|
||||
} else {
|
||||
writeln!(
|
||||
output,
|
||||
" def __init__(self, client: BrkClientBase, base_path: str):"
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let syntax = PythonSyntax;
|
||||
for field in &pattern.fields {
|
||||
if is_parameterizable {
|
||||
generate_parameterized_field(output, &syntax, field, pattern, metadata, " ");
|
||||
} else {
|
||||
generate_tree_path_field(output, &syntax, field, metadata, " ");
|
||||
}
|
||||
}
|
||||
|
||||
writeln!(output).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
/// Get Python type annotation for a field with optional generic value type.
|
||||
pub fn field_type_with_generic(
|
||||
field: &PatternField,
|
||||
metadata: &ClientMetadata,
|
||||
is_generic: bool,
|
||||
generic_value_type: Option<&str>,
|
||||
) -> String {
|
||||
metadata.field_type_annotation(field, is_generic, generic_value_type, GenericSyntax::PYTHON)
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
//! Python client generation.
|
||||
//!
|
||||
//! This module generates a Python client with type hints for the BRK API.
|
||||
|
||||
mod api;
|
||||
mod client;
|
||||
mod tree;
|
||||
mod types;
|
||||
|
||||
use std::{fmt::Write, fs, io, path::Path};
|
||||
|
||||
use crate::{ClientMetadata, Endpoint, TypeSchemas};
|
||||
|
||||
/// Generate Python client from metadata and OpenAPI endpoints.
|
||||
///
|
||||
/// `output_path` is the full path to the output file (e.g., "packages/brk_client/__init__.py").
|
||||
pub fn generate_python_client(
|
||||
metadata: &ClientMetadata,
|
||||
endpoints: &[Endpoint],
|
||||
schemas: &TypeSchemas,
|
||||
output_path: &Path,
|
||||
) -> io::Result<()> {
|
||||
let mut output = String::new();
|
||||
|
||||
writeln!(output, "# Auto-generated BRK Python client").unwrap();
|
||||
writeln!(output, "# Do not edit manually\n").unwrap();
|
||||
writeln!(output, "from __future__ import annotations").unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
"from typing import TypeVar, Generic, Any, Optional, List, Literal, TypedDict, Final, Union, Protocol"
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output, "import httpx\n").unwrap();
|
||||
writeln!(output, "T = TypeVar('T')\n").unwrap();
|
||||
|
||||
types::generate_type_definitions(&mut output, schemas);
|
||||
client::generate_base_client(&mut output);
|
||||
client::generate_endpoint_class(&mut output);
|
||||
client::generate_index_accessors(&mut output, &metadata.index_set_patterns);
|
||||
client::generate_structural_patterns(&mut output, &metadata.structural_patterns, metadata);
|
||||
tree::generate_tree_classes(&mut output, &metadata.catalog, metadata);
|
||||
api::generate_main_client(&mut output, endpoints);
|
||||
|
||||
fs::write(output_path, output)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
//! Python tree structure generation.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::fmt::Write;
|
||||
|
||||
use brk_types::TreeNode;
|
||||
|
||||
use crate::{
|
||||
ClientMetadata, PatternField, child_type_name, get_fields_with_child_info, get_node_fields,
|
||||
get_pattern_instance_base, to_snake_case,
|
||||
};
|
||||
|
||||
use super::client::field_type_with_generic;
|
||||
|
||||
/// Generate tree classes
|
||||
pub fn generate_tree_classes(output: &mut String, catalog: &TreeNode, metadata: &ClientMetadata) {
|
||||
writeln!(output, "# Catalog tree classes\n").unwrap();
|
||||
|
||||
let pattern_lookup = metadata.pattern_lookup();
|
||||
let mut generated = HashSet::new();
|
||||
generate_tree_class(
|
||||
output,
|
||||
"CatalogTree",
|
||||
catalog,
|
||||
&pattern_lookup,
|
||||
metadata,
|
||||
&mut generated,
|
||||
);
|
||||
}
|
||||
|
||||
/// Recursively generate tree classes
|
||||
fn generate_tree_class(
|
||||
output: &mut String,
|
||||
name: &str,
|
||||
node: &TreeNode,
|
||||
pattern_lookup: &std::collections::HashMap<Vec<PatternField>, String>,
|
||||
metadata: &ClientMetadata,
|
||||
generated: &mut HashSet<String>,
|
||||
) {
|
||||
let TreeNode::Branch(children) = node else {
|
||||
return;
|
||||
};
|
||||
|
||||
let fields_with_child_info = get_fields_with_child_info(children, name, pattern_lookup);
|
||||
let fields: Vec<PatternField> = fields_with_child_info
|
||||
.iter()
|
||||
.map(|(f, _)| f.clone())
|
||||
.collect();
|
||||
|
||||
// Skip if this matches a pattern (already generated)
|
||||
if pattern_lookup.contains_key(&fields)
|
||||
&& pattern_lookup.get(&fields) != Some(&name.to_string())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if generated.contains(name) {
|
||||
return;
|
||||
}
|
||||
generated.insert(name.to_string());
|
||||
|
||||
writeln!(output, "class {}:", name).unwrap();
|
||||
writeln!(output, " \"\"\"Catalog tree node.\"\"\"").unwrap();
|
||||
writeln!(output, " ").unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" def __init__(self, client: BrkClientBase, base_path: str = ''):"
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
for ((field, child_fields_opt), (_child_name, child_node)) in
|
||||
fields_with_child_info.iter().zip(children.iter())
|
||||
{
|
||||
// Look up type parameter for generic patterns
|
||||
let generic_value_type = child_fields_opt
|
||||
.as_ref()
|
||||
.and_then(|cf| metadata.get_type_param(cf))
|
||||
.map(String::as_str);
|
||||
let py_type = field_type_with_generic(field, metadata, false, generic_value_type);
|
||||
let field_name_py = to_snake_case(&field.name);
|
||||
|
||||
if metadata.is_pattern_type(&field.rust_type) {
|
||||
let pattern = metadata.find_pattern(&field.rust_type);
|
||||
let is_parameterizable = pattern.is_some_and(|p| p.is_parameterizable());
|
||||
|
||||
if is_parameterizable {
|
||||
let metric_base = get_pattern_instance_base(child_node);
|
||||
writeln!(
|
||||
output,
|
||||
" self.{}: {} = {}(client, '{}')",
|
||||
field_name_py, py_type, field.rust_type, metric_base
|
||||
)
|
||||
.unwrap();
|
||||
} else {
|
||||
writeln!(
|
||||
output,
|
||||
" self.{}: {} = {}(client, f'{{base_path}}_{}')",
|
||||
field_name_py, py_type, field.rust_type, field.name
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
} else if metadata.field_uses_accessor(field) {
|
||||
let accessor = metadata.find_index_set_pattern(&field.indexes).unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" self.{}: {} = {}(client, f'{{base_path}}_{}')",
|
||||
field_name_py, py_type, accessor.name, field.name
|
||||
)
|
||||
.unwrap();
|
||||
} else if field.is_branch() {
|
||||
// Non-pattern branch - instantiate the nested class
|
||||
writeln!(
|
||||
output,
|
||||
" self.{}: {} = {}(client, f'{{base_path}}_{}')",
|
||||
field_name_py, py_type, field.rust_type, field.name
|
||||
)
|
||||
.unwrap();
|
||||
} else {
|
||||
// All metrics must be indexed - this should not be reached
|
||||
panic!(
|
||||
"Field '{}' has no matching index pattern. All metrics must be indexed.",
|
||||
field.name
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
writeln!(output).unwrap();
|
||||
|
||||
// Generate child classes
|
||||
for (child_name, child_node) in children {
|
||||
if let TreeNode::Branch(grandchildren) = child_node {
|
||||
let child_fields = get_node_fields(grandchildren, pattern_lookup);
|
||||
if !pattern_lookup.contains_key(&child_fields) {
|
||||
let child_class = child_type_name(name, child_name);
|
||||
generate_tree_class(
|
||||
output,
|
||||
&child_class,
|
||||
child_node,
|
||||
pattern_lookup,
|
||||
metadata,
|
||||
generated,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
//! Python type definitions generation.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Write;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{TypeSchemas, escape_python_keyword, ref_to_type_name};
|
||||
|
||||
/// Generate type definitions from schemas.
|
||||
pub fn generate_type_definitions(output: &mut String, schemas: &TypeSchemas) {
|
||||
if schemas.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
writeln!(output, "# Type definitions\n").unwrap();
|
||||
|
||||
let sorted_names = topological_sort_schemas(schemas);
|
||||
|
||||
for name in sorted_names {
|
||||
let Some(schema) = schemas.get(&name) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) {
|
||||
writeln!(output, "class {}(TypedDict):", name).unwrap();
|
||||
for (prop_name, prop_schema) in props {
|
||||
let prop_type = schema_to_python_type_ctx(prop_schema, Some(&name));
|
||||
let safe_name = escape_python_keyword(prop_name);
|
||||
writeln!(output, " {}: {}", safe_name, prop_type).unwrap();
|
||||
}
|
||||
writeln!(output).unwrap();
|
||||
} else {
|
||||
let py_type = schema_to_python_type_ctx(schema, Some(&name));
|
||||
writeln!(output, "{} = {}", name, py_type).unwrap();
|
||||
}
|
||||
}
|
||||
writeln!(output).unwrap();
|
||||
}
|
||||
|
||||
/// Topologically sort schema names so dependencies come before dependents (avoids forward references).
|
||||
/// Types that reference other types (via $ref) must be defined after their dependencies.
|
||||
fn topological_sort_schemas(schemas: &TypeSchemas) -> Vec<String> {
|
||||
// Build dependency graph
|
||||
let mut deps: HashMap<String, HashSet<String>> = HashMap::new();
|
||||
for (name, schema) in schemas {
|
||||
let mut type_deps = HashSet::new();
|
||||
collect_schema_refs(schema, &mut type_deps);
|
||||
// Only keep deps that are in our schemas
|
||||
type_deps.retain(|d| schemas.contains_key(d));
|
||||
deps.insert(name.clone(), type_deps);
|
||||
}
|
||||
|
||||
// Kahn's algorithm for topological sort
|
||||
let mut in_degree: HashMap<String, usize> = HashMap::new();
|
||||
for name in schemas.keys() {
|
||||
in_degree.insert(name.clone(), 0);
|
||||
}
|
||||
for type_deps in deps.values() {
|
||||
for dep in type_deps {
|
||||
*in_degree.entry(dep.clone()).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Start with types that have no dependents (are not referenced by others)
|
||||
let mut queue: Vec<String> = in_degree
|
||||
.iter()
|
||||
.filter(|(_, count)| **count == 0)
|
||||
.map(|(name, _)| name.clone())
|
||||
.collect();
|
||||
queue.sort(); // Deterministic order
|
||||
|
||||
let mut result = Vec::new();
|
||||
while let Some(name) = queue.pop() {
|
||||
result.push(name.clone());
|
||||
if let Some(type_deps) = deps.get(&name) {
|
||||
for dep in type_deps {
|
||||
if let Some(count) = in_degree.get_mut(dep) {
|
||||
*count = count.saturating_sub(1);
|
||||
if *count == 0 {
|
||||
queue.push(dep.clone());
|
||||
queue.sort(); // Keep sorted for determinism
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reverse so dependencies come first
|
||||
result.reverse();
|
||||
|
||||
// Add any types that weren't processed (e.g., due to circular refs or other edge cases)
|
||||
let result_set: HashSet<_> = result.iter().cloned().collect();
|
||||
let mut missing: Vec<_> = schemas
|
||||
.keys()
|
||||
.filter(|k| !result_set.contains(*k))
|
||||
.cloned()
|
||||
.collect();
|
||||
missing.sort();
|
||||
result.extend(missing);
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Collect all type references ($ref) from a schema
|
||||
fn collect_schema_refs(schema: &Value, refs: &mut HashSet<String>) {
|
||||
match schema {
|
||||
Value::Object(map) => {
|
||||
if let Some(ref_path) = map.get("$ref").and_then(|r| r.as_str())
|
||||
&& let Some(type_name) = ref_to_type_name(ref_path)
|
||||
{
|
||||
refs.insert(type_name.to_string());
|
||||
}
|
||||
for value in map.values() {
|
||||
collect_schema_refs(value, refs);
|
||||
}
|
||||
}
|
||||
Value::Array(arr) => {
|
||||
for item in arr {
|
||||
collect_schema_refs(item, refs);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a single JSON type string to Python type
|
||||
fn json_type_to_python(ty: &str, schema: &Value, current_type: Option<&str>) -> String {
|
||||
match ty {
|
||||
"integer" => "int".to_string(),
|
||||
"number" => "float".to_string(),
|
||||
"boolean" => "bool".to_string(),
|
||||
"string" => "str".to_string(),
|
||||
"null" => "None".to_string(),
|
||||
"array" => {
|
||||
let item_type = schema
|
||||
.get("items")
|
||||
.map(|s| schema_to_python_type_ctx(s, current_type))
|
||||
.unwrap_or_else(|| "Any".to_string());
|
||||
format!("List[{}]", item_type)
|
||||
}
|
||||
"object" => {
|
||||
if let Some(add_props) = schema.get("additionalProperties") {
|
||||
let value_type = schema_to_python_type_ctx(add_props, current_type);
|
||||
return format!("dict[str, {}]", value_type);
|
||||
}
|
||||
"dict".to_string()
|
||||
}
|
||||
_ => "Any".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert JSON Schema to Python type with context for detecting self-references
|
||||
pub fn schema_to_python_type_ctx(schema: &Value, current_type: Option<&str>) -> String {
|
||||
if let Some(all_of) = schema.get("allOf").and_then(|v| v.as_array()) {
|
||||
for item in all_of {
|
||||
let resolved = schema_to_python_type_ctx(item, current_type);
|
||||
if resolved != "Any" {
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle $ref
|
||||
if let Some(ref_path) = schema.get("$ref").and_then(|r| r.as_str()) {
|
||||
let type_name = ref_to_type_name(ref_path).unwrap_or("Any");
|
||||
// Quote self-references to handle recursive types
|
||||
if current_type == Some(type_name) {
|
||||
return format!("\"{}\"", type_name);
|
||||
}
|
||||
return type_name.to_string();
|
||||
}
|
||||
|
||||
// Handle enum (array of string values)
|
||||
if let Some(enum_values) = schema.get("enum").and_then(|e| e.as_array()) {
|
||||
let literals: Vec<String> = enum_values
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.map(|s| format!("\"{}\"", s))
|
||||
.collect();
|
||||
if !literals.is_empty() {
|
||||
return format!("Literal[{}]", literals.join(", "));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ty) = schema.get("type") {
|
||||
if let Some(type_array) = ty.as_array() {
|
||||
let types: Vec<String> = type_array
|
||||
.iter()
|
||||
.filter_map(|t| t.as_str())
|
||||
.filter(|t| *t != "null") // Filter out null for cleaner Optional handling
|
||||
.map(|t| json_type_to_python(t, schema, current_type))
|
||||
.collect();
|
||||
let has_null = type_array.iter().any(|t| t.as_str() == Some("null"));
|
||||
|
||||
if types.len() == 1 {
|
||||
let base_type = &types[0];
|
||||
return if has_null {
|
||||
format!("Optional[{}]", base_type)
|
||||
} else {
|
||||
base_type.clone()
|
||||
};
|
||||
} else if !types.is_empty() {
|
||||
let union = format!("Union[{}]", types.join(", "));
|
||||
return if has_null {
|
||||
format!("Optional[{}]", union)
|
||||
} else {
|
||||
union
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ty_str) = ty.as_str() {
|
||||
return json_type_to_python(ty_str, schema, current_type);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(variants) = schema
|
||||
.get("anyOf")
|
||||
.or_else(|| schema.get("oneOf"))
|
||||
.and_then(|v| v.as_array())
|
||||
{
|
||||
let types: Vec<String> = variants
|
||||
.iter()
|
||||
.map(|v| schema_to_python_type_ctx(v, current_type))
|
||||
.collect();
|
||||
let filtered: Vec<_> = types.iter().filter(|t| *t != "Any").collect();
|
||||
if !filtered.is_empty() {
|
||||
return format!(
|
||||
"Union[{}]",
|
||||
filtered
|
||||
.iter()
|
||||
.map(|s| s.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
);
|
||||
}
|
||||
return format!("Union[{}]", types.join(", "));
|
||||
}
|
||||
|
||||
// Check for format hint without type (common in OpenAPI)
|
||||
if let Some(format) = schema.get("format").and_then(|f| f.as_str()) {
|
||||
return match format {
|
||||
"int32" | "int64" => "int".to_string(),
|
||||
"float" | "double" => "float".to_string(),
|
||||
"date" | "date-time" => "str".to_string(),
|
||||
_ => "Any".to_string(),
|
||||
};
|
||||
}
|
||||
|
||||
"Any".to_string()
|
||||
}
|
||||
|
||||
/// Convert JS-style type to Python type (e.g., "Txid[]" -> "List[Txid]", "number" -> "int")
|
||||
pub fn js_type_to_python(js_type: &str) -> String {
|
||||
if let Some(inner) = js_type.strip_suffix("[]") {
|
||||
format!("List[{}]", js_type_to_python(inner))
|
||||
} else {
|
||||
match js_type {
|
||||
"number" => "int".to_string(),
|
||||
"boolean" => "bool".to_string(),
|
||||
"string" => "str".to_string(),
|
||||
"null" => "None".to_string(),
|
||||
"Object" | "object" => "dict".to_string(),
|
||||
"*" => "Any".to_string(),
|
||||
_ => js_type.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
//! Rust API method generation.
|
||||
|
||||
use std::fmt::Write;
|
||||
|
||||
use crate::{Endpoint, VERSION, to_snake_case};
|
||||
|
||||
use super::types::js_type_to_rust;
|
||||
|
||||
/// Generate the main BrkClient struct.
|
||||
pub fn generate_main_client(output: &mut String, endpoints: &[Endpoint]) {
|
||||
writeln!(
|
||||
output,
|
||||
r#"/// Main BRK client with catalog tree and API methods.
|
||||
pub struct BrkClient {{
|
||||
base: Arc<BrkClientBase>,
|
||||
tree: CatalogTree,
|
||||
}}
|
||||
|
||||
impl BrkClient {{
|
||||
/// Client version.
|
||||
pub const VERSION: &'static str = "v{VERSION}";
|
||||
|
||||
/// Create a new client with the given base URL.
|
||||
pub fn new(base_url: impl Into<String>) -> Self {{
|
||||
let base = Arc::new(BrkClientBase::new(base_url));
|
||||
let tree = CatalogTree::new(base.clone(), String::new());
|
||||
Self {{ base, tree }}
|
||||
}}
|
||||
|
||||
/// Create a new client with options.
|
||||
pub fn with_options(options: BrkClientOptions) -> Self {{
|
||||
let base = Arc::new(BrkClientBase::with_options(options));
|
||||
let tree = CatalogTree::new(base.clone(), String::new());
|
||||
Self {{ base, tree }}
|
||||
}}
|
||||
|
||||
/// Get the catalog tree for navigating metrics.
|
||||
pub fn tree(&self) -> &CatalogTree {{
|
||||
&self.tree
|
||||
}}
|
||||
"#,
|
||||
VERSION = VERSION
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
generate_api_methods(output, endpoints);
|
||||
|
||||
writeln!(output, "}}").unwrap();
|
||||
}
|
||||
|
||||
/// Generate API methods from OpenAPI endpoints.
|
||||
pub fn generate_api_methods(output: &mut String, endpoints: &[Endpoint]) {
|
||||
for endpoint in endpoints {
|
||||
if !endpoint.should_generate() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let method_name = endpoint_to_method_name(endpoint);
|
||||
let return_type = endpoint
|
||||
.response_type
|
||||
.as_deref()
|
||||
.map(js_type_to_rust)
|
||||
.unwrap_or_else(|| "serde_json::Value".to_string());
|
||||
|
||||
writeln!(
|
||||
output,
|
||||
" /// {}",
|
||||
endpoint.summary.as_deref().unwrap_or(&method_name)
|
||||
)
|
||||
.unwrap();
|
||||
if let Some(desc) = &endpoint.description
|
||||
&& endpoint.summary.as_ref() != Some(desc)
|
||||
{
|
||||
writeln!(output, " ///").unwrap();
|
||||
writeln!(output, " /// {}", desc).unwrap();
|
||||
}
|
||||
|
||||
let params = build_method_params(endpoint);
|
||||
writeln!(
|
||||
output,
|
||||
" pub fn {}(&self{}) -> Result<{}> {{",
|
||||
method_name, params, return_type
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let path = build_path_template(&endpoint.path);
|
||||
|
||||
if endpoint.query_params.is_empty() {
|
||||
writeln!(output, " self.base.get(&format!(\"{}\"))", path).unwrap();
|
||||
} else {
|
||||
writeln!(output, " let mut query = Vec::new();").unwrap();
|
||||
for param in &endpoint.query_params {
|
||||
if param.required {
|
||||
writeln!(
|
||||
output,
|
||||
" query.push(format!(\"{}={{}}\", {}));",
|
||||
param.name, param.name
|
||||
)
|
||||
.unwrap();
|
||||
} else {
|
||||
writeln!(
|
||||
output,
|
||||
" if let Some(v) = {} {{ query.push(format!(\"{}={{}}\", v)); }}",
|
||||
param.name, param.name
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
writeln!(output, " let query_str = if query.is_empty() {{ String::new() }} else {{ format!(\"?{{}}\", query.join(\"&\")) }};").unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" self.base.get(&format!(\"{}{{}}\", query_str))",
|
||||
path
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
writeln!(output, " }}\n").unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn endpoint_to_method_name(endpoint: &Endpoint) -> String {
|
||||
to_snake_case(&endpoint.operation_name())
|
||||
}
|
||||
|
||||
fn build_method_params(endpoint: &Endpoint) -> String {
|
||||
let mut params = Vec::new();
|
||||
for param in &endpoint.path_params {
|
||||
params.push(format!(", {}: &str", param.name));
|
||||
}
|
||||
for param in &endpoint.query_params {
|
||||
if param.required {
|
||||
params.push(format!(", {}: &str", param.name));
|
||||
} else {
|
||||
params.push(format!(", {}: Option<&str>", param.name));
|
||||
}
|
||||
}
|
||||
params.join("")
|
||||
}
|
||||
|
||||
/// OpenAPI path placeholders `{param}` are already valid Rust format string syntax.
|
||||
fn build_path_template(path: &str) -> &str {
|
||||
path
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
//! Rust base client and pattern factory generation.
|
||||
|
||||
use std::fmt::Write;
|
||||
|
||||
use crate::{
|
||||
ClientMetadata, GenericSyntax, IndexSetPattern, PatternField, RustSyntax,
|
||||
StructuralPattern, generate_parameterized_field, generate_tree_path_field,
|
||||
index_to_field_name, to_snake_case,
|
||||
};
|
||||
|
||||
/// Generate import statements.
|
||||
pub fn generate_imports(output: &mut String) {
|
||||
writeln!(
|
||||
output,
|
||||
r#"use std::sync::Arc;
|
||||
use serde::de::DeserializeOwned;
|
||||
pub use brk_cohort::*;
|
||||
pub use brk_types::*;
|
||||
|
||||
"#
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Generate the base BrkClientBase struct and error types.
|
||||
pub fn generate_base_client(output: &mut String) {
|
||||
writeln!(
|
||||
output,
|
||||
r#"/// Error type for BRK client operations.
|
||||
#[derive(Debug)]
|
||||
pub struct BrkError {{
|
||||
pub message: String,
|
||||
}}
|
||||
|
||||
impl std::fmt::Display for BrkError {{
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{
|
||||
write!(f, "{{}}", self.message)
|
||||
}}
|
||||
}}
|
||||
|
||||
impl std::error::Error for BrkError {{}}
|
||||
|
||||
/// Result type for BRK client operations.
|
||||
pub type Result<T> = std::result::Result<T, BrkError>;
|
||||
|
||||
/// Options for configuring the BRK client.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BrkClientOptions {{
|
||||
pub base_url: String,
|
||||
pub timeout_secs: u64,
|
||||
}}
|
||||
|
||||
impl Default for BrkClientOptions {{
|
||||
fn default() -> Self {{
|
||||
Self {{
|
||||
base_url: "http://localhost:3000".to_string(),
|
||||
timeout_secs: 30,
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
/// Base HTTP client for making requests.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BrkClientBase {{
|
||||
base_url: String,
|
||||
timeout_secs: u64,
|
||||
}}
|
||||
|
||||
impl BrkClientBase {{
|
||||
/// Create a new client with the given base URL.
|
||||
pub fn new(base_url: impl Into<String>) -> Self {{
|
||||
Self {{
|
||||
base_url: base_url.into(),
|
||||
timeout_secs: 30,
|
||||
}}
|
||||
}}
|
||||
|
||||
/// Create a new client with options.
|
||||
pub fn with_options(options: BrkClientOptions) -> Self {{
|
||||
Self {{
|
||||
base_url: options.base_url,
|
||||
timeout_secs: options.timeout_secs,
|
||||
}}
|
||||
}}
|
||||
|
||||
/// Make a GET request.
|
||||
pub fn get<T: DeserializeOwned>(&self, path: &str) -> Result<T> {{
|
||||
let base = self.base_url.trim_end_matches('/');
|
||||
let url = format!("{{}}{{}}", base, path);
|
||||
let response = minreq::get(&url)
|
||||
.with_timeout(self.timeout_secs)
|
||||
.send()
|
||||
.map_err(|e| BrkError {{ message: e.to_string() }})?;
|
||||
|
||||
if response.status_code >= 400 {{
|
||||
return Err(BrkError {{
|
||||
message: format!("HTTP {{}}", response.status_code),
|
||||
}});
|
||||
}}
|
||||
|
||||
response
|
||||
.json()
|
||||
.map_err(|e| BrkError {{ message: e.to_string() }})
|
||||
}}
|
||||
}}
|
||||
|
||||
/// Build metric name with optional prefix.
|
||||
#[inline]
|
||||
fn _m(acc: &str, s: &str) -> String {{
|
||||
if acc.is_empty() {{ s.to_string() }} else {{ format!("{{acc}}_{{s}}") }}
|
||||
}}
|
||||
|
||||
"#
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Generate the MetricPattern trait.
|
||||
pub fn generate_metric_pattern_trait(output: &mut String) {
|
||||
writeln!(
|
||||
output,
|
||||
r#"/// Non-generic trait for metric patterns (usable in collections).
|
||||
pub trait AnyMetricPattern {{
|
||||
/// Get the metric name.
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Get the list of available indexes for this metric.
|
||||
fn indexes(&self) -> &'static [Index];
|
||||
}}
|
||||
|
||||
/// Generic trait for metric patterns with endpoint access.
|
||||
pub trait MetricPattern<T>: AnyMetricPattern {{
|
||||
/// Get an endpoint for a specific index, if supported.
|
||||
fn get(&self, index: Index) -> Option<Endpoint<T>>;
|
||||
}}
|
||||
|
||||
"#
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Generate the Endpoint struct.
|
||||
pub fn generate_endpoint(output: &mut String) {
|
||||
writeln!(
|
||||
output,
|
||||
r#"/// An endpoint for a specific metric + index combination.
|
||||
pub struct Endpoint<T> {{
|
||||
client: Arc<BrkClientBase>,
|
||||
name: Arc<str>,
|
||||
index: Index,
|
||||
_marker: std::marker::PhantomData<T>,
|
||||
}}
|
||||
|
||||
impl<T: DeserializeOwned> Endpoint<T> {{
|
||||
pub fn new(client: Arc<BrkClientBase>, name: Arc<str>, index: Index) -> Self {{
|
||||
Self {{
|
||||
client,
|
||||
name,
|
||||
index,
|
||||
_marker: std::marker::PhantomData,
|
||||
}}
|
||||
}}
|
||||
|
||||
/// Fetch all data points for this metric/index.
|
||||
pub fn get(&self) -> Result<Vec<T>> {{
|
||||
self.client.get(&self.path())
|
||||
}}
|
||||
|
||||
/// Fetch data points within a range.
|
||||
pub fn range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {{
|
||||
let mut params = Vec::new();
|
||||
if let Some(f) = from {{ params.push(format!("from={{}}", f)); }}
|
||||
if let Some(t) = to {{ params.push(format!("to={{}}", t)); }}
|
||||
let p = self.path();
|
||||
let path = if params.is_empty() {{
|
||||
p
|
||||
}} else {{
|
||||
format!("{{}}?{{}}", p, params.join("&"))
|
||||
}};
|
||||
self.client.get(&path)
|
||||
}}
|
||||
|
||||
/// Get the endpoint path.
|
||||
pub fn path(&self) -> String {{
|
||||
format!("/api/metric/{{}}/{{}}", self.name, self.index.serialize_long())
|
||||
}}
|
||||
}}
|
||||
|
||||
"#
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Generate index accessor structs.
|
||||
pub fn generate_index_accessors(output: &mut String, patterns: &[IndexSetPattern]) {
|
||||
if patterns.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
writeln!(output, "// Index accessor structs\n").unwrap();
|
||||
|
||||
for pattern in patterns {
|
||||
let by_name = format!("{}By", pattern.name);
|
||||
|
||||
// Generate the "By" struct with lazy endpoint methods
|
||||
writeln!(output, "/// Container for index endpoint methods.").unwrap();
|
||||
writeln!(output, "pub struct {}<T> {{", by_name).unwrap();
|
||||
writeln!(output, " client: Arc<BrkClientBase>,").unwrap();
|
||||
writeln!(output, " name: Arc<str>,").unwrap();
|
||||
writeln!(output, " _marker: std::marker::PhantomData<T>,").unwrap();
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
|
||||
// Generate impl with methods for each index
|
||||
writeln!(output, "impl<T: DeserializeOwned> {}<T> {{", by_name).unwrap();
|
||||
for index in &pattern.indexes {
|
||||
let method_name = index_to_field_name(index);
|
||||
writeln!(output, " pub fn {}(&self) -> Endpoint<T> {{", method_name).unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" Endpoint::new(self.client.clone(), self.name.clone(), Index::{})",
|
||||
index
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
}
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
|
||||
// Generate the main accessor struct
|
||||
writeln!(
|
||||
output,
|
||||
"/// Index accessor for metrics with {} indexes.",
|
||||
pattern.indexes.len()
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output, "pub struct {}<T> {{", pattern.name).unwrap();
|
||||
writeln!(output, " client: Arc<BrkClientBase>,").unwrap();
|
||||
writeln!(output, " name: Arc<str>,").unwrap();
|
||||
writeln!(output, " pub by: {}<T>,", by_name).unwrap();
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
|
||||
// Generate impl block with constructor
|
||||
writeln!(output, "impl<T: DeserializeOwned> {}<T> {{", pattern.name).unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" pub fn new(client: Arc<BrkClientBase>, name: String) -> Self {{"
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output, " let name: Arc<str> = name.into();").unwrap();
|
||||
writeln!(output, " Self {{").unwrap();
|
||||
writeln!(output, " client: client.clone(),").unwrap();
|
||||
writeln!(output, " name: name.clone(),").unwrap();
|
||||
writeln!(output, " by: {} {{", by_name).unwrap();
|
||||
writeln!(output, " client,").unwrap();
|
||||
writeln!(output, " name,").unwrap();
|
||||
writeln!(output, " _marker: std::marker::PhantomData,").unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output).unwrap();
|
||||
writeln!(output, " /// Get the metric name.").unwrap();
|
||||
writeln!(output, " pub fn name(&self) -> &str {{").unwrap();
|
||||
writeln!(output, " &self.name").unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
|
||||
// Implement AnyMetricPattern trait
|
||||
writeln!(output, "impl<T> AnyMetricPattern for {}<T> {{", pattern.name).unwrap();
|
||||
writeln!(output, " fn name(&self) -> &str {{").unwrap();
|
||||
writeln!(output, " &self.name").unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output).unwrap();
|
||||
writeln!(output, " fn indexes(&self) -> &'static [Index] {{").unwrap();
|
||||
writeln!(output, " &[").unwrap();
|
||||
for index in &pattern.indexes {
|
||||
writeln!(output, " Index::{},", index).unwrap();
|
||||
}
|
||||
writeln!(output, " ]").unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
|
||||
// Implement MetricPattern<T> trait
|
||||
writeln!(output, "impl<T: DeserializeOwned> MetricPattern<T> for {}<T> {{", pattern.name).unwrap();
|
||||
writeln!(output, " fn get(&self, index: Index) -> Option<Endpoint<T>> {{").unwrap();
|
||||
writeln!(output, " match index {{").unwrap();
|
||||
for index in &pattern.indexes {
|
||||
let method_name = index_to_field_name(index);
|
||||
writeln!(output, " Index::{} => Some(self.by.{}()),", index, method_name).unwrap();
|
||||
}
|
||||
writeln!(output, " _ => None,").unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate structural pattern structs.
|
||||
pub fn generate_pattern_structs(
|
||||
output: &mut String,
|
||||
patterns: &[StructuralPattern],
|
||||
metadata: &ClientMetadata,
|
||||
) {
|
||||
if patterns.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
writeln!(output, "// Reusable pattern structs\n").unwrap();
|
||||
|
||||
for pattern in patterns {
|
||||
let is_parameterizable = pattern.is_parameterizable();
|
||||
let generic_params = if pattern.is_generic { "<T>" } else { "" };
|
||||
|
||||
writeln!(output, "/// Pattern struct for repeated tree structure.").unwrap();
|
||||
writeln!(output, "pub struct {}{} {{", pattern.name, generic_params).unwrap();
|
||||
|
||||
for field in &pattern.fields {
|
||||
let field_name = to_snake_case(&field.name);
|
||||
let type_annotation =
|
||||
field_type_with_generic(field, metadata, pattern.is_generic, None);
|
||||
writeln!(output, " pub {}: {},", field_name, type_annotation).unwrap();
|
||||
}
|
||||
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
|
||||
// Generate impl block with constructor
|
||||
let impl_generic = if pattern.is_generic {
|
||||
"<T: DeserializeOwned>"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
writeln!(
|
||||
output,
|
||||
"impl{} {}{} {{",
|
||||
impl_generic, pattern.name, generic_params
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
if is_parameterizable {
|
||||
writeln!(
|
||||
output,
|
||||
" /// Create a new pattern node with accumulated metric name."
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" pub fn new(client: Arc<BrkClientBase>, acc: String) -> Self {{"
|
||||
)
|
||||
.unwrap();
|
||||
} else {
|
||||
writeln!(
|
||||
output,
|
||||
" pub fn new(client: Arc<BrkClientBase>, base_path: String) -> Self {{"
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
writeln!(output, " Self {{").unwrap();
|
||||
|
||||
let syntax = RustSyntax;
|
||||
for field in &pattern.fields {
|
||||
if is_parameterizable {
|
||||
generate_parameterized_field(output, &syntax, field, pattern, metadata, " ");
|
||||
} else {
|
||||
generate_tree_path_field(output, &syntax, field, metadata, " ");
|
||||
}
|
||||
}
|
||||
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
/// Get Rust type annotation for a field with optional generic value type.
|
||||
pub fn field_type_with_generic(
|
||||
field: &PatternField,
|
||||
metadata: &ClientMetadata,
|
||||
is_generic: bool,
|
||||
generic_value_type: Option<&str>,
|
||||
) -> String {
|
||||
metadata.field_type_annotation(field, is_generic, generic_value_type, GenericSyntax::RUST)
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
//! Rust client generation.
|
||||
//!
|
||||
//! This module generates a Rust client with full type safety for the BRK API.
|
||||
|
||||
mod api;
|
||||
mod client;
|
||||
mod tree;
|
||||
mod types;
|
||||
|
||||
use std::{fmt::Write, fs, io, path::Path};
|
||||
|
||||
use crate::{ClientMetadata, Endpoint};
|
||||
|
||||
/// Generate Rust client from metadata and OpenAPI endpoints.
|
||||
///
|
||||
/// `output_path` is the full path to the output file (e.g., "crates/brk_client/src/lib.rs").
|
||||
pub fn generate_rust_client(
|
||||
metadata: &ClientMetadata,
|
||||
endpoints: &[Endpoint],
|
||||
output_path: &Path,
|
||||
) -> io::Result<()> {
|
||||
let mut output = String::new();
|
||||
|
||||
writeln!(output, "// Auto-generated BRK Rust client").unwrap();
|
||||
writeln!(output, "// Do not edit manually\n").unwrap();
|
||||
writeln!(output, "#![allow(non_camel_case_types)]").unwrap();
|
||||
writeln!(output, "#![allow(dead_code)]").unwrap();
|
||||
writeln!(output, "#![allow(unused_variables)]").unwrap();
|
||||
writeln!(output, "#![allow(clippy::useless_format)]").unwrap();
|
||||
writeln!(output, "#![allow(clippy::unnecessary_to_owned)]\n").unwrap();
|
||||
|
||||
client::generate_imports(&mut output);
|
||||
client::generate_base_client(&mut output);
|
||||
client::generate_metric_pattern_trait(&mut output);
|
||||
client::generate_endpoint(&mut output);
|
||||
client::generate_index_accessors(&mut output, &metadata.index_set_patterns);
|
||||
client::generate_pattern_structs(&mut output, &metadata.structural_patterns, metadata);
|
||||
tree::generate_tree(&mut output, &metadata.catalog, metadata);
|
||||
api::generate_main_client(&mut output, endpoints);
|
||||
|
||||
fs::write(output_path, output)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
//! Rust tree structure generation.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::fmt::Write;
|
||||
|
||||
use brk_types::TreeNode;
|
||||
|
||||
use crate::{
|
||||
ClientMetadata, PatternField, RustSyntax, child_type_name, generate_tree_node_field,
|
||||
get_fields_with_child_info, get_node_fields, get_pattern_instance_base, to_snake_case,
|
||||
};
|
||||
|
||||
use super::client::field_type_with_generic;
|
||||
|
||||
/// Generate tree structs.
|
||||
pub fn generate_tree(output: &mut String, catalog: &TreeNode, metadata: &ClientMetadata) {
|
||||
writeln!(output, "// Catalog tree\n").unwrap();
|
||||
|
||||
let pattern_lookup = metadata.pattern_lookup();
|
||||
let mut generated = HashSet::new();
|
||||
generate_tree_node(
|
||||
output,
|
||||
"CatalogTree",
|
||||
catalog,
|
||||
&pattern_lookup,
|
||||
metadata,
|
||||
&mut generated,
|
||||
);
|
||||
}
|
||||
|
||||
fn generate_tree_node(
|
||||
output: &mut String,
|
||||
name: &str,
|
||||
node: &TreeNode,
|
||||
pattern_lookup: &std::collections::HashMap<Vec<PatternField>, String>,
|
||||
metadata: &ClientMetadata,
|
||||
generated: &mut HashSet<String>,
|
||||
) {
|
||||
let TreeNode::Branch(children) = node else {
|
||||
return;
|
||||
};
|
||||
|
||||
let fields_with_child_info = get_fields_with_child_info(children, name, pattern_lookup);
|
||||
let fields: Vec<PatternField> = fields_with_child_info
|
||||
.iter()
|
||||
.map(|(f, _)| f.clone())
|
||||
.collect();
|
||||
|
||||
if let Some(pattern_name) = pattern_lookup.get(&fields)
|
||||
&& pattern_name != name
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if generated.contains(name) {
|
||||
return;
|
||||
}
|
||||
generated.insert(name.to_string());
|
||||
|
||||
writeln!(output, "/// Catalog tree node.").unwrap();
|
||||
writeln!(output, "pub struct {} {{", name).unwrap();
|
||||
|
||||
for (field, child_fields) in &fields_with_child_info {
|
||||
let field_name = to_snake_case(&field.name);
|
||||
// Look up type parameter for generic patterns
|
||||
let generic_value_type = child_fields
|
||||
.as_ref()
|
||||
.and_then(|cf| metadata.get_type_param(cf))
|
||||
.map(String::as_str);
|
||||
let type_annotation = field_type_with_generic(field, metadata, false, generic_value_type);
|
||||
writeln!(output, " pub {}: {},", field_name, type_annotation).unwrap();
|
||||
}
|
||||
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
|
||||
writeln!(output, "impl {} {{", name).unwrap();
|
||||
writeln!(
|
||||
output,
|
||||
" pub fn new(client: Arc<BrkClientBase>, base_path: String) -> Self {{"
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(output, " Self {{").unwrap();
|
||||
|
||||
let syntax = RustSyntax;
|
||||
for (field, (child_name, child_node)) in fields.iter().zip(children.iter()) {
|
||||
// Detect pattern base for parameterizable patterns
|
||||
let pattern_base = if metadata.is_pattern_type(&field.rust_type) {
|
||||
let pattern = metadata.find_pattern(&field.rust_type);
|
||||
if pattern.is_some_and(|p| p.is_parameterizable()) {
|
||||
Some(get_pattern_instance_base(child_node))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
generate_tree_node_field(output, &syntax, field, metadata, " ", child_name, pattern_base.as_deref());
|
||||
}
|
||||
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output, " }}").unwrap();
|
||||
writeln!(output, "}}\n").unwrap();
|
||||
|
||||
for (child_name, child_node) in children {
|
||||
if let TreeNode::Branch(grandchildren) = child_node {
|
||||
let child_fields = get_node_fields(grandchildren, pattern_lookup);
|
||||
if !pattern_lookup.contains_key(&child_fields) {
|
||||
let child_struct = child_type_name(name, child_name);
|
||||
generate_tree_node(
|
||||
output,
|
||||
&child_struct,
|
||||
child_node,
|
||||
pattern_lookup,
|
||||
metadata,
|
||||
generated,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
//! Rust type conversion utilities.
|
||||
|
||||
/// Convert JS-style type to Rust type.
|
||||
pub fn js_type_to_rust(js_type: &str) -> String {
|
||||
if let Some(inner) = js_type.strip_suffix("[]") {
|
||||
format!("Vec<{}>", js_type_to_rust(inner))
|
||||
} else {
|
||||
match js_type {
|
||||
"string" => "String".to_string(),
|
||||
"number" => "f64".to_string(),
|
||||
"boolean" => "bool".to_string(),
|
||||
"*" => "serde_json::Value".to_string(),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
#![allow(clippy::type_complexity)]
|
||||
|
||||
use std::{collections::btree_map::Entry, fs::create_dir_all, io, path::PathBuf};
|
||||
|
||||
use brk_query::Vecs;
|
||||
|
||||
/// Output path configuration for each language client.
|
||||
///
|
||||
/// Each path should be the full path to the output file, not just a directory.
|
||||
/// Parent directories will be created automatically if they don't exist.
|
||||
///
|
||||
/// # Example
|
||||
/// ```ignore
|
||||
/// let paths = ClientOutputPaths::new()
|
||||
/// .rust("crates/brk_client/src/lib.rs")
|
||||
/// .javascript("modules/brk-client/index.js")
|
||||
/// .python("packages/brk_client/__init__.py");
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ClientOutputPaths {
|
||||
/// Full path to Rust client file (e.g., "crates/brk_client/src/lib.rs")
|
||||
pub rust: Option<PathBuf>,
|
||||
/// Full path to JavaScript client file (e.g., "modules/brk-client/index.js")
|
||||
pub javascript: Option<PathBuf>,
|
||||
/// Full path to Python client file (e.g., "packages/brk_client/__init__.py")
|
||||
pub python: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl ClientOutputPaths {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn rust(mut self, path: impl Into<PathBuf>) -> Self {
|
||||
self.rust = Some(path.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn javascript(mut self, path: impl Into<PathBuf>) -> Self {
|
||||
self.javascript = Some(path.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn python(mut self, path: impl Into<PathBuf>) -> Self {
|
||||
self.python = Some(path.into());
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
mod analysis;
|
||||
mod backends;
|
||||
mod generate;
|
||||
mod generators;
|
||||
mod openapi;
|
||||
mod syntax;
|
||||
mod types;
|
||||
|
||||
pub use analysis::*;
|
||||
pub use backends::*;
|
||||
pub use generate::*;
|
||||
pub use generators::{generate_javascript_client, generate_python_client, generate_rust_client};
|
||||
pub use openapi::*;
|
||||
pub use syntax::*;
|
||||
pub use types::*;
|
||||
|
||||
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
/// Generate all client libraries from the query vecs and OpenAPI JSON.
|
||||
///
|
||||
/// Uses `ClientOutputPaths` to specify the output file path for each language.
|
||||
/// Only languages with a configured path will be generated.
|
||||
///
|
||||
/// # Example
|
||||
/// ```ignore
|
||||
/// let paths = ClientOutputPaths::new()
|
||||
/// .rust("crates/brk_client/src/lib.rs")
|
||||
/// .javascript("modules/brk-client/index.js")
|
||||
/// .python("packages/brk_client/__init__.py");
|
||||
///
|
||||
/// generate_clients(&vecs, &openapi_json, &paths)?;
|
||||
/// ```
|
||||
pub fn generate_clients(
|
||||
vecs: &Vecs,
|
||||
openapi_json: &str,
|
||||
output_paths: &ClientOutputPaths,
|
||||
) -> io::Result<()> {
|
||||
let metadata = ClientMetadata::from_vecs(vecs);
|
||||
|
||||
// Parse OpenAPI spec
|
||||
let spec = parse_openapi_json(openapi_json)?;
|
||||
let endpoints = extract_endpoints(&spec);
|
||||
let mut schemas = extract_schemas(openapi_json);
|
||||
|
||||
// Collect leaf type schemas from the catalog and merge into schemas
|
||||
collect_leaf_type_schemas(&metadata.catalog, &mut schemas);
|
||||
|
||||
// Also collect definitions from all schemas (including OpenAPI schemas)
|
||||
// We need to do this after collecting leaf schemas so we process everything
|
||||
let schema_values: Vec<_> = schemas.values().cloned().collect();
|
||||
for schema in &schema_values {
|
||||
collect_schema_definitions(schema, &mut schemas);
|
||||
}
|
||||
|
||||
// Generate Rust client (uses real brk_types, no schema conversion needed)
|
||||
if let Some(rust_path) = &output_paths.rust {
|
||||
if let Some(parent) = rust_path.parent() {
|
||||
create_dir_all(parent)?;
|
||||
}
|
||||
generate_rust_client(&metadata, &endpoints, rust_path)?;
|
||||
}
|
||||
|
||||
// Generate JavaScript client (needs schemas for type definitions)
|
||||
if let Some(js_path) = &output_paths.javascript {
|
||||
if let Some(parent) = js_path.parent() {
|
||||
create_dir_all(parent)?;
|
||||
}
|
||||
generate_javascript_client(&metadata, &endpoints, &schemas, js_path)?;
|
||||
}
|
||||
|
||||
// Generate Python client (needs schemas for type definitions)
|
||||
if let Some(python_path) = &output_paths.python {
|
||||
if let Some(parent) = python_path.parent() {
|
||||
create_dir_all(parent)?;
|
||||
}
|
||||
generate_python_client(&metadata, &endpoints, &schemas, python_path)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
use brk_types::TreeNode;
|
||||
use serde_json::Value;
|
||||
|
||||
/// Recursively collect leaf type schemas from the tree and add to schemas map.
|
||||
/// Only adds schemas that aren't already present (OpenAPI schemas take precedence).
|
||||
/// Collects definitions from schemars-generated schemas (for referenced types).
|
||||
fn collect_leaf_type_schemas(node: &TreeNode, schemas: &mut TypeSchemas) {
|
||||
match node {
|
||||
TreeNode::Leaf(leaf) => {
|
||||
// Collect definitions from the schema (schemars puts type schemas here)
|
||||
// This includes the inner types like `Bitcoin` from `Close<Bitcoin>`
|
||||
collect_schema_definitions(&leaf.schema, schemas);
|
||||
|
||||
// Get the type name for this leaf
|
||||
let type_name = extract_inner_type(leaf.value_type());
|
||||
|
||||
if let Entry::Vacant(e) = schemas.entry(type_name) {
|
||||
// Unwrap single-element allOf
|
||||
let schema = unwrap_allof(&leaf.schema);
|
||||
|
||||
// Add the schema if it's usable:
|
||||
// - Simple type (has "type")
|
||||
// - Object type with properties (complex types like OHLCCents, EmptyAddressData)
|
||||
// - Enum type (has "enum" or "oneOf")
|
||||
// - Or a $ref to another type
|
||||
let has_type = schema.get("type").is_some();
|
||||
let has_properties = schema.get("properties").is_some();
|
||||
let has_enum = schema.get("enum").is_some() || schema.get("oneOf").is_some();
|
||||
let is_ref = schema.get("$ref").is_some();
|
||||
|
||||
if has_type || has_properties || has_enum || is_ref {
|
||||
e.insert(schema.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
TreeNode::Branch(children) => {
|
||||
for child in children.values() {
|
||||
collect_leaf_type_schemas(child, schemas);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Collect type definitions from schemars-generated schema's definitions section.
|
||||
/// Schemars uses `definitions` or `$defs` to store referenced types.
|
||||
fn collect_schema_definitions(schema: &Value, schemas: &mut TypeSchemas) {
|
||||
// Check both JSON Schema draft-07 style ("definitions") and draft 2019-09+ style ("$defs")
|
||||
for key in ["definitions", "$defs"] {
|
||||
if let Some(defs) = schema.get(key).and_then(|d| d.as_object()) {
|
||||
for (name, def_schema) in defs {
|
||||
if !schemas.contains_key(name) {
|
||||
schemas.insert(name.clone(), def_schema.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
use std::{collections::BTreeMap, io};
|
||||
|
||||
use crate::ref_to_type_name;
|
||||
use oas3::Spec;
|
||||
use oas3::spec::{ObjectOrReference, Operation, ParameterIn, PathItem, Schema, SchemaTypeSet};
|
||||
use serde_json::Value;
|
||||
|
||||
/// Type schema extracted from OpenAPI components
|
||||
pub type TypeSchemas = BTreeMap<String, Value>;
|
||||
|
||||
/// Endpoint information extracted from OpenAPI spec
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Endpoint {
|
||||
/// HTTP method (GET, POST, etc.)
|
||||
pub method: String,
|
||||
/// Path template (e.g., "/blocks/{hash}")
|
||||
pub path: String,
|
||||
/// Operation ID (e.g., "getBlockByHash")
|
||||
pub operation_id: Option<String>,
|
||||
/// Short summary
|
||||
pub summary: Option<String>,
|
||||
/// Detailed description
|
||||
pub description: Option<String>,
|
||||
/// Tags for grouping
|
||||
pub tags: Vec<String>,
|
||||
/// Path parameters
|
||||
pub path_params: Vec<Parameter>,
|
||||
/// Query parameters
|
||||
pub query_params: Vec<Parameter>,
|
||||
/// Response type (simplified)
|
||||
pub response_type: Option<String>,
|
||||
/// Whether this endpoint is deprecated
|
||||
pub deprecated: bool,
|
||||
}
|
||||
|
||||
impl Endpoint {
|
||||
/// Returns true if this endpoint should be included in client generation.
|
||||
/// Only non-deprecated GET endpoints are included.
|
||||
pub fn should_generate(&self) -> bool {
|
||||
self.method == "GET" && !self.deprecated
|
||||
}
|
||||
|
||||
/// Returns the operation ID or generates one from the path.
|
||||
/// The returned string uses the raw case from the spec (typically camelCase).
|
||||
pub fn operation_name(&self) -> String {
|
||||
if let Some(op_id) = &self.operation_id {
|
||||
return op_id.clone();
|
||||
}
|
||||
// Generate from path: /api/block/{hash} -> "get_block"
|
||||
// Skip "api" prefix, convert hyphens to underscores, avoid redundant param names
|
||||
let mut parts: Vec<String> = Vec::new();
|
||||
let mut prev_segment = "";
|
||||
|
||||
for segment in self.path.split('/').filter(|s| !s.is_empty()) {
|
||||
if segment == "api" {
|
||||
continue;
|
||||
}
|
||||
if let Some(param) = segment.strip_prefix('{').and_then(|s| s.strip_suffix('}')) {
|
||||
// Only add "by_{param}" if the previous segment doesn't already contain the param name
|
||||
let prev_normalized = prev_segment.replace('-', "_");
|
||||
if !prev_normalized.ends_with(param) {
|
||||
parts.push(format!("by_{}", param));
|
||||
}
|
||||
} else {
|
||||
let normalized = segment.replace('-', "_");
|
||||
parts.push(normalized);
|
||||
prev_segment = segment;
|
||||
}
|
||||
}
|
||||
format!("get_{}", parts.join("_"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameter information
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Parameter {
|
||||
pub name: String,
|
||||
pub required: bool,
|
||||
pub param_type: String,
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
/// Parse OpenAPI spec from JSON string
|
||||
///
|
||||
/// Pre-processes the JSON to handle oas3 limitations:
|
||||
/// - Removes unsupported siblings from `$ref` objects (oas3 only supports `summary` and `description`)
|
||||
pub fn parse_openapi_json(json: &str) -> io::Result<Spec> {
|
||||
let mut value: Value =
|
||||
serde_json::from_str(json).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
|
||||
|
||||
// Clean up for oas3 compatibility
|
||||
clean_for_oas3(&mut value);
|
||||
|
||||
let cleaned_json =
|
||||
serde_json::to_string(&value).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
|
||||
|
||||
oas3::from_json(&cleaned_json).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
|
||||
}
|
||||
|
||||
/// Extract type schemas from OpenAPI JSON
|
||||
pub fn extract_schemas(json: &str) -> TypeSchemas {
|
||||
let Ok(value) = serde_json::from_str::<Value>(json) else {
|
||||
return BTreeMap::new();
|
||||
};
|
||||
|
||||
value
|
||||
.get("components")
|
||||
.and_then(|c| c.get("schemas"))
|
||||
.and_then(|s| s.as_object())
|
||||
.map(|schemas| {
|
||||
schemas
|
||||
.iter()
|
||||
.map(|(name, schema)| (name.clone(), schema.clone()))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Clean up OpenAPI spec for oas3 compatibility.
|
||||
/// - Removes unsupported siblings from $ref objects (oas3 only supports summary and description)
|
||||
/// - Converts boolean schemas to object schemas (oas3 doesn't handle `"schema": true`)
|
||||
fn clean_for_oas3(value: &mut Value) {
|
||||
match value {
|
||||
Value::Object(map) => {
|
||||
// Handle $ref with unsupported siblings
|
||||
if map.contains_key("$ref") {
|
||||
map.retain(|k, _| k == "$ref" || k == "summary" || k == "description");
|
||||
} else {
|
||||
// Convert boolean schemas to empty object schemas
|
||||
if let Some(schema) = map.get_mut("schema")
|
||||
&& schema.is_boolean()
|
||||
{
|
||||
*schema = Value::Object(serde_json::Map::new());
|
||||
}
|
||||
for v in map.values_mut() {
|
||||
clean_for_oas3(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
Value::Array(arr) => {
|
||||
for v in arr {
|
||||
clean_for_oas3(v);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract all endpoints from OpenAPI spec
|
||||
pub fn extract_endpoints(spec: &Spec) -> Vec<Endpoint> {
|
||||
let mut endpoints = Vec::new();
|
||||
|
||||
let Some(paths) = &spec.paths else {
|
||||
return endpoints;
|
||||
};
|
||||
|
||||
for (path, path_item) in paths {
|
||||
for (method, operation) in get_operations(path_item) {
|
||||
if let Some(endpoint) = extract_endpoint(path, &method, operation) {
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
endpoints
|
||||
}
|
||||
|
||||
fn get_operations(path_item: &PathItem) -> Vec<(String, &Operation)> {
|
||||
let mut ops = Vec::new();
|
||||
if let Some(op) = &path_item.get {
|
||||
ops.push(("GET".to_string(), op));
|
||||
}
|
||||
if let Some(op) = &path_item.post {
|
||||
ops.push(("POST".to_string(), op));
|
||||
}
|
||||
if let Some(op) = &path_item.put {
|
||||
ops.push(("PUT".to_string(), op));
|
||||
}
|
||||
if let Some(op) = &path_item.delete {
|
||||
ops.push(("DELETE".to_string(), op));
|
||||
}
|
||||
if let Some(op) = &path_item.patch {
|
||||
ops.push(("PATCH".to_string(), op));
|
||||
}
|
||||
ops
|
||||
}
|
||||
|
||||
fn extract_endpoint(path: &str, method: &str, operation: &Operation) -> Option<Endpoint> {
|
||||
let path_params = extract_parameters(operation, ParameterIn::Path);
|
||||
let query_params = extract_parameters(operation, ParameterIn::Query);
|
||||
|
||||
let response_type = extract_response_type(operation);
|
||||
|
||||
Some(Endpoint {
|
||||
method: method.to_string(),
|
||||
path: path.to_string(),
|
||||
operation_id: operation.operation_id.clone(),
|
||||
summary: operation.summary.clone(),
|
||||
description: operation.description.clone(),
|
||||
tags: operation.tags.clone(),
|
||||
path_params,
|
||||
query_params,
|
||||
response_type,
|
||||
deprecated: operation.deprecated.unwrap_or(false),
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_parameters(operation: &Operation, location: ParameterIn) -> Vec<Parameter> {
|
||||
operation
|
||||
.parameters
|
||||
.iter()
|
||||
.filter_map(|p| match p {
|
||||
ObjectOrReference::Object(param) if param.location == location => {
|
||||
let param_type = param
|
||||
.schema
|
||||
.as_ref()
|
||||
.and_then(|s| match s {
|
||||
ObjectOrReference::Ref { ref_path, .. } => {
|
||||
ref_to_type_name(ref_path).map(|s| s.to_string())
|
||||
}
|
||||
ObjectOrReference::Object(obj_schema) => schema_to_type_name(obj_schema),
|
||||
})
|
||||
.unwrap_or_else(|| "string".to_string());
|
||||
Some(Parameter {
|
||||
name: param.name.clone(),
|
||||
required: param.required.unwrap_or(false),
|
||||
param_type,
|
||||
description: param.description.clone(),
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn extract_response_type(operation: &Operation) -> Option<String> {
|
||||
let responses = operation.responses.as_ref()?;
|
||||
|
||||
// Look for 200 OK response
|
||||
let response = responses.get("200")?;
|
||||
|
||||
match response {
|
||||
ObjectOrReference::Object(response) => {
|
||||
// Look for JSON content
|
||||
let content = response.content.get("application/json")?;
|
||||
|
||||
match &content.schema {
|
||||
Some(ObjectOrReference::Ref { ref_path, .. }) => {
|
||||
// Extract type name from reference like "#/components/schemas/Block"
|
||||
Some(ref_to_type_name(ref_path)?.to_string())
|
||||
}
|
||||
Some(ObjectOrReference::Object(schema)) => schema_to_type_name(schema),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
ObjectOrReference::Ref { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn schema_type_from_schema(schema: &Schema) -> Option<String> {
|
||||
match schema {
|
||||
Schema::Boolean(_) => Some("boolean".to_string()),
|
||||
Schema::Object(obj_or_ref) => match obj_or_ref.as_ref() {
|
||||
ObjectOrReference::Object(obj_schema) => schema_to_type_name(obj_schema),
|
||||
ObjectOrReference::Ref { ref_path, .. } => {
|
||||
// Return the type name as-is (e.g., "Height", "Address")
|
||||
// These should have definitions generated from schemas
|
||||
ref_to_type_name(ref_path).map(|s| s.to_string())
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn schema_to_type_name(schema: &oas3::spec::ObjectSchema) -> Option<String> {
|
||||
let schema_type = schema.schema_type.as_ref()?;
|
||||
|
||||
match schema_type {
|
||||
SchemaTypeSet::Single(t) => match t {
|
||||
oas3::spec::SchemaType::String => Some("string".to_string()),
|
||||
oas3::spec::SchemaType::Number => Some("number".to_string()),
|
||||
oas3::spec::SchemaType::Integer => Some("number".to_string()),
|
||||
oas3::spec::SchemaType::Boolean => Some("boolean".to_string()),
|
||||
oas3::spec::SchemaType::Array => {
|
||||
let inner = match &schema.items {
|
||||
Some(boxed_schema) => schema_type_from_schema(boxed_schema),
|
||||
None => Some("*".to_string()),
|
||||
};
|
||||
inner.map(|t| format!("{}[]", t))
|
||||
}
|
||||
oas3::spec::SchemaType::Object => Some("Object".to_string()),
|
||||
oas3::spec::SchemaType::Null => Some("null".to_string()),
|
||||
},
|
||||
SchemaTypeSet::Multiple(_) => Some("*".to_string()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
//! Language-specific syntax traits for code generation.
|
||||
//!
|
||||
//! This module defines the `LanguageSyntax` trait that abstracts over
|
||||
//! language-specific code generation patterns, allowing shared generation
|
||||
//! logic to work across Python, JavaScript, and Rust backends.
|
||||
|
||||
use crate::{FieldNamePosition, GenericSyntax};
|
||||
|
||||
/// Language-specific syntax for code generation.
|
||||
///
|
||||
/// Implementations of this trait provide the language-specific formatting
|
||||
/// for generated client code. This allows the core generation logic to be
|
||||
/// written once and reused across all supported languages.
|
||||
pub trait LanguageSyntax {
|
||||
/// Convert a field name to the language's naming convention.
|
||||
///
|
||||
/// - Python/Rust: `snake_case`
|
||||
/// - JavaScript: `camelCase`
|
||||
fn field_name(&self, name: &str) -> String;
|
||||
|
||||
/// Format an interpolated path expression.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `base_var` - The variable name to interpolate (e.g., "acc", "base_path")
|
||||
/// * `suffix` - The suffix to append (e.g., "_field_name")
|
||||
///
|
||||
/// # Returns
|
||||
/// - Python: `f'{acc}_suffix'`
|
||||
/// - JavaScript: `` `${acc}_suffix` ``
|
||||
/// - Rust: `format!("{acc}_suffix")`
|
||||
fn path_expr(&self, base_var: &str, suffix: &str) -> String;
|
||||
|
||||
/// Format a `FieldNamePosition` as a path expression.
|
||||
///
|
||||
/// This handles the different name transformation patterns (append, prepend,
|
||||
/// identity, set_base) in a language-specific way.
|
||||
fn position_expr(&self, pos: &FieldNamePosition, base_var: &str) -> String;
|
||||
|
||||
/// Generate a constructor call for patterns and accessors.
|
||||
///
|
||||
/// - Python: `TypeName(client, path)`
|
||||
/// - JavaScript: `createTypeName(client, path)`
|
||||
/// - Rust: `TypeName::new(client.clone(), path)`
|
||||
fn constructor(&self, type_name: &str, path_expr: &str) -> String;
|
||||
|
||||
/// Generate a field initialization line.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `indent` - The indentation string
|
||||
/// * `name` - The field name (already converted to language convention)
|
||||
/// * `type_ann` - The type annotation (may be ignored by some languages)
|
||||
/// * `value` - The initialization value/expression
|
||||
///
|
||||
/// # Returns
|
||||
/// - Python: `{indent}self.{name}: {type_ann} = {value}`
|
||||
/// - JavaScript: `{indent}{name}: {value},`
|
||||
/// - Rust: `{indent}{name}: {value},`
|
||||
fn field_init(&self, indent: &str, name: &str, type_ann: &str, value: &str) -> String;
|
||||
|
||||
/// Get the generic type syntax for this language.
|
||||
///
|
||||
/// - Python: `[T]` with default `Any`
|
||||
/// - JavaScript: `<T>` with default `unknown`
|
||||
/// - Rust: `<T>` with default `_`
|
||||
fn generic_syntax(&self) -> GenericSyntax;
|
||||
|
||||
/// Generate a struct/class header.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `name` - The type name
|
||||
/// * `generic_params` - Generic parameters (e.g., "<T>" or "[T]"), empty if none
|
||||
/// * `doc` - Optional documentation string
|
||||
fn struct_header(&self, name: &str, generic_params: &str, doc: Option<&str>) -> String;
|
||||
|
||||
/// Generate a struct/class footer.
|
||||
fn struct_footer(&self) -> String;
|
||||
|
||||
/// Generate a constructor/init method header.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `params` - Constructor parameters (language-specific format)
|
||||
fn constructor_header(&self, params: &str) -> String;
|
||||
|
||||
/// Generate a constructor/init method footer.
|
||||
fn constructor_footer(&self) -> String;
|
||||
|
||||
/// Generate a field declaration (for struct body, not init).
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `indent` - The indentation string
|
||||
/// * `name` - The field name
|
||||
/// * `type_ann` - The type annotation
|
||||
fn field_declaration(&self, indent: &str, name: &str, type_ann: &str) -> String;
|
||||
|
||||
/// Format an index field name from an Index.
|
||||
///
|
||||
/// E.g., `by_date_height`, `by_date`, etc.
|
||||
fn index_field_name(&self, index_name: &str) -> String;
|
||||
|
||||
/// Format a string literal.
|
||||
///
|
||||
/// - Python/JavaScript: `'value'` (single quotes)
|
||||
/// - Rust: `"value"` (double quotes)
|
||||
fn string_literal(&self, value: &str) -> String;
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
use brk_types::Index;
|
||||
|
||||
/// Convert a string to PascalCase (e.g., "fee_rate" -> "FeeRate").
|
||||
pub fn to_pascal_case(s: &str) -> String {
|
||||
s.replace('-', "_")
|
||||
.split('_')
|
||||
.map(|word| {
|
||||
let mut chars = word.chars();
|
||||
match chars.next() {
|
||||
None => String::new(),
|
||||
Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Convert a string to snake_case, handling Rust keywords.
|
||||
pub fn to_snake_case(s: &str) -> String {
|
||||
// Convert to lowercase and replace dashes with underscores
|
||||
let sanitized = s.to_lowercase().replace('-', "_");
|
||||
|
||||
// Prefix with _ if starts with digit
|
||||
let sanitized = if sanitized.chars().next().is_some_and(|c| c.is_ascii_digit()) {
|
||||
format!("_{}", sanitized)
|
||||
} else {
|
||||
sanitized
|
||||
};
|
||||
|
||||
// Handle Rust keywords
|
||||
match sanitized.as_str() {
|
||||
"type" | "const" | "static" | "match" | "if" | "else" | "loop" | "while" | "for"
|
||||
| "break" | "continue" | "return" | "fn" | "let" | "mut" | "ref" | "self" | "super"
|
||||
| "mod" | "use" | "pub" | "crate" | "extern" | "impl" | "trait" | "struct" | "enum"
|
||||
| "where" | "async" | "await" | "dyn" | "move" => format!("r#{}", sanitized),
|
||||
_ => sanitized,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a string to camelCase (e.g., "fee_rate" -> "feeRate").
|
||||
pub fn to_camel_case(s: &str) -> String {
|
||||
let pascal = to_pascal_case(s);
|
||||
let mut chars = pascal.chars();
|
||||
|
||||
let result = match chars.next() {
|
||||
None => String::new(),
|
||||
Some(first) => first.to_lowercase().collect::<String>() + chars.as_str(),
|
||||
};
|
||||
|
||||
// Prefix with _ if starts with digit
|
||||
if result.chars().next().is_some_and(|c| c.is_ascii_digit()) {
|
||||
format!("_{}", result)
|
||||
} else {
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert an Index to a snake_case field name (e.g., DateIndex -> by_dateindex).
|
||||
pub fn index_to_field_name(index: &Index) -> String {
|
||||
format!("by_{}", to_snake_case(index.serialize_long()))
|
||||
}
|
||||
|
||||
/// Generate a child type/struct/class name (e.g., ParentName + child_name -> ParentName_ChildName).
|
||||
pub fn child_type_name(parent: &str, child: &str) -> String {
|
||||
format!("{}_{}", parent, to_pascal_case(child))
|
||||
}
|
||||
|
||||
/// Escape Python reserved keywords by appending an underscore.
|
||||
/// Also prefixes names starting with digits with an underscore.
|
||||
pub fn escape_python_keyword(name: &str) -> String {
|
||||
const PYTHON_KEYWORDS: &[&str] = &[
|
||||
"False", "None", "True", "and", "as", "assert", "async", "await", "break", "class",
|
||||
"continue", "def", "del", "elif", "else", "except", "finally", "for", "from", "global",
|
||||
"if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return",
|
||||
"try", "while", "with", "yield",
|
||||
];
|
||||
|
||||
// Prefix with underscore if starts with digit
|
||||
let name = if name.starts_with(|c: char| c.is_ascii_digit()) {
|
||||
format!("_{}", name)
|
||||
} else {
|
||||
name.to_string()
|
||||
};
|
||||
|
||||
// Append underscore if it's a keyword
|
||||
if PYTHON_KEYWORDS.contains(&name.as_str()) {
|
||||
format!("{}_", name)
|
||||
} else {
|
||||
name
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
//! Client metadata extracted from brk_query.
|
||||
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
|
||||
use brk_query::Vecs;
|
||||
use brk_types::Index;
|
||||
|
||||
use super::{GenericSyntax, IndexSetPattern, PatternField, StructuralPattern, extract_inner_type};
|
||||
use crate::analysis;
|
||||
|
||||
/// Metadata extracted from brk_query for client generation.
|
||||
#[derive(Debug)]
|
||||
pub struct ClientMetadata {
|
||||
/// The catalog tree structure (with schemas in leaves)
|
||||
pub catalog: brk_types::TreeNode,
|
||||
/// Structural patterns - tree node shapes that repeat
|
||||
pub structural_patterns: Vec<StructuralPattern>,
|
||||
/// All indexes used across the catalog
|
||||
pub used_indexes: BTreeSet<Index>,
|
||||
/// Index set patterns - sets of indexes that appear together on metrics
|
||||
pub index_set_patterns: Vec<IndexSetPattern>,
|
||||
/// Maps concrete field signatures to pattern names
|
||||
concrete_to_pattern: HashMap<Vec<PatternField>, String>,
|
||||
/// Maps concrete field signatures to their type parameter (for generic patterns)
|
||||
concrete_to_type_param: HashMap<Vec<PatternField>, String>,
|
||||
}
|
||||
|
||||
impl ClientMetadata {
|
||||
/// Extract metadata from brk_query::Vecs.
|
||||
pub fn from_vecs(vecs: &Vecs) -> Self {
|
||||
let catalog = vecs.catalog().clone();
|
||||
let (structural_patterns, concrete_to_pattern, concrete_to_type_param) =
|
||||
analysis::detect_structural_patterns(&catalog);
|
||||
let (used_indexes, index_set_patterns) = analysis::detect_index_patterns(&catalog);
|
||||
|
||||
ClientMetadata {
|
||||
catalog,
|
||||
structural_patterns,
|
||||
used_indexes,
|
||||
index_set_patterns,
|
||||
concrete_to_pattern,
|
||||
concrete_to_type_param,
|
||||
}
|
||||
}
|
||||
|
||||
/// Find an index set pattern that matches the given indexes.
|
||||
pub fn find_index_set_pattern(&self, indexes: &BTreeSet<Index>) -> Option<&IndexSetPattern> {
|
||||
self.index_set_patterns
|
||||
.iter()
|
||||
.find(|p| &p.indexes == indexes)
|
||||
}
|
||||
|
||||
/// Check if a type is a structural pattern name.
|
||||
pub fn is_pattern_type(&self, type_name: &str) -> bool {
|
||||
self.structural_patterns.iter().any(|p| p.name == type_name)
|
||||
}
|
||||
|
||||
/// Find a pattern by name.
|
||||
pub fn find_pattern(&self, name: &str) -> Option<&StructuralPattern> {
|
||||
self.structural_patterns.iter().find(|p| p.name == name)
|
||||
}
|
||||
|
||||
/// Check if a pattern is generic.
|
||||
pub fn is_pattern_generic(&self, name: &str) -> bool {
|
||||
self.find_pattern(name).is_some_and(|p| p.is_generic)
|
||||
}
|
||||
|
||||
/// Get the type parameter for a generic pattern given its concrete fields.
|
||||
pub fn get_type_param(&self, fields: &[PatternField]) -> Option<&String> {
|
||||
self.concrete_to_type_param.get(fields)
|
||||
}
|
||||
|
||||
/// Build a lookup map from field signatures to pattern names.
|
||||
pub fn pattern_lookup(&self) -> HashMap<Vec<PatternField>, String> {
|
||||
let mut lookup = self.concrete_to_pattern.clone();
|
||||
for p in &self.structural_patterns {
|
||||
lookup.insert(p.fields.clone(), p.name.clone());
|
||||
}
|
||||
lookup
|
||||
}
|
||||
|
||||
/// Check if a field should use a shared index accessor.
|
||||
pub fn field_uses_accessor(&self, field: &PatternField) -> bool {
|
||||
self.find_index_set_pattern(&field.indexes).is_some()
|
||||
}
|
||||
|
||||
/// Generate type annotation for a field with language-specific syntax.
|
||||
pub fn field_type_annotation(
|
||||
&self,
|
||||
field: &PatternField,
|
||||
is_generic: bool,
|
||||
generic_value_type: Option<&str>,
|
||||
syntax: GenericSyntax,
|
||||
) -> String {
|
||||
let value_type = if is_generic && field.rust_type == "T" {
|
||||
"T".to_string()
|
||||
} else {
|
||||
extract_inner_type(&field.rust_type)
|
||||
};
|
||||
|
||||
if self.is_pattern_type(&field.rust_type) {
|
||||
if self.is_pattern_generic(&field.rust_type) {
|
||||
let type_param = field
|
||||
.type_param
|
||||
.as_deref()
|
||||
.or(generic_value_type)
|
||||
.unwrap_or(if is_generic { "T" } else { syntax.default_type });
|
||||
return syntax.wrap(&field.rust_type, type_param);
|
||||
}
|
||||
field.rust_type.clone()
|
||||
} else if field.is_branch() {
|
||||
field.rust_type.clone()
|
||||
} else if let Some(accessor) = self.find_index_set_pattern(&field.indexes) {
|
||||
syntax.wrap(&accessor.name, &value_type)
|
||||
} else {
|
||||
syntax.wrap("MetricNode", &value_type)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
//! Core types for client generation.
|
||||
|
||||
mod case;
|
||||
mod metadata;
|
||||
mod positions;
|
||||
mod schema;
|
||||
mod structs;
|
||||
|
||||
pub use case::*;
|
||||
pub use metadata::*;
|
||||
pub use positions::*;
|
||||
pub use schema::*;
|
||||
pub use structs::*;
|
||||
|
||||
/// Language-specific syntax for generic type annotations.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct GenericSyntax {
|
||||
pub open: char,
|
||||
pub close: char,
|
||||
pub default_type: &'static str,
|
||||
}
|
||||
|
||||
impl GenericSyntax {
|
||||
pub const PYTHON: Self = Self { open: '[', close: ']', default_type: "Any" };
|
||||
pub const JAVASCRIPT: Self = Self { open: '<', close: '>', default_type: "unknown" };
|
||||
pub const RUST: Self = Self { open: '<', close: '>', default_type: "_" };
|
||||
|
||||
pub fn wrap(&self, name: &str, type_param: &str) -> String {
|
||||
format!("{}{}{}{}", name, self.open, type_param, self.close)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
//! Field name position types for metric name reconstruction.
|
||||
|
||||
/// How a field modifies the accumulated metric name.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum FieldNamePosition {
|
||||
/// Field prepends a prefix: leaf.name() = prefix + accumulated
|
||||
Prepend(String),
|
||||
/// Field appends a suffix: leaf.name() = accumulated + suffix
|
||||
Append(String),
|
||||
/// Field IS the accumulated name (no modification)
|
||||
Identity,
|
||||
/// Field sets a new base name (used at pattern entry points)
|
||||
SetBase(String),
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
use serde_json::Value;
|
||||
|
||||
/// Unwrap allOf with a single element, returning the inner schema.
|
||||
/// Schemars uses allOf for composition, but often with just one $ref.
|
||||
pub fn unwrap_allof(schema: &Value) -> &Value {
|
||||
if let Some(all_of) = schema.get("allOf").and_then(|v| v.as_array())
|
||||
&& all_of.len() == 1
|
||||
{
|
||||
return &all_of[0];
|
||||
}
|
||||
schema
|
||||
}
|
||||
|
||||
/// Extract inner type from a wrapper generic like `Close<Dollars>` -> `Dollars`.
|
||||
/// Also handles malformed types like `Dollars>` (from vecdb's short_type_name).
|
||||
pub fn extract_inner_type(type_str: &str) -> String {
|
||||
// Handle proper generic wrappers like `Close<Dollars>` -> `Dollars`
|
||||
if let Some(start) = type_str.find('<')
|
||||
&& let Some(end) = type_str.rfind('>')
|
||||
&& start < end
|
||||
{
|
||||
return type_str[start + 1..end].to_string();
|
||||
}
|
||||
// Handle malformed types like `Dollars>` (trailing > without <)
|
||||
if type_str.ends_with('>') && !type_str.contains('<') {
|
||||
return type_str.trim_end_matches('>').to_string();
|
||||
}
|
||||
type_str.to_string()
|
||||
}
|
||||
|
||||
/// Extract JSON type from a schema ("integer", "number", "string", etc).
|
||||
pub fn schema_to_json_type(schema: &Value) -> String {
|
||||
schema
|
||||
.get("type")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("object")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Extract type name from a JSON Schema $ref path.
|
||||
/// E.g., "#/definitions/MyType" -> "MyType", "#/$defs/Foo" -> "Foo"
|
||||
pub fn ref_to_type_name(ref_path: &str) -> Option<&str> {
|
||||
ref_path.rsplit('/').next()
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
//! Structural pattern and field types.
|
||||
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
|
||||
use brk_types::Index;
|
||||
|
||||
use super::FieldNamePosition;
|
||||
|
||||
/// A pattern of indexes that appear together on multiple metrics.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IndexSetPattern {
|
||||
/// Pattern name (e.g., "DateHeightIndexes")
|
||||
pub name: String,
|
||||
/// The set of indexes
|
||||
pub indexes: BTreeSet<Index>,
|
||||
}
|
||||
|
||||
/// A structural pattern - a branch structure that appears multiple times.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct StructuralPattern {
|
||||
/// Pattern name
|
||||
pub name: String,
|
||||
/// Ordered list of child fields
|
||||
pub fields: Vec<PatternField>,
|
||||
/// How each field modifies the accumulated name
|
||||
pub field_positions: HashMap<String, FieldNamePosition>,
|
||||
/// If true, all leaf fields use a type parameter T
|
||||
pub is_generic: bool,
|
||||
}
|
||||
|
||||
impl StructuralPattern {
|
||||
/// Returns true if this pattern contains any leaf fields.
|
||||
pub fn contains_leaves(&self) -> bool {
|
||||
self.fields.iter().any(|f| f.is_leaf())
|
||||
}
|
||||
|
||||
/// Returns true if all leaf fields have consistent name transformations.
|
||||
pub fn is_parameterizable(&self) -> bool {
|
||||
!self.field_positions.is_empty()
|
||||
&& self
|
||||
.fields
|
||||
.iter()
|
||||
.all(|f| f.is_branch() || self.field_positions.contains_key(&f.name))
|
||||
}
|
||||
|
||||
/// Get the field position for a given field name.
|
||||
pub fn get_field_position(&self, field_name: &str) -> Option<&FieldNamePosition> {
|
||||
self.field_positions.get(field_name)
|
||||
}
|
||||
}
|
||||
|
||||
/// A field in a structural pattern.
|
||||
#[derive(Debug, Clone, PartialOrd, Ord)]
|
||||
pub struct PatternField {
|
||||
/// Field name
|
||||
pub name: String,
|
||||
/// Rust type for leaves or pattern name for branches
|
||||
pub rust_type: String,
|
||||
/// JSON type from schema
|
||||
pub json_type: String,
|
||||
/// For leaves: the set of supported indexes. Empty for branches.
|
||||
pub indexes: BTreeSet<Index>,
|
||||
/// For branches referencing generic patterns: the concrete type parameter
|
||||
pub type_param: Option<String>,
|
||||
}
|
||||
|
||||
impl PatternField {
|
||||
/// Returns true if this is a leaf field (has indexes).
|
||||
pub fn is_leaf(&self) -> bool {
|
||||
!self.indexes.is_empty()
|
||||
}
|
||||
|
||||
/// Returns true if this is a branch field (no indexes).
|
||||
pub fn is_branch(&self) -> bool {
|
||||
self.indexes.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::hash::Hash for PatternField {
|
||||
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
|
||||
self.name.hash(state);
|
||||
self.rust_type.hash(state);
|
||||
self.json_type.hash(state);
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for PatternField {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.name == other.name
|
||||
&& self.rust_type == other.rust_type
|
||||
&& self.json_type == other.json_type
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for PatternField {}
|
||||
@@ -6,11 +6,15 @@ edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
build = "build.rs"
|
||||
|
||||
[dependencies]
|
||||
log = { workspace = true }
|
||||
notify = "8.1.0"
|
||||
brk_rolldown = "0.1.0"
|
||||
# brk_rolldown = { path = "../../../rolldown/crates/rolldown"}
|
||||
sugar_path = "1.2.0"
|
||||
notify = "8.2.0"
|
||||
# rolldown = { path = "../../../rolldown/crates/rolldown", package = "brk_rolldown" }
|
||||
rolldown = { version = "0.7.0", package = "brk_rolldown" }
|
||||
sugar_path = "1.2.1"
|
||||
tokio = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
env_logger = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
# brk_bundler
|
||||
|
||||
JavaScript bundling with watch mode for BRK web interfaces.
|
||||
|
||||
## What It Enables
|
||||
|
||||
Bundle and minify JavaScript modules using Rolldown, with file watching for development. Handles module copying, source map generation, and cache-busting via hashed filenames.
|
||||
|
||||
## Key Features
|
||||
|
||||
- **Rolldown integration**: Fast Rust-based bundler with tree-shaking and minification
|
||||
- **Watch mode**: Rebuilds on file changes with live module syncing
|
||||
- **Source maps**: Full debugging support in production builds
|
||||
- **Cache busting**: Hashes main bundle filename, updates HTML references automatically
|
||||
- **Service worker versioning**: Injects package version into service worker files
|
||||
|
||||
## Core API
|
||||
|
||||
```rust,ignore
|
||||
// One-shot build
|
||||
let dist = bundle(modules_path, websites_path, "src", false).await?;
|
||||
|
||||
// Watch mode for development
|
||||
bundle(modules_path, websites_path, "src", true).await?;
|
||||
```
|
||||
|
||||
## Build Pipeline
|
||||
|
||||
1. Copy shared modules to source scripts directory
|
||||
2. Bundle with Rolldown (minified, with source maps)
|
||||
3. Update `index.html` with hashed script references
|
||||
4. Inject version into service worker
|
||||
@@ -0,0 +1,8 @@
|
||||
fn main() {
|
||||
let profile = std::env::var("PROFILE").unwrap_or_default();
|
||||
|
||||
if profile == "release" {
|
||||
println!("cargo:rustc-flag=-C");
|
||||
println!("cargo:rustc-flag=target-cpu=native");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
use std::{io, path::PathBuf, thread, time::Duration};
|
||||
|
||||
use brk_bundler::bundle;
|
||||
|
||||
fn find_dev_dirs() -> Option<(PathBuf, PathBuf)> {
|
||||
let mut dir = std::env::current_dir().ok()?;
|
||||
loop {
|
||||
let websites = dir.join("websites");
|
||||
let modules = dir.join("modules");
|
||||
if websites.exists() && modules.exists() {
|
||||
return Some((websites, modules));
|
||||
}
|
||||
// Stop at workspace root (crates/ indicates we're there)
|
||||
if dir.join("crates").exists() {
|
||||
return None;
|
||||
}
|
||||
dir = dir.parent()?.to_path_buf();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> io::Result<()> {
|
||||
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("debug")).init();
|
||||
|
||||
let (websites_path, modules_path) =
|
||||
find_dev_dirs().expect("Run from within the brk workspace");
|
||||
let source_folder = "bitview";
|
||||
|
||||
let dist_path = bundle(&modules_path, &websites_path, source_folder, true).await?;
|
||||
|
||||
println!("Bundle created at: {}", dist_path.display());
|
||||
println!("Watching for changes... (Ctrl+C to stop)");
|
||||
|
||||
loop {
|
||||
thread::sleep(Duration::from_secs(60));
|
||||
}
|
||||
}
|
||||
@@ -1,133 +1,234 @@
|
||||
use std::{fs, io, path::Path, sync::Arc};
|
||||
#![doc = include_str!("../README.md")]
|
||||
|
||||
use brk_rolldown::{Bundler, BundlerOptions, RawMinifyOptions, SourceMapType};
|
||||
use log::error;
|
||||
use notify::{EventKind, RecursiveMode, Watcher};
|
||||
use std::{
|
||||
fs, io,
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use log::{debug, error, info};
|
||||
use notify::{EventKind, PollWatcher, RecursiveMode, Watcher};
|
||||
use rolldown::{
|
||||
Bundler, BundlerConfig, BundlerOptions, InlineConstConfig, InlineConstMode, InlineConstOption,
|
||||
OptimizationOption, RawMinifyOptions, SourceMapType,
|
||||
};
|
||||
use sugar_path::SugarPath;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> io::Result<()> {
|
||||
let source_path = websites_path.join(source_folder);
|
||||
let dist_path = websites_path.join("dist");
|
||||
|
||||
let _ = fs::remove_dir_all(&dist_path);
|
||||
copy_dir_all(&source_path, &dist_path)?;
|
||||
|
||||
let source_scripts = format!("./{source_folder}/scripts");
|
||||
let source_entry = format!("{source_scripts}/entry.js");
|
||||
pub async fn bundle(
|
||||
modules_path: &Path,
|
||||
websites_path: &Path,
|
||||
source_folder: &str,
|
||||
watch: bool,
|
||||
) -> io::Result<PathBuf> {
|
||||
let relative_modules_path = modules_path;
|
||||
let relative_source_path = websites_path.join(source_folder);
|
||||
let relative_dist_path = websites_path.join("dist");
|
||||
|
||||
let absolute_modules_path = relative_modules_path.absolutize();
|
||||
let absolute_modules_path_clone = absolute_modules_path.clone();
|
||||
let absolute_websites_path = websites_path.absolutize();
|
||||
let absolute_websites_path_clone = absolute_websites_path.clone();
|
||||
|
||||
let mut bundler = Bundler::new(BundlerOptions {
|
||||
input: Some(vec![source_entry.into()]),
|
||||
let absolute_source_path = relative_source_path.absolutize();
|
||||
let absolute_source_index_path = absolute_source_path.join("index.html");
|
||||
let absolute_source_index_path_clone = absolute_source_index_path.clone();
|
||||
let absolute_source_scripts_path = absolute_source_path.join("scripts");
|
||||
let absolute_source_scripts_modules_path = absolute_source_scripts_path.join("modules");
|
||||
let absolute_source_sw_path = absolute_source_path.join("service-worker.js");
|
||||
let absolute_source_sw_path_clone = absolute_source_sw_path.clone();
|
||||
|
||||
let absolute_dist_path = relative_dist_path.absolutize();
|
||||
let absolute_dist_scripts_path = absolute_dist_path.join("scripts");
|
||||
let absolute_dist_scripts_entry_path = absolute_dist_scripts_path.join("entry.js");
|
||||
let absolute_dist_scripts_entry_path_clone = absolute_dist_scripts_entry_path.clone();
|
||||
let absolute_dist_index_path = absolute_dist_path.join("index.html");
|
||||
let absolute_dist_sw_path = absolute_dist_path.join("service-worker.js");
|
||||
|
||||
info!("Bundling {source_folder}...");
|
||||
info!(" modules: {absolute_modules_path:?}");
|
||||
info!(" source: {absolute_source_path:?}");
|
||||
info!(" dist: {absolute_dist_path:?}");
|
||||
|
||||
let _ = fs::remove_dir_all(&absolute_dist_path);
|
||||
let _ = fs::remove_dir_all(&absolute_source_scripts_modules_path);
|
||||
copy_dir_all(
|
||||
&absolute_modules_path,
|
||||
&absolute_source_scripts_modules_path,
|
||||
)?;
|
||||
copy_dir_all(&absolute_source_path, &absolute_dist_path)?;
|
||||
fs::remove_dir_all(&absolute_dist_scripts_path)?;
|
||||
fs::create_dir(&absolute_dist_scripts_path)?;
|
||||
|
||||
// dbg!(BundlerOptions::default());
|
||||
|
||||
let bundler_options = BundlerOptions {
|
||||
input: Some(vec![format!("./{source_folder}/scripts/entry.js").into()]),
|
||||
dir: Some("./dist/scripts".to_string()),
|
||||
cwd: Some(absolute_websites_path),
|
||||
minify: Some(RawMinifyOptions::Bool(true)),
|
||||
sourcemap: Some(SourceMapType::File),
|
||||
// advanced_chunks: Some(AdvancedChunksOptions {
|
||||
// // min_size: Some(1000.0),
|
||||
// min_share_count: Some(20),
|
||||
// // min_module_size: S
|
||||
// // include_dependencies_recursively: Some(true),
|
||||
// ..Default::default()
|
||||
// }),
|
||||
//
|
||||
// inline_dynamic_imports
|
||||
// experimental: Some(ExperimentalOptions {
|
||||
// strict_execution_order: Some(true),
|
||||
// ..Default::default()
|
||||
// }),
|
||||
optimization: Some(OptimizationOption {
|
||||
inline_const: Some(InlineConstOption::Config(InlineConstConfig {
|
||||
mode: Some(InlineConstMode::All),
|
||||
..Default::default()
|
||||
})),
|
||||
// Needs benchmarks
|
||||
// pife_for_module_wrappers: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
};
|
||||
|
||||
bundler.write().await.unwrap();
|
||||
let mut bundler = Bundler::new(bundler_options.clone()).unwrap();
|
||||
|
||||
let absolute_source_index_path = source_path.join("index.html").absolutize();
|
||||
let absolute_source_index_path_clone = absolute_source_index_path.clone();
|
||||
let absolute_source_path = source_path.absolutize();
|
||||
let absolute_source_path_clone = absolute_source_path.clone();
|
||||
let absolute_source_scripts_path = websites_path.join(source_scripts).absolutize();
|
||||
let absolute_source_sw_path = source_path.join("service-worker.js").absolutize();
|
||||
let absolute_source_sw_path_clone = absolute_source_sw_path.clone();
|
||||
if let Err(error) = bundler.write().await {
|
||||
error!("{error:?}");
|
||||
}
|
||||
|
||||
let absolute_dist_entry_path = dist_path.join("scripts/entry.js").absolutize();
|
||||
let absolute_dist_index_path = dist_path.join("index.html").absolutize();
|
||||
let absolute_dist_path = dist_path.absolutize();
|
||||
let absolute_dist_path_clone = absolute_dist_path.clone();
|
||||
let absolute_dist_sw_path = dist_path.join("service-worker.js").absolutize();
|
||||
|
||||
let write_index = move || {
|
||||
let update_dist_index = move || {
|
||||
let mut contents = fs::read_to_string(&absolute_source_index_path).unwrap();
|
||||
|
||||
if let Ok(entry) = fs::read_to_string(absolute_dist_path_clone.join("scripts/entry.js")) {
|
||||
if let Some(start) = entry.find("main") {
|
||||
if let Some(end) = entry.find(".js") {
|
||||
let main_hashed = &entry[start..end];
|
||||
contents =
|
||||
contents.replace("/scripts/main.js", &format!("/scripts/{main_hashed}.js"));
|
||||
}
|
||||
}
|
||||
if let Ok(entry) = fs::read_to_string(&absolute_dist_scripts_entry_path_clone)
|
||||
&& let Some(start) = entry.find("main")
|
||||
&& let Some(end) = entry.find(".js")
|
||||
{
|
||||
let main_hashed = &entry[start..end];
|
||||
contents = contents.replace("/scripts/main.js", &format!("/scripts/{main_hashed}.js"));
|
||||
}
|
||||
|
||||
let _ = fs::write(&absolute_dist_index_path, contents);
|
||||
};
|
||||
|
||||
let write_sw = move || {
|
||||
let update_source_sw = move || {
|
||||
let contents = fs::read_to_string(&absolute_source_sw_path)
|
||||
.unwrap()
|
||||
.replace("__VERSION__", &format!("v{VERSION}"));
|
||||
let _ = fs::write(&absolute_dist_sw_path, contents);
|
||||
};
|
||||
|
||||
write_index();
|
||||
write_sw();
|
||||
update_dist_index();
|
||||
update_source_sw();
|
||||
|
||||
if !watch {
|
||||
return Ok(());
|
||||
return Ok(relative_dist_path);
|
||||
}
|
||||
|
||||
// Clone paths for the second watcher
|
||||
let absolute_websites_path_clone2 = absolute_websites_path_clone.clone();
|
||||
let absolute_modules_path_clone2 = absolute_modules_path_clone.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let write_index_clone = write_index.clone();
|
||||
let handle_event = {
|
||||
let absolute_dist_scripts_entry_path = absolute_dist_scripts_entry_path.clone();
|
||||
let absolute_source_index_path_clone = absolute_source_index_path_clone.clone();
|
||||
let absolute_source_sw_path_clone = absolute_source_sw_path_clone.clone();
|
||||
let absolute_modules_path = absolute_modules_path.clone();
|
||||
let absolute_source_scripts_modules_path = absolute_source_scripts_modules_path.clone();
|
||||
let absolute_source_path = absolute_source_path.clone();
|
||||
let absolute_source_scripts_path = absolute_source_scripts_path.clone();
|
||||
let absolute_dist_path = absolute_dist_path.clone();
|
||||
let update_dist_index = update_dist_index.clone();
|
||||
let update_source_sw = update_source_sw.clone();
|
||||
|
||||
let mut entry_watcher = notify::recommended_watcher(
|
||||
move |res: Result<notify::Event, notify::Error>| match res {
|
||||
Ok(_) => write_index_clone(),
|
||||
Err(e) => error!("watch error: {e:?}"),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
move |path: PathBuf| {
|
||||
let path = path.absolutize();
|
||||
|
||||
entry_watcher
|
||||
.watch(&absolute_dist_entry_path, RecursiveMode::Recursive)
|
||||
.unwrap();
|
||||
if path == absolute_dist_scripts_entry_path
|
||||
|| path == absolute_source_index_path_clone
|
||||
{
|
||||
update_dist_index();
|
||||
} else if path == absolute_source_sw_path_clone {
|
||||
update_source_sw();
|
||||
} else if let Ok(suffix) = path.strip_prefix(&absolute_modules_path) {
|
||||
let dest = absolute_source_scripts_modules_path.join(suffix);
|
||||
if path.is_file() {
|
||||
debug!("Copying module: {path:?} -> {dest:?}");
|
||||
let _ = fs::create_dir_all(dest.parent().unwrap());
|
||||
if let Err(e) = fs::copy(&path, &dest) {
|
||||
error!("Copy failed: {e}");
|
||||
}
|
||||
}
|
||||
} else if let Ok(suffix) = path.strip_prefix(&absolute_source_path)
|
||||
// scripts are handled by rolldown
|
||||
&& !path.starts_with(&absolute_source_scripts_path)
|
||||
{
|
||||
let dist_path = absolute_dist_path.join(suffix);
|
||||
if path.is_file() {
|
||||
let _ = fs::create_dir_all(path.parent().unwrap());
|
||||
let _ = fs::copy(&path, &dist_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let mut source_watcher = notify::recommended_watcher(
|
||||
// FSEvents watcher for instant response to manual saves
|
||||
let handle_event_clone = handle_event.clone();
|
||||
let mut fs_watcher = notify::recommended_watcher(
|
||||
move |res: Result<notify::Event, notify::Error>| match res {
|
||||
Ok(event) => match event.kind {
|
||||
EventKind::Create(_) => event.paths,
|
||||
EventKind::Modify(_) => event.paths,
|
||||
_ => vec![],
|
||||
}
|
||||
.into_iter()
|
||||
.filter(|path| path.starts_with(&absolute_source_path))
|
||||
.filter(|path| !path.starts_with(&absolute_source_scripts_path))
|
||||
.for_each(|source_path| {
|
||||
let suffix = source_path.strip_prefix(&absolute_source_path).unwrap();
|
||||
let dist_path = absolute_dist_path.join(suffix);
|
||||
|
||||
if source_path == absolute_source_index_path_clone {
|
||||
write_index();
|
||||
} else if source_path == absolute_source_sw_path_clone {
|
||||
write_sw();
|
||||
} else {
|
||||
let _ = fs::copy(&source_path, &dist_path);
|
||||
EventKind::Create(_) | EventKind::Modify(_) => {
|
||||
event.paths.into_iter().for_each(&handle_event_clone);
|
||||
}
|
||||
}),
|
||||
Err(e) => error!("watch error: {e:?}"),
|
||||
_ => {}
|
||||
},
|
||||
Err(e) => error!("fs watch error: {e:?}"),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
source_watcher
|
||||
.watch(&absolute_source_path_clone, RecursiveMode::Recursive)
|
||||
fs_watcher
|
||||
.watch(&absolute_websites_path_clone, RecursiveMode::Recursive)
|
||||
.unwrap();
|
||||
fs_watcher
|
||||
.watch(&absolute_modules_path_clone, RecursiveMode::Recursive)
|
||||
.unwrap();
|
||||
|
||||
let watcher =
|
||||
brk_rolldown::Watcher::new(vec![Arc::new(Mutex::new(bundler))], None).unwrap();
|
||||
// Poll watcher to catch programmatic edits (e.g., Claude Code's atomic writes)
|
||||
let poll_config = notify::Config::default()
|
||||
.with_poll_interval(Duration::from_secs(1));
|
||||
let mut poll_watcher = PollWatcher::new(
|
||||
move |res: Result<notify::Event, notify::Error>| match res {
|
||||
Ok(event) => match event.kind {
|
||||
EventKind::Create(_) | EventKind::Modify(_) => {
|
||||
event.paths.into_iter().for_each(&handle_event);
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
Err(e) => error!("poll watch error: {e:?}"),
|
||||
},
|
||||
poll_config,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
poll_watcher
|
||||
.watch(&absolute_websites_path_clone2, RecursiveMode::Recursive)
|
||||
.unwrap();
|
||||
poll_watcher
|
||||
.watch(&absolute_modules_path_clone2, RecursiveMode::Recursive)
|
||||
.unwrap();
|
||||
|
||||
let config = BundlerConfig::new(bundler_options, vec![]);
|
||||
let watcher = rolldown::Watcher::new(config, None).unwrap();
|
||||
|
||||
watcher.start().await;
|
||||
});
|
||||
|
||||
Ok(())
|
||||
Ok(relative_dist_path)
|
||||
}
|
||||
|
||||
fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
|
||||
|
||||
@@ -1,34 +1,41 @@
|
||||
[package]
|
||||
name = "brk_cli"
|
||||
description = "A command line interface to run a Bitcoin Research Kit instance"
|
||||
description = "A command line interface to run a BRK instance"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
build = "build.rs"
|
||||
|
||||
[dependencies]
|
||||
bitcoincore-rpc = { workspace = true }
|
||||
brk_bindgen = { workspace = true }
|
||||
brk_bundler = { workspace = true }
|
||||
brk_computer = { workspace = true }
|
||||
brk_core = { workspace = true }
|
||||
brk_exit = { workspace = true }
|
||||
brk_error = { workspace = true }
|
||||
brk_fetcher = { workspace = true }
|
||||
brk_indexer = { workspace = true }
|
||||
brk_iterator = { workspace = true }
|
||||
brk_logger = { workspace = true }
|
||||
brk_parser = { workspace = true }
|
||||
brk_mempool = { workspace = true }
|
||||
brk_query = { workspace = true }
|
||||
brk_reader = { workspace = true }
|
||||
brk_rpc = { workspace = true }
|
||||
brk_server = { workspace = true }
|
||||
brk_vec = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
clap_derive = { workspace = true }
|
||||
clap = { version = "4.5.54", features = ["derive", "string"] }
|
||||
color-eyre = { workspace = true }
|
||||
brk_alloc = { workspace = true }
|
||||
log = { workspace = true }
|
||||
minreq = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
toml = "0.9.0"
|
||||
toml = "0.9.10"
|
||||
vecdb = { workspace = true }
|
||||
zip = { version = "7.0.0", default-features = false, features = ["deflate"] }
|
||||
|
||||
[[bin]]
|
||||
name = "brk"
|
||||
path = "src/main.rs"
|
||||
|
||||
[package.metadata.dist]
|
||||
dist = false
|
||||
dist = true
|
||||
|
||||
@@ -1,96 +1,60 @@
|
||||
# BRK CLI
|
||||
# brk_cli
|
||||
|
||||
<p align="left">
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
<a href="https://crates.io/crates/brk_cli">
|
||||
<img src="https://img.shields.io/crates/v/brk_cli" alt="Version" />
|
||||
</a>
|
||||
<a href="https://docs.rs/brk_cli">
|
||||
<img src="https://img.shields.io/docsrs/brk_cli" alt="Documentation" />
|
||||
</a>
|
||||
<img src="https://img.shields.io/crates/size/brk_cli" alt="Size" />
|
||||
<a href="https://deps.rs/crate/brk_cli">
|
||||
<img src="https://deps.rs/crate/brk_cli/latest/status.svg" alt="Dependency status">
|
||||
</a>
|
||||
<a href="https://discord.gg/HaR3wpH3nr">
|
||||
<img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
|
||||
</a>
|
||||
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
|
||||
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
|
||||
</a>
|
||||
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
|
||||
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
|
||||
</a>
|
||||
<a href="https://x.com/brkdotorg">
|
||||
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
|
||||
</a>
|
||||
</p>
|
||||
Command-line interface for running the Bitcoin Research Kit.
|
||||
|
||||
A command line interface to run a Bitcoin Research Kit instance.
|
||||
## What It Enables
|
||||
|
||||
It's very customizable with all parameters from the underlying tools (crates) used inside.
|
||||
Run a full BRK instance: index the blockchain, compute metrics, serve the API, and optionally host a web interface. Continuously syncs with new blocks.
|
||||
|
||||
Run `brk -h` for more information.
|
||||
## Key Features
|
||||
|
||||
## Requirements
|
||||
- **All-in-one**: Single binary runs indexer, computer, mempool monitor, and server
|
||||
- **Auto-sync**: Waits for new blocks and processes them automatically
|
||||
- **Web interface**: Downloads and bundles frontend from GitHub releases
|
||||
- **Configurable**: TOML config for RPC, paths, and features
|
||||
- **Collision checking**: Optional TXID collision validation mode
|
||||
- **Memory optimized**: Uses mimalloc allocator, 512MB stack for deep recursion
|
||||
|
||||
### Hardware
|
||||
|
||||
#### Recommended
|
||||
|
||||
- [Latest base model Mac mini](https://www.apple.com/mac-mini/)
|
||||
- [Thunderbolt 4 SSD enclosure](https://satechi.net/products/usb4-nvme-ssd-pro-enclosure/Z2lkOi8vc2hvcGlmeS9Qcm9kdWN0VmFyaWFudC80MDE4ODQ3MDA2NzI4OA==?queryID=7961465089021ee203a60db7e62e90d2)
|
||||
- [2 TB NVMe SSD](https://shop.sandisk.com/products/ssd/internal-ssd/wd-black-sn850x-nvme-ssd?sku=WDS200T2X0E-00BCA0)
|
||||
|
||||
#### Minimum
|
||||
|
||||
To be determined
|
||||
|
||||
### Software
|
||||
|
||||
- [Bitcoin](https://bitcoin.org/en/full-node)
|
||||
- [Rust](https://www.rust-lang.org/tools/install)
|
||||
- Unix based operating system (Mac OS or Linux)
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Ubuntu users need to install `open-ssl` via `sudo apt install libssl-dev pkg-config`
|
||||
|
||||
## Download
|
||||
|
||||
### Binaries
|
||||
|
||||
You can find a pre-built binary for your operating system in the [releases page](https://github.com/bitcoinresearchkit/brk/releases/latest).
|
||||
|
||||
### Cargo
|
||||
## Install
|
||||
|
||||
```bash
|
||||
# Install
|
||||
cargo install brk # or `cargo install brk_cli`, the result is the same
|
||||
|
||||
# Update
|
||||
cargo install brk # or `cargo install-update -a` if you have `cargo-update` installed
|
||||
```
|
||||
|
||||
### Source
|
||||
|
||||
```bash
|
||||
git clone https://github.com/bitcoinresearchkit/brk.git
|
||||
cd brk/crates/brk
|
||||
cargo run -r
|
||||
cargo install --locked brk_cli
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
Run `brk -h` to view each available parameter and their respective description.
|
||||
```bash
|
||||
# See all options
|
||||
brk --help
|
||||
|
||||
> [!TIP]
|
||||
> Every parameter set will be saved at `~/.brk/config.toml`, which allows you to simply run `brk` next time.
|
||||
# The CLI will:
|
||||
# 1. Index new blocks
|
||||
# 2. Compute derived metrics
|
||||
# 3. Start mempool monitor
|
||||
# 4. Launch API server (port 3110)
|
||||
# 5. Wait for new blocks and repeat
|
||||
```
|
||||
|
||||
## Tunnel
|
||||
## Components
|
||||
|
||||
The easiest way to let others access your server is to use `cloudflared` which will also cache requests. For more information see [Cloudflare Tunnel](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/) documentation.
|
||||
1. **Indexer**: Processes blocks into queryable indexes
|
||||
2. **Computer**: Derives 1000+ on-chain metrics
|
||||
3. **Mempool**: Real-time fee estimation
|
||||
4. **Server**: REST API + MCP endpoint
|
||||
5. **Bundler**: JS bundling for web interface (if enabled)
|
||||
|
||||
## Performance
|
||||
|
||||
| Machine | Time | Disk | Peak Disk | Memory | Peak Memory |
|
||||
|---------|------|------|-----------|--------|-------------|
|
||||
| MBP M3 Pro (36GB, internal SSD) | 5.2h | 341 GB | 415 GB | 6.4 GB | 12 GB |
|
||||
|
||||
Full benchmark data: [`https://github.com/bitcoinresearchkit/benches/tree/main/brk`](/benches/brk)
|
||||
|
||||
## Built On
|
||||
|
||||
- `brk_indexer` for blockchain indexing
|
||||
- `brk_computer` for metric computation
|
||||
- `brk_mempool` for mempool monitoring
|
||||
- `brk_server` for HTTP API
|
||||
- `brk_bundler` for web interface bundling
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
fn main() {
|
||||
let profile = std::env::var("PROFILE").unwrap_or_default();
|
||||
|
||||
if profile == "release" {
|
||||
println!("cargo:rustc-flag=-C");
|
||||
println!("cargo:rustc-flag=target-cpu=native");
|
||||
}
|
||||
}
|
||||
@@ -3,17 +3,15 @@ use std::{
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use bitcoincore_rpc::{self, Auth, Client};
|
||||
use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
|
||||
use brk_error::{Error, Result};
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_server::Website;
|
||||
use brk_vec::{Computation, Format};
|
||||
use brk_rpc::{Auth, Client};
|
||||
use clap::Parser;
|
||||
use clap_derive::Parser;
|
||||
use color_eyre::eyre::eyre;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
|
||||
use crate::services::Services;
|
||||
use crate::{default_brk_path, dot_brk_path, website::Website};
|
||||
|
||||
const DOWNLOADS: &str = "downloads";
|
||||
|
||||
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
|
||||
#[command(version, about)]
|
||||
@@ -33,27 +31,17 @@ pub struct Config {
|
||||
#[arg(long, value_name = "PATH")]
|
||||
brkdir: Option<String>,
|
||||
|
||||
/// Activated services, default: all, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
services: Option<Services>,
|
||||
|
||||
/// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
computation: Option<Computation>,
|
||||
|
||||
/// Format of computed datasets, `compressed` to save disk space (experimental), `raw` to prioritize speed, default: `raw`, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
format: Option<Format>,
|
||||
|
||||
/// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
|
||||
/// Activate fetching prices from BRK's API and the computation of all price related datasets, default: true, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short = 'F', long, value_name = "BOOL")]
|
||||
fetch: Option<bool>,
|
||||
|
||||
/// Website served by the server (if active), default: default, saved
|
||||
/// Activate fetching prices from exchanges APIs if `fetch` is also set to `true`, default: true, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
exchanges: Option<bool>,
|
||||
|
||||
/// Website served by the server, default: default, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
website: Option<Website>,
|
||||
@@ -83,29 +71,14 @@ pub struct Config {
|
||||
#[arg(long, value_name = "PASSWORD")]
|
||||
rpcpassword: Option<String>,
|
||||
|
||||
/// Delay between runs, default: 0, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "SECONDS")]
|
||||
delay: Option<u64>,
|
||||
|
||||
/// Activate the Model Context Protocol (MCP) endpoint to give LLMs access to BRK (experimental), default: true, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
mcp: Option<bool>,
|
||||
|
||||
/// DEV: Activate watching the selected website's folder for changes, default: false, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
watch: Option<bool>,
|
||||
|
||||
/// DEV: Activate checking address hashes for collisions when indexing, default: false, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(long, value_name = "BOOL")]
|
||||
#[arg(skip)]
|
||||
check_collisions: Option<bool>,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn import() -> color_eyre::Result<Self> {
|
||||
pub fn import() -> Result<Self> {
|
||||
let config_args = Some(Config::parse());
|
||||
|
||||
let path = dot_brk_path();
|
||||
@@ -129,20 +102,12 @@ impl Config {
|
||||
config_saved.brkdir = Some(brkdir);
|
||||
}
|
||||
|
||||
if let Some(services) = config_args.services.take() {
|
||||
config_saved.services = Some(services);
|
||||
}
|
||||
|
||||
if let Some(computation) = config_args.computation.take() {
|
||||
config_saved.computation = Some(computation);
|
||||
}
|
||||
|
||||
if let Some(fetch) = config_args.fetch.take() {
|
||||
config_saved.fetch = Some(fetch);
|
||||
}
|
||||
|
||||
if let Some(format) = config_args.format.take() {
|
||||
config_saved.format = Some(format);
|
||||
if let Some(exchanges) = config_args.exchanges.take() {
|
||||
config_saved.exchanges = Some(exchanges);
|
||||
}
|
||||
|
||||
if let Some(website) = config_args.website.take() {
|
||||
@@ -169,22 +134,10 @@ impl Config {
|
||||
config_saved.rpcpassword = Some(rpcpassword);
|
||||
}
|
||||
|
||||
if let Some(delay) = config_args.delay.take() {
|
||||
config_saved.delay = Some(delay);
|
||||
}
|
||||
|
||||
if let Some(check_collisions) = config_args.check_collisions.take() {
|
||||
config_saved.check_collisions = Some(check_collisions);
|
||||
}
|
||||
|
||||
if let Some(mcp) = config_args.mcp.take() {
|
||||
config_saved.mcp = Some(mcp);
|
||||
}
|
||||
|
||||
if let Some(watch) = config_args.watch.take() {
|
||||
config_saved.watch = Some(watch);
|
||||
}
|
||||
|
||||
if config_args != Config::default() {
|
||||
dbg!(config_args);
|
||||
panic!("Didn't consume the full config")
|
||||
@@ -224,7 +177,9 @@ impl Config {
|
||||
|
||||
if self.rpc_auth().is_err() {
|
||||
println!(
|
||||
"No way found to authenticate the RPC client, please either set --rpccookiefile or --rpcuser and --rpcpassword.\nRun the program with '-h' for help."
|
||||
"Unsuccessful authentication with the RPC client.
|
||||
First make sure that `bitcoind` is running. If it is then please either set --rpccookiefile or --rpcuser and --rpcpassword as the default values seemed to have failed.
|
||||
Finally, you can run the program with '-h' for help."
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
@@ -241,18 +196,18 @@ impl Config {
|
||||
fs::write(path, toml::to_string(self).unwrap())
|
||||
}
|
||||
|
||||
pub fn rpc(&self) -> color_eyre::Result<&'static Client> {
|
||||
Ok(Box::leak(Box::new(Client::new(
|
||||
pub fn rpc(&self) -> Result<Client> {
|
||||
Client::new(
|
||||
&format!(
|
||||
"http://{}:{}",
|
||||
self.rpcconnect().unwrap_or(&"localhost".to_string()),
|
||||
self.rpcport().unwrap_or(8332)
|
||||
),
|
||||
self.rpc_auth().unwrap(),
|
||||
)?)))
|
||||
self.rpc_auth()?,
|
||||
)
|
||||
}
|
||||
|
||||
fn rpc_auth(&self) -> color_eyre::Result<Auth> {
|
||||
fn rpc_auth(&self) -> Result<Auth> {
|
||||
let cookie = self.path_cookiefile();
|
||||
|
||||
if cookie.is_file() {
|
||||
@@ -263,7 +218,7 @@ impl Config {
|
||||
self.rpcpassword.clone().unwrap(),
|
||||
))
|
||||
} else {
|
||||
Err(eyre!("Failed to find correct auth"))
|
||||
Err(Error::AuthFailed)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,14 +230,12 @@ impl Config {
|
||||
self.rpcport
|
||||
}
|
||||
|
||||
pub fn delay(&self) -> Option<u64> {
|
||||
self.delay
|
||||
}
|
||||
|
||||
pub fn bitcoindir(&self) -> PathBuf {
|
||||
self.bitcoindir
|
||||
.as_ref()
|
||||
.map_or_else(default_bitcoin_path, |s| Self::fix_user_path(s.as_ref()))
|
||||
.map_or_else(Client::default_bitcoin_path, |s| {
|
||||
Self::fix_user_path(s.as_ref())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn blocksdir(&self) -> PathBuf {
|
||||
@@ -298,22 +251,12 @@ impl Config {
|
||||
.map_or_else(default_brk_path, |s| Self::fix_user_path(s.as_ref()))
|
||||
}
|
||||
|
||||
pub fn outputsdir(&self) -> PathBuf {
|
||||
self.brkdir().join("outputs")
|
||||
}
|
||||
|
||||
pub fn harsdir(&self) -> PathBuf {
|
||||
self.outputsdir().join("hars")
|
||||
self.brkdir().join("hars")
|
||||
}
|
||||
|
||||
pub fn process(&self) -> bool {
|
||||
self.services
|
||||
.is_none_or(|m| m == Services::All || m == Services::Processor)
|
||||
}
|
||||
|
||||
pub fn serve(&self) -> bool {
|
||||
self.services
|
||||
.is_none_or(|m| m == Services::All || m == Services::Server)
|
||||
pub fn downloads_dir(&self) -> PathBuf {
|
||||
dot_brk_path().join(DOWNLOADS)
|
||||
}
|
||||
|
||||
fn path_cookiefile(&self) -> PathBuf {
|
||||
@@ -342,35 +285,34 @@ impl Config {
|
||||
}
|
||||
|
||||
pub fn website(&self) -> Website {
|
||||
self.website.unwrap_or(Website::Default)
|
||||
self.website.unwrap_or(Website::Bitview)
|
||||
}
|
||||
|
||||
pub fn fetch(&self) -> bool {
|
||||
self.fetch.is_none_or(|b| b)
|
||||
}
|
||||
|
||||
pub fn exchanges(&self) -> bool {
|
||||
self.exchanges.is_none_or(|b| b)
|
||||
}
|
||||
|
||||
pub fn fetcher(&self) -> Option<Fetcher> {
|
||||
self.fetch()
|
||||
.then(|| Fetcher::import(Some(self.harsdir().as_path())).unwrap())
|
||||
}
|
||||
|
||||
pub fn computation(&self) -> Computation {
|
||||
self.computation.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn format(&self) -> Format {
|
||||
self.format.unwrap_or_default()
|
||||
.then(|| Fetcher::import(self.exchanges(), Some(self.harsdir().as_path())).unwrap())
|
||||
}
|
||||
|
||||
pub fn check_collisions(&self) -> bool {
|
||||
self.check_collisions.is_some_and(|b| b)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mcp(&self) -> bool {
|
||||
self.mcp.is_none_or(|b| b)
|
||||
}
|
||||
|
||||
pub fn watch(&self) -> bool {
|
||||
self.watch.is_some_and(|b| b)
|
||||
fn default_on_error<'de, D, T>(deserializer: D) -> Result<T, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
T: Deserialize<'de> + Default,
|
||||
{
|
||||
match T::deserialize(deserializer) {
|
||||
Ok(v) => Ok(v),
|
||||
Err(_) => Ok(T::default()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
use std::{fs, thread};
|
||||
|
||||
use brk_core::{dot_brk_log_path, dot_brk_path};
|
||||
|
||||
mod config;
|
||||
mod run;
|
||||
mod services;
|
||||
|
||||
use run::*;
|
||||
|
||||
pub fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
fs::create_dir_all(dot_brk_path())?;
|
||||
|
||||
brk_logger::init(Some(&dot_brk_log_path()));
|
||||
|
||||
thread::Builder::new()
|
||||
.stack_size(256 * 1024 * 1024)
|
||||
.spawn(run)?
|
||||
.join()
|
||||
.unwrap()
|
||||
}
|
||||
@@ -1 +1,178 @@
|
||||
use brk_cli::main;
|
||||
#![doc = include_str!("../README.md")]
|
||||
|
||||
use std::{
|
||||
fs,
|
||||
io::Cursor,
|
||||
path::PathBuf,
|
||||
thread::{self, sleep},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use brk_alloc::Mimalloc;
|
||||
use brk_bundler::bundle;
|
||||
use brk_computer::Computer;
|
||||
use brk_error::Result;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_iterator::Blocks;
|
||||
use brk_mempool::Mempool;
|
||||
use brk_query::AsyncQuery;
|
||||
use brk_reader::Reader;
|
||||
use brk_server::{Server, VERSION};
|
||||
use log::info;
|
||||
use vecdb::Exit;
|
||||
|
||||
mod config;
|
||||
mod paths;
|
||||
mod website;
|
||||
|
||||
use crate::{config::Config, paths::*};
|
||||
|
||||
pub fn main() -> color_eyre::Result<()> {
|
||||
// Can't increase main thread's stack size, thus we need to use another thread
|
||||
thread::Builder::new()
|
||||
.stack_size(512 * 1024 * 1024)
|
||||
.spawn(run)?
|
||||
.join()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub fn run() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
fs::create_dir_all(dot_brk_path())?;
|
||||
|
||||
brk_logger::init(Some(&dot_brk_log_path()))?;
|
||||
|
||||
let config = Config::import()?;
|
||||
|
||||
let client = config.rpc()?;
|
||||
|
||||
let exit = Exit::new();
|
||||
exit.set_ctrlc_handler();
|
||||
|
||||
let reader = Reader::new(config.blocksdir(), &client);
|
||||
|
||||
let blocks = Blocks::new(&client, &reader);
|
||||
|
||||
let mut indexer = Indexer::forced_import(&config.brkdir())?;
|
||||
|
||||
// Pre-run indexer if too far behind, then drop and reimport to reduce memory
|
||||
let chain_height = client.get_last_height()?;
|
||||
let indexed_height = indexer.vecs.starting_height();
|
||||
if chain_height.saturating_sub(*indexed_height) > 1000 {
|
||||
indexer.index(&blocks, &client, &exit)?;
|
||||
drop(indexer);
|
||||
Mimalloc::collect();
|
||||
indexer = Indexer::forced_import(&config.brkdir())?;
|
||||
}
|
||||
|
||||
let mut computer = Computer::forced_import(&config.brkdir(), &indexer, config.fetcher())?;
|
||||
|
||||
let mempool = Mempool::new(&client);
|
||||
|
||||
let mempool_clone = mempool.clone();
|
||||
thread::spawn(move || {
|
||||
mempool_clone.start();
|
||||
});
|
||||
|
||||
let query = AsyncQuery::build(&reader, &indexer, &computer, Some(mempool));
|
||||
|
||||
let website = config.website();
|
||||
|
||||
let downloads_path = config.downloads_dir();
|
||||
|
||||
let future = async move {
|
||||
let bundle_path = if website.is_some() {
|
||||
// Try to find local dev directories - check cwd and parent directories
|
||||
let find_dev_dirs = || -> Option<(PathBuf, PathBuf)> {
|
||||
let mut dir = std::env::current_dir().ok()?;
|
||||
loop {
|
||||
let websites = dir.join("websites");
|
||||
let modules = dir.join("modules");
|
||||
if websites.exists() && modules.exists() {
|
||||
return Some((websites, modules));
|
||||
}
|
||||
// Stop at workspace root (crates/ indicates we're there)
|
||||
if dir.join("crates").exists() {
|
||||
return None;
|
||||
}
|
||||
dir = dir.parent()?.to_path_buf();
|
||||
}
|
||||
};
|
||||
|
||||
let websites_path;
|
||||
let modules_path;
|
||||
|
||||
if let Some((websites, modules)) = find_dev_dirs() {
|
||||
websites_path = websites;
|
||||
modules_path = modules;
|
||||
} else {
|
||||
let downloaded_brk_path = downloads_path.join(format!("brk-{VERSION}"));
|
||||
|
||||
let downloaded_websites_path = downloaded_brk_path.join("websites");
|
||||
let downloaded_modules_path = downloaded_brk_path.join("modules");
|
||||
|
||||
if !fs::exists(&downloaded_websites_path)? {
|
||||
info!("Downloading source from Github...");
|
||||
|
||||
let url = format!(
|
||||
"https://github.com/bitcoinresearchkit/brk/archive/refs/tags/v{VERSION}.zip",
|
||||
);
|
||||
|
||||
let response = minreq::get(url).with_timeout(60).send()?;
|
||||
let bytes = response.as_bytes();
|
||||
let cursor = Cursor::new(bytes);
|
||||
|
||||
let mut zip = zip::ZipArchive::new(cursor).unwrap();
|
||||
|
||||
zip.extract(downloads_path).unwrap();
|
||||
}
|
||||
|
||||
websites_path = downloaded_websites_path;
|
||||
modules_path = downloaded_modules_path;
|
||||
}
|
||||
|
||||
Some(websites_path.join(website.to_folder_name()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let server = Server::new(&query, bundle_path);
|
||||
|
||||
tokio::spawn(async move {
|
||||
server.serve(true).await.unwrap();
|
||||
});
|
||||
|
||||
Ok(()) as Result<()>
|
||||
};
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
|
||||
let _handle = runtime.spawn(future);
|
||||
|
||||
loop {
|
||||
client.wait_for_synced_node()?;
|
||||
|
||||
let last_height = client.get_last_height()?;
|
||||
|
||||
info!("{} blocks found.", u32::from(last_height) + 1);
|
||||
|
||||
let starting_indexes = if config.check_collisions() {
|
||||
indexer.checked_index(&blocks, &client, &exit)?
|
||||
} else {
|
||||
indexer.index(&blocks, &client, &exit)?
|
||||
};
|
||||
|
||||
Mimalloc::collect();
|
||||
|
||||
computer.compute(&indexer, starting_indexes, &reader, &exit)?;
|
||||
|
||||
info!("Waiting for new blocks...");
|
||||
|
||||
while last_height == client.get_last_height()? {
|
||||
sleep(Duration::from_secs(1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub fn dot_brk_path() -> PathBuf {
|
||||
let home = std::env::var("HOME").unwrap();
|
||||
Path::new(&home).join(".brk")
|
||||
}
|
||||
|
||||
pub fn dot_brk_log_path() -> PathBuf {
|
||||
dot_brk_path().join("log")
|
||||
}
|
||||
|
||||
pub fn default_brk_path() -> PathBuf {
|
||||
dot_brk_path()
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
use std::{thread::sleep, time::Duration};
|
||||
|
||||
use bitcoincore_rpc::{self, RpcApi};
|
||||
use brk_computer::Computer;
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_server::Server;
|
||||
use log::info;
|
||||
|
||||
use crate::config::Config;
|
||||
|
||||
pub fn run() -> color_eyre::Result<()> {
|
||||
let config = Config::import()?;
|
||||
|
||||
let rpc = config.rpc()?;
|
||||
|
||||
let exit = Exit::new();
|
||||
|
||||
let parser = brk_parser::Parser::new(config.blocksdir(), rpc);
|
||||
|
||||
let format = config.format();
|
||||
|
||||
let mut indexer = Indexer::forced_import(&config.outputsdir())?;
|
||||
|
||||
let wait_for_synced_node = || -> color_eyre::Result<()> {
|
||||
let is_synced = || -> color_eyre::Result<bool> {
|
||||
let info = rpc.get_blockchain_info()?;
|
||||
Ok(info.headers == info.blocks)
|
||||
};
|
||||
|
||||
if !is_synced()? {
|
||||
info!("Waiting for node to be synced...");
|
||||
while !is_synced()? {
|
||||
sleep(Duration::from_secs(1))
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
};
|
||||
|
||||
let mut computer = Computer::forced_import(
|
||||
&config.outputsdir(),
|
||||
&indexer,
|
||||
config.computation(),
|
||||
config.fetcher(),
|
||||
format,
|
||||
)?;
|
||||
|
||||
tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?
|
||||
.block_on(async {
|
||||
let server = if config.serve() {
|
||||
let served_indexer = indexer.clone();
|
||||
let served_computer = computer.clone();
|
||||
|
||||
let server = Server::new(served_indexer, served_computer, config.website())?;
|
||||
|
||||
let watch = config.watch();
|
||||
let mcp = config.mcp();
|
||||
let opt = Some(tokio::spawn(async move {
|
||||
server.serve(watch, mcp).await.unwrap();
|
||||
}));
|
||||
|
||||
sleep(Duration::from_secs(1));
|
||||
|
||||
opt
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if config.process() {
|
||||
loop {
|
||||
wait_for_synced_node()?;
|
||||
|
||||
let block_count = rpc.get_block_count()?;
|
||||
|
||||
info!("{} blocks found.", block_count + 1);
|
||||
|
||||
let starting_indexes =
|
||||
indexer.index(&parser, rpc, &exit, config.check_collisions())?;
|
||||
|
||||
computer.compute(&mut indexer, starting_indexes, &exit)?;
|
||||
|
||||
if let Some(delay) = config.delay() {
|
||||
sleep(Duration::from_secs(delay))
|
||||
}
|
||||
|
||||
info!("Waiting for new blocks...");
|
||||
|
||||
while block_count == rpc.get_block_count()? {
|
||||
sleep(Duration::from_secs(1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(handle) = server {
|
||||
handle.await.unwrap();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
use clap_derive::{Parser, ValueEnum};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(
|
||||
Default,
|
||||
Debug,
|
||||
Clone,
|
||||
Copy,
|
||||
Parser,
|
||||
ValueEnum,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
PartialEq,
|
||||
Eq,
|
||||
PartialOrd,
|
||||
Ord,
|
||||
)]
|
||||
pub enum Services {
|
||||
#[default]
|
||||
All,
|
||||
Processor,
|
||||
Server,
|
||||
}
|
||||