From b5e3262b674a884e819e1d46f3a36b1030ca5686 Mon Sep 17 00:00:00 2001 From: nym21 Date: Sat, 16 Aug 2025 18:21:44 +0200 Subject: [PATCH] readmes: update --- Cargo.lock | 6 +- crates/brk/README.md | 225 +++++++++++++++++++++++------ crates/brk_bundler/README.md | 201 +++++++++++++++++++++----- crates/brk_cli/README.md | 196 +++++++++++++++++++------ crates/brk_computer/README.md | 204 +++++++++++++++++++++----- crates/brk_error/README.md | 155 +++++++++++++++----- crates/brk_fetcher/README.md | 199 +++++++++++++++++++++----- crates/brk_indexer/README.md | 211 +++++++++++++++++++-------- crates/brk_interface/README.md | 254 ++++++++++++++++++++++++++------- crates/brk_logger/README.md | 199 +++++++++++++++++++++++--- crates/brk_mcp/README.md | 227 ++++++++++++++++++++++++++--- crates/brk_parser/README.md | 176 ++++++++++++++++------- crates/brk_server/README.md | 241 ++++++++++++++++++++++--------- crates/brk_store/README.md | 170 +++++++++++++++------- crates/brk_structs/README.md | 160 +++++++++++---------- 15 files changed, 2184 insertions(+), 640 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d6d2c6612..db8888d19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3863,7 +3863,7 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "seqdb" -version = "0.1.2" +version = "0.2.0" dependencies = [ "libc", "log", @@ -4688,7 +4688,7 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vecdb" -version = "0.1.2" +version = "0.2.0" dependencies = [ "ctrlc", "log", @@ -4706,7 +4706,7 @@ dependencies = [ [[package]] name = "vecdb_derive" -version = "0.1.2" +version = "0.2.0" dependencies = [ "quote", "syn 2.0.106", diff --git a/crates/brk/README.md b/crates/brk/README.md index 153f7f1f4..cb9313364 100644 --- a/crates/brk/README.md +++ b/crates/brk/README.md @@ -1,66 +1,197 @@ # brk -Bitcoin Research Kit (BRK) is a high-performance toolchain for parsing, indexing, computing, and serving Bitcoin blockchain data. It provides an alternative to services like Glassnode and mempool.space with a focus on self-hosting and open-source transparency. +**Main wrapper crate for the Bitcoin Research Kit (BRK)** -This is the main wrapper crate that re-exports all workspace crates through feature flags. +The `brk` crate serves as the primary entry point for the Bitcoin Research Kit, providing a unified interface to all BRK components through feature flags. It enables developers to selectively include only the components they need while maintaining a consistent API. -## Crates +## What it provides -- [`brk`](https://crates.io/crates/brk): A wrapper around all other `brk-*` crates -- [`brk_bundler`](https://crates.io/crates/brk_bundler): A thin wrapper around [`rolldown`](https://rolldown.rs/) -- [`brk_cli`](https://crates.io/crates/brk_cli): A command line interface to run a BRK instance -- [`brk_computer`](https://crates.io/crates/brk_computer): A Bitcoin dataset computer built on top of [`brk_indexer`](https://crates.io/crates/brk_indexer) -- [`brk_error`](https://crates.io/crates/brk_error): Errors used throughout BRK -- [`brk_fetcher`](https://crates.io/crates/brk_fetcher): A Bitcoin price fetcher -- [`brk_indexer`](https://crates.io/crates/brk_indexer): A Bitcoin indexer built on top of [`brk_parser`](https://crates.io/crates/brk_parser) -- [`brk_interface`](https://crates.io/crates/brk_interface): An interface to find and format data from BRK -- [`brk_logger`](https://crates.io/crates/brk_logger): A thin wrapper around [`env_logger`](https://crates.io/crates/env_logger) -- [`brk_mcp`](https://crates.io/crates/brk_mcp): A bridge for LLMs to access BRK -- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin block parser and iterator built on top of [`bitcoin-rust`](https://crates.io/crates/bitcoin) -- [`brk_server`](https://crates.io/crates/brk_server): A server with an API for anything from BRK -- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall) -- [`brk_structs`](https://crates.io/crates/brk_structs): Structs used throughout BRK +- **Unified Access**: Single crate providing access to the entire BRK ecosystem +- **Feature-based Selection**: Choose only the components you need +- **Consistent Versioning**: All components versioned together for compatibility +- **Simplified Dependencies**: Single dependency instead of multiple individual crates -## Features +## Available Components -- `full` - Enable all workspace crates -- `bundler` - Re-export `brk_bundler` -- `cli` - Re-export `brk_cli` (always enabled) -- `computer` - Re-export `brk_computer` -- `error` - Re-export `brk_error` -- `fetcher` - Re-export `brk_fetcher` -- `indexer` - Re-export `brk_indexer` -- `interface` - Re-export `brk_interface` -- `logger` - Re-export `brk_logger` -- `mcp` - Re-export `brk_mcp` -- `parser` - Re-export `brk_parser` -- `server` - Re-export `brk_server` -- `store` - Re-export `brk_store` -- `structs` - Re-export `brk_structs` +### Core Data Pipeline +- **`parser`** ([brk_parser](../brk_parser/)) - High-performance Bitcoin block parser +- **`indexer`** ([brk_indexer](../brk_indexer/)) - Blockchain data indexer with dual storage +- **`computer`** ([brk_computer](../brk_computer/)) - Analytics engine for computed datasets +- **`interface`** ([brk_interface](../brk_interface/)) - Unified data query and formatting API + +### Infrastructure +- **`structs`** ([brk_structs](../brk_structs/)) - Bitcoin-aware type system and data structures +- **`error`** ([brk_error](../brk_error/)) - Centralized error handling +- **`store`** ([brk_store](../brk_store/)) - Blockchain-aware key-value storage + +### External Integration +- **`fetcher`** ([brk_fetcher](../brk_fetcher/)) - Bitcoin price data fetcher with multi-source fallback +- **`server`** ([brk_server](../brk_server/)) - HTTP server with REST API +- **`mcp`** ([brk_mcp](../brk_mcp/)) - Model Context Protocol for LLM integration + +### Utilities +- **`cli`** ([brk_cli](../brk_cli/)) - Command line interface (always enabled) +- **`logger`** ([brk_logger](../brk_logger/)) - Logging utilities +- **`bundler`** ([brk_bundler](../brk_bundler/)) - Asset bundling for web interfaces ## Usage -Add to your `Cargo.toml`: +### Full Installation ```toml [dependencies] -brk = { version = "0.1", features = ["full"] } +brk = { version = "0.0.88", features = ["full"] } ``` -Or enable specific components: - -```toml -[dependencies] -brk = { version = "0.1", features = ["parser", "indexer", "computer"] } -``` - -## Example - ```rust -use brk::{cli, parser, indexer, computer}; +use brk::*; -// Use individual crates as needed +// Access all components let config = cli::Config::load()?; -let blocks = parser::BlockIterator::new(&config.bitcoin_path)?; -// ... +let parser = parser::Parser::new(/* ... */); +let indexer = indexer::Indexer::forced_import("./data")?; +let computer = computer::Computer::forced_import("./data", &indexer, None)?; ``` + +### Selective Components + +```toml +[dependencies] +brk = { version = "0.0.88", features = ["parser", "indexer", "computer"] } +``` + +```rust +use brk::{parser, indexer, computer}; + +// Core data pipeline only +let parser = parser::Parser::new(blocks_dir, output_dir, rpc); +let mut indexer = indexer::Indexer::forced_import(output_dir)?; +let mut computer = computer::Computer::forced_import(output_dir, &indexer, None)?; +``` + +### Minimal Setup + +```toml +[dependencies] +brk = { version = "0.0.88", features = ["structs", "parser"] } +``` + +```rust +use brk::{structs, parser}; + +// Just parsing and types +let height = structs::Height::new(800_000); +let parser = parser::Parser::new(blocks_dir, output_dir, rpc); +``` + +## Feature Flags + +| Feature | Description | Dependencies | +|---------|-------------|--------------| +| `full` | Enable all components | All crates | +| `cli` | Command line interface | Always enabled | +| `structs` | Core type system | Foundation for other crates | +| `error` | Error handling | Used by most crates | +| `parser` | Block parsing | `structs`, `error` | +| `store` | Key-value storage | `structs`, `error` | +| `indexer` | Blockchain indexing | `parser`, `store` | +| `computer` | Analytics computation | `indexer`, `fetcher` (optional) | +| `fetcher` | Price data fetching | `structs`, `error` | +| `interface` | Data query API | `indexer`, `computer` | +| `server` | HTTP server | `interface`, `mcp` | +| `mcp` | LLM integration | `interface` | +| `logger` | Logging utilities | Standalone | +| `bundler` | Asset bundling | Standalone | + +## Common Usage Patterns + +### Complete BRK Instance + +```rust +use brk::*; + +// Full data pipeline setup +let config = cli::Config::load()?; +let rpc = /* Bitcoin Core RPC client */; +let parser = parser::Parser::new(config.blocks_dir, config.output_dir, rpc); +let mut indexer = indexer::Indexer::forced_import(&config.output_dir)?; +let mut computer = computer::Computer::forced_import(&config.output_dir, &indexer, None)?; +let interface = interface::Interface::build(&indexer, &computer); +let server = server::Server::new(interface, config.website_path); + +// Start server +server.serve(true).await?; +``` + +### Data Analysis + +```rust +use brk::{indexer, computer, interface}; + +// Analysis-focused setup +let indexer = indexer::Indexer::forced_import("./brk_data")?; +let computer = computer::Computer::forced_import("./brk_data", &indexer, None)?; +let interface = interface::Interface::build(&indexer, &computer); + +// Query data +let params = interface::Params { + index: interface::Index::Height, + ids: vec!["price_usd", "difficulty"].into(), + rest: interface::ParamsOpt::default() + .set_from(-100) + .set_format(interface::Format::CSV), +}; + +let csv_data = interface.search_and_format(params)?; +``` + +### Custom Integration + +```rust +use brk::{structs, parser, error}; + +// Custom application with BRK components +fn analyze_blocks() -> error::Result<()> { + let parser = parser::Parser::new(blocks_dir, output_dir, rpc); + + parser.parse(None, None) + .iter() + .take(1000) // First 1000 blocks + .for_each(|(height, block, hash)| { + println!("Block {}: {} transactions", height, block.txdata.len()); + }); + + Ok(()) +} +``` + +## Version Compatibility + +All BRK crates are released together with synchronized versions. When using the `brk` wrapper crate, you're guaranteed compatibility between all components. + +- **Current version**: 0.0.88 +- **Rust MSRV**: 1.89+ +- **Bitcoin Core**: v25.0 - v29.0 + +## Performance Characteristics + +The `brk` crate itself adds no runtime overhead - it simply re-exports the underlying crates. Performance characteristics depend on which components you use: + +- **Full pipeline**: ~13-15 hours initial sync, ~40GB storage overhead +- **Parser only**: ~4 minutes to parse entire blockchain +- **Indexer only**: ~7-8 hours to index blockchain, ~5-6GB RAM usage +- **Server**: Low latency API responses with caching and compression + +## Dependencies + +The `brk` crate's dependencies are determined by enabled features. Core dependencies include: + +- `brk_cli` - Always included for configuration and CLI support +- Individual `brk_*` crates based on enabled features +- Transitive dependencies from enabled components + +For specific dependency information, see individual crate READMEs. + +--- + +*This README was generated by Claude Code* \ No newline at end of file diff --git a/crates/brk_bundler/README.md b/crates/brk_bundler/README.md index 67c81565a..9ce18be2a 100644 --- a/crates/brk_bundler/README.md +++ b/crates/brk_bundler/README.md @@ -1,63 +1,186 @@ # brk_bundler -Web asset bundler built on Rolldown that provides JavaScript bundling, minification, and live reloading for BRK's web interface. This crate wraps Rolldown with BRK-specific functionality including automatic file watching, version injection, and hash-based cache busting for optimal web performance. +**Asset bundling for BRK web interfaces using Rolldown** -## Features +`brk_bundler` provides JavaScript/TypeScript bundling capabilities for BRK's web interfaces. It's a thin wrapper around Rolldown (Rust-based Rollup alternative) with BRK-specific configuration for building optimized web assets with file watching and automatic rebuilding. -- **JavaScript bundling**: Bundles and minifies JavaScript with source maps -- **Live reloading**: Automatic rebuilding and file watching during development -- **Cache busting**: Hash-based filenames for browser cache invalidation -- **Version injection**: Automatic version injection into service workers -- **Asset copying**: Copies static assets from source to distribution directory -- **Development mode**: Optional watch mode for real-time development +## What it provides + +- **JavaScript Bundling**: Modern ES modules bundling with minification +- **File Watching**: Automatic rebuilding on source file changes +- **Asset Processing**: Copies and processes static assets +- **Version Injection**: Automatic version string replacement in service workers +- **Development Mode**: Live rebuilding for rapid development + +## Key Features + +### Bundling Capabilities +- **ES Module Support**: Modern JavaScript bundling with tree-shaking +- **Minification**: Automatic code minification for production builds +- **Source Maps**: Generated source maps for debugging +- **Entry Point Processing**: Configurable entry points with hashed output names + +### File System Operations +- **Directory Copying**: Copies entire source directories to distribution +- **Selective Processing**: Special handling for specific file types +- **Path Resolution**: Automatic path resolution and asset linking + +### Development Features +- **Hot Rebuilding**: Automatic rebuilds on file changes +- **Watch Mode**: Monitors source files and triggers rebuilds +- **Version Replacement**: Injects build version into service workers ## Usage +### Basic Bundling + ```rust use brk_bundler::bundle; use std::path::Path; -async fn build_website() -> std::io::Result<()> { - let websites_path = Path::new("./websites"); - let source_folder = "src"; - let watch = false; // Set to true for development +// Bundle without watching (production) +let websites_path = Path::new("./websites"); +let source_folder = "default"; +let dist_path = bundle(websites_path, source_folder, false).await?; - // Bundle the website - let dist_path = bundle(websites_path, source_folder, watch).await?; +println!("Bundled to: {:?}", dist_path); +``` - println!("Website built to: {}", dist_path.display()); - Ok(()) +### Development Mode with Watching + +```rust +// Bundle with file watching (development) +let dist_path = bundle(websites_path, "default", true).await?; + +// Bundler now watches for changes and rebuilds automatically +// This will run in the background until the process exits +``` + +### Integration with BRK CLI + +```rust +// Typically called from brk_cli when serving websites +async fn setup_website(config: &Config) -> Result { + let websites_path = config.websites_path(); + let source_folder = match config.website_mode { + WebsiteMode::Default => "default", + WebsiteMode::Custom => "custom", + WebsiteMode::None => return Ok(PathBuf::new()), + }; + + // Bundle the website assets + let dist_path = bundle(websites_path, source_folder, config.dev_mode).await?; + + Ok(dist_path) } ``` -## Build Process - -1. **Clean**: Removes existing distribution directory -2. **Copy**: Copies all source files to distribution directory -3. **Bundle**: Processes JavaScript entry point with Rolldown -4. **Process**: Updates HTML to reference hashed JS files -5. **Version**: Injects version strings into service workers -6. **Watch** (optional): Monitors files for changes and rebuilds - ## File Structure -Expected source structure: +The bundler expects this directory structure: + ``` websites/ -├── src/ +├── default/ # Default website source │ ├── index.html # Main HTML file -│ ├── service-worker.js # Service worker (optional) -│ ├── scripts/ -│ │ └── entry.js # JavaScript entry point -│ └── ... # Other static assets -└── dist/ # Generated distribution files +│ ├── service-worker.js # Service worker (version injected) +│ ├── scripts/ # JavaScript/TypeScript source +│ │ ├── entry.js # Main entry point +│ │ ├── main.js # Application logic +│ │ └── ... # Other JS modules +│ └── assets/ # Static assets +└── dist/ # Generated output directory + ├── index.html # Processed HTML with updated script references + ├── service-worker.js # Service worker with version injected + ├── scripts/ # Bundled and minified JavaScript + │ └── main-[hash].js # Hashed output file + └── assets/ # Copied static assets ``` -## Watch Mode +## Bundling Process -When enabled, the bundler: -- Monitors source files for changes -- Automatically rebuilds JavaScript bundles -- Updates HTML with new hashed filenames -- Reprocesses service workers with version updates -- Copies modified static assets +1. **Clean**: Removes existing `dist/` directory +2. **Copy**: Copies all source files to `dist/` +3. **Bundle JavaScript**: + - Processes `scripts/entry.js` as entry point + - Generates minified bundle with source maps + - Creates hashed filename for cache busting +4. **Process HTML**: Updates script references to hashed filenames +5. **Process Service Worker**: Injects current version string +6. **Watch** (if enabled): Monitors for file changes and rebuilds + +## Configuration + +The bundler uses Rolldown with these optimized settings: + +```rust +BundlerOptions { + input: Some(vec![source_entry.into()]), // scripts/entry.js + dir: Some("./dist/scripts".to_string()), // Output directory + cwd: Some(websites_path), // Working directory + minify: Some(RawMinifyOptions::Bool(true)), // Enable minification + sourcemap: Some(SourceMapType::File), // Generate source maps + ..Default::default() +} +``` + +## File Watching + +In watch mode, the bundler monitors: + +- **Source files**: Non-script files are copied on change +- **JavaScript files**: Trigger full rebuild via Rolldown watcher +- **HTML files**: Processed to update script references +- **Service worker**: Version injection on changes + +### Watch Events Handled + +- `Create` - New files added +- `Modify` - Existing files changed +- Ignores `Delete` and other events + +## Version Injection + +Service workers get automatic version injection: + +```javascript +// In source service-worker.js +const VERSION = '__VERSION__'; + +// After bundling +const VERSION = 'v0.0.88'; +``` + +This enables proper cache invalidation across releases. + +## Performance Features + +- **Async Operations**: All bundling operations are async +- **Incremental Builds**: Only rebuilds changed files in watch mode +- **Parallel Processing**: Uses Tokio for concurrent file operations +- **Efficient Copying**: Direct file system operations + +## Error Handling + +- **Graceful Failures**: Logs errors but continues watching +- **Path Resolution**: Automatic path absolutization and validation +- **File System Errors**: Proper error propagation with context + +## Dependencies + +- `brk_rolldown` - Rust-based Rollup bundler +- `notify` - File system watching +- `tokio` - Async runtime for file operations +- `sugar_path` - Path manipulation utilities +- `log` - Error logging + +## Integration Points + +The bundler integrates with: +- **brk_cli**: Called during website setup +- **brk_server**: Serves bundled assets +- **Development workflow**: Provides live rebuilding + +--- + +*This README was generated by Claude Code* \ No newline at end of file diff --git a/crates/brk_cli/README.md b/crates/brk_cli/README.md index fcd70aadd..ac65a0941 100644 --- a/crates/brk_cli/README.md +++ b/crates/brk_cli/README.md @@ -1,73 +1,189 @@ # brk_cli -Command line interface for running BRK (Bitcoin Research Kit) instances. Orchestrates the complete pipeline: parsing Bitcoin Core blocks, indexing data, computing analytics, and serving via HTTP API. +**Command line interface for running complete BRK instances** -## Overview +`brk_cli` provides the main command-line interface for operating the Bitcoin Research Kit. It orchestrates the complete data pipeline from Bitcoin Core block parsing through analytics computation to HTTP API serving, with automatic configuration management and graceful operation. -**Core Operation**: Continuous loop that waits for Bitcoin node sync, indexes new blocks, computes analytics, and serves data via HTTP. +## What it provides -**Key Components**: -- **Parser**: Reads Bitcoin Core block files -- **Indexer**: Processes and stores blockchain data in vecs/stores -- **Computer**: Computes analytics across 9 specialized domains -- **Server**: HTTP API with multiple output formats (JSON, CSV, TSV, Markdown) -- **Website**: Optional web interface (none/default/custom) +- **Complete Pipeline Orchestration**: Coordinates parser, indexer, computer, and server components +- **Automatic Configuration**: Saves settings to `~/.brk/config.toml` for consistent operation +- **Continuous Operation**: Handles blockchain updates and incremental processing +- **Web Interface Options**: Configurable website serving (none, default, custom) +- **Graceful Shutdown**: Ctrl+C handling with proper cleanup -## Requirements +## Key Features -- **Bitcoin Core**: Fully synced node with RPC enabled -- **Storage**: ~32% of blockchain size (~233GB currently) -- **Memory**: ~7-8GB peak during indexing, ~4-5GB steady state -- **OS**: macOS or Linux (Ubuntu: `sudo apt install libssl-dev pkg-config`) +### Pipeline Management +- **Automatic dependency handling**: Ensures Bitcoin Core sync before processing +- **Incremental updates**: Only processes new blocks since last run +- **Error recovery**: Automatic retry logic and graceful error handling +- **Resource management**: Optimized memory usage and disk I/O + +### Configuration System +- **Auto-save configuration**: All CLI options saved to persistent config +- **Flexible paths**: Configurable Bitcoin directory, blocks directory, and output directory +- **RPC authentication**: Cookie file or username/password authentication +- **Data source options**: Configurable price fetching and exchange APIs + +### Operation Modes +- **Initial sync**: Full blockchain processing from genesis +- **Continuous operation**: Real-time processing of new blocks +- **Update mode**: Resume from last processed block +- **Server mode**: HTTP API with optional web interface ## Installation +### Binary Release ```bash -# Binary +# Download from GitHub releases # https://github.com/bitcoinresearchkit/brk/releases/latest +``` -# Via cargo +### Via Cargo +```bash cargo install brk --locked +``` -# From source +### From Source +```bash git clone https://github.com/bitcoinresearchkit/brk.git cd brk && cargo build --release ``` ## Usage -```bash -# First run (set configuration) -brk --brkdir ./my_data --fetch true --website default +### First Run (Configuration Setup) -# Subsequent runs (uses saved config) +```bash +# Basic setup with default options +brk --brkdir ./my_brk_data + +# Full configuration +brk --bitcoindir ~/.bitcoin \ + --brkdir ./brk_data \ + --fetch true \ + --exchanges true \ + --website default +``` + +### Subsequent Runs + +```bash +# Uses saved configuration from ~/.brk/config.toml brk -# View all options +# Override specific options +brk --website none --fetch false +``` + +### Command Line Options + +```bash brk --help ``` -## Configuration +## Configuration Reference -All options auto-save to `~/.brk/config.toml` for subsequent runs: +All options are automatically saved to `~/.brk/config.toml`: -```bash -# Core paths ---bitcoindir # Bitcoin directory (default: ~/.bitcoin) ---blocksdir # Block files (default: bitcoindir/blocks) ---brkdir # BRK output directory (default: ~/.brk) +### Core Paths +- `--bitcoindir ` - Bitcoin Core directory (default: `~/.bitcoin`) +- `--blocksdir ` - Block files directory (default: `bitcoindir/blocks`) +- `--brkdir ` - BRK output directory (default: `~/.brk`) -# Data sources --F, --fetch # Enable price data fetching (default: true) ---exchanges # Use exchange APIs for prices (default: true) +### Data Sources +- `--fetch ` - Enable price data fetching (default: `true`) +- `--exchanges ` - Use exchange APIs for prices (default: `true`) -# Server --w, --website # Web interface: none|default|custom +### Web Interface +- `--website