diff --git a/Cargo.lock b/Cargo.lock index 1e3eac41a..68c48520e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1003,6 +1003,7 @@ dependencies = [ "log", "minreq", "serde", + "serde_json", "tokio", "tower-http", "tracing", diff --git a/DOCKER.md b/DOCKER.md index 404c44ead..e1c43c2ab 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -19,33 +19,88 @@ This guide explains how to run BRK using Docker and Docker Compose. 2. **Run with Docker Compose** ```bash - docker compose up -d + # Multi-container mode (recommended) + docker compose up -d brk-processor brk-server ``` 3. **Access BRK** - Web interface: http://localhost:7070 - API: http://localhost:7070/api + - Health check: http://localhost:7070/health + +## Deployment Modes + +BRK supports flexible deployment modes to suit different use cases: + +### 1. Multi-Container Mode (Recommended) + +Deploy indexer and server as separate containers. This means the following: +- Better resource isolation +- Independent scaling of components +- Server doesn't need Bitcoin Core or RPC access +- Cleaner failure isolation +- Server can start independently (will serve empty data until processor indexes blocks) + +```bash +# Start both processor and server +docker compose up brk-processor brk-server + +# Or run in background +docker compose up -d brk-processor brk-server +``` + +### 2. Processor-Only Mode + +For indexing without web interface: + +```bash +docker compose up brk-processor +``` + +### 3. Server-Only Mode + +For serving pre-indexed data: + +```bash +docker compose up brk-server +``` ## Configuration ### Environment Variables -| Variable | Description | Default | -|----------|-------------|---------| -| `BITCOIN_DATA_DIR` | Path to Bitcoin Core data directory | Required | -| `BTC_RPC_HOST` | Bitcoin Core RPC host | `localhost` | -| `BTC_RPC_PORT` | Bitcoin Core RPC port | `8332` | -| `BRK_SERVICES` | Services to run (`all`, `processor`, `server`) | `all` | -| `BRK_COMPUTATION` | Computation mode (`lazy`, `eager`) | `lazy` | -| `BRK_FORMAT` | Data format (`raw`, `compressed`) | `raw` | -| `BRK_FETCH` | Enable price fetching | `true` | -| `BRK_MCP` | Enable MCP for AI/LLM | `true` | +| Variable | Description | Default | Required For | +|----------|-------------|---------|-------------| +| `BITCOIN_DATA_DIR` | Path to Bitcoin Core data directory | Required | Processor | +| `BTC_RPC_HOST` | Bitcoin Core RPC host | `localhost` | Processor | +| `BTC_RPC_PORT` | Bitcoin Core RPC port | `8332` | Processor | +| `BTC_RPC_USER` | Bitcoin RPC username | - | Processor | +| `BTC_RPC_PASSWORD` | Bitcoin RPC password | - | Processor | +| `BRK_DATA_VOLUME` | Docker volume name for BRK data | `brk-data` | Both | +| `BRK_COMPUTATION` | Computation mode (`lazy`, `eager`) | `lazy` | Processor | +| `BRK_FORMAT` | Data format (`raw`, `compressed`) | `raw` | Processor | +| `BRK_FETCH` | Enable price fetching | `true` | Processor | +| `BRK_MCP` | Enable MCP for AI/LLM | `true` | Server | -### Service Modes +### Example .env File -- **`all`**: Run both processor and server (default) -- **`processor`**: Only process blockchain data -- **`server`**: Only serve API/web interface +```env +# Bitcoin Core paths +BITCOIN_DATA_DIR=/path/to/bitcoin/data +BRK_DATA_VOLUME=brk-data + +# Bitcoin RPC (required for processor) +BTC_RPC_HOST=localhost +BTC_RPC_PORT=8332 +BTC_RPC_USER=your_username +BTC_RPC_PASSWORD=your_password + +# BRK settings +BRK_COMPUTATION=lazy +BRK_FORMAT=raw +BRK_FETCH=true +BRK_MCP=true +``` ### Connecting to Bitcoin Core @@ -53,8 +108,7 @@ This guide explains how to run BRK using Docker and Docker Compose. BRK will automatically use the `.cookie` file from your Bitcoin Core directory. #### Option 2: Username/Password -1. Uncomment the RPC user/password lines in `docker-compose.yml` -2. Set `BTC_RPC_USER` and `BTC_RPC_PASSWORD` in your `.env` file +Set `BTC_RPC_USER` and `BTC_RPC_PASSWORD` in your `.env` file. #### Network Connectivity - **Same host**: @@ -64,12 +118,12 @@ BRK will automatically use the `.cookie` file from your Bitcoin Core directory. ## Building the Image -### Using Docker Compose (Simple) +### Using Docker Compose... ```bash docker compose build ``` -### Using Docker Build Script +### or ... Using Docker Build Script ```bash # Build with default settings ./docker-build.sh @@ -85,68 +139,145 @@ BRK supports two options for storing its data: ### Option 1: Docker Named Volume (Default) Uses a Docker-managed named volume called `brk-data`. This is the recommended approach for most users. -**Advantages:** -- Managed by Docker -- Easy backup/restore -- Platform-independent - -**Usage:** This is the default configuration - no changes needed. - ### Option 2: Bind Mount Maps a specific directory on your host to the container's data directory. +This may be desirable if you want to use a specific storage location for BRK data (e.g. a different disk). -**Advantages:** -- Direct access to files from host -- Easy to locate and manage -- Can be on specific storage devices - -**Usage:** 1. Set `BRK_DATA_DIR` in your `.env` file to your desired host directory 2. In `docker-compose.yml`, comment out the named volume line and uncomment the bind mount line -**Example:** ```bash # In .env file BRK_DATA_DIR=/home/user/brk-data ``` -```yaml -# In docker-compose.yml, uncomment and change as necessary: - # - ${BRK_DATA_VOLUME:-brk-data}:/home/brk/.brk - - ${BRK_DATA_DIR:-./brk-data}:/home/brk/.brk +```bash +# In docker-compose.yml, for BOTH the processor and server services. +# Comment out: + - ${BRK_DATA_VOLUME:-brk-data}:/home/brk/.brk + +# Uncomment: + # - ${BRK_DATA_DIR:-./brk-data}:/home/brk/.brk ``` -### Volume Details -- **BRK data**: Stores computed datasets, indexes, and application state -- **Bitcoin data**: Mounted read-only from host (always a bind mount) +Can also remove or comment out the `volumes` section from the docker-compose.yml file (right at the bottom): +```bash +# Comment out: +volumes: + brk-data: + driver: local +``` + +## Health Checks + +Both containers include health checks: + +- `brk-processor`: checks that the BRK process is running +- `brk-server`: tests network connectivity on port 3110 ## Monitoring -View logs: +### Check Container Status ```bash -docker compose logs -f brk +# View running containers +docker compose ps + +# Check health status +docker compose ps --format \"table {{.Service}}\\t{{.Status}}\\t{{.Health}}\" ``` -Check status: +### View Logs ```bash -docker compose ps +# View logs from both containers +docker compose logs brk-processor brk-server + +# Follow logs in real-time +docker compose logs -f brk-processor brk-server + +# View logs from specific container +docker compose logs -f brk-server ``` ## Troubleshooting -### Cannot connect to Bitcoin Core +### Server Issues + +#### Server returns empty data +- This is normal if processor hasn't indexed any blocks yet +- Server can start before processor and will serve data as it becomes available +- Check that BRK data volume is properly shared between containers + +#### Server won't start +- Check Docker Compose logs: `docker compose logs brk-server` +- Verify health endpoint: `curl http://localhost:7070/health` +- Ensure no port conflicts on 7070 + +### Processor Issues + +#### Cannot connect to Bitcoin Core 1. Ensure Bitcoin Core is running with `-server=1` 2. Check RPC credentials are correct 3. Verify network connectivity from container +4. Test RPC connection: `docker compose exec brk-processor brk --help` -### Permission denied errors -Ensure the Bitcoin data directory is readable by the container user (UID 1000). +#### Processor fails to start +- Verify Bitcoin RPC credentials in `.env` +- Ensure Bitcoin Core is running and accessible +- Check Bitcoin data directory permissions (should be readable by UID 1000) -### Out of memory -Increase Docker's memory limit or use `BRK_COMPUTATION=lazy` to reduce memory usage. +### Performance Issues + +#### Slow indexing +- Ensure adequate disk space for indexed data +- Monitor memory usage during initial indexing +- Use `BRK_COMPUTATION=lazy` to reduce memory usage + +#### Out of memory +- Increase Docker's memory limit +- Use `BRK_COMPUTATION=lazy` mode +- Monitor container resource usage: `docker stats` + +### Permission Issues + +#### Permission denied errors +- Ensure the Bitcoin data directory is readable by the container user (UID 1000) +- Check that volumes are properly mounted +- Verify file ownership: `ls -la $BITCOIN_DATA_DIR` + +### Network Issues + +#### Cannot access web interface +- Verify port mapping: `docker compose ps` +- Check firewall settings +- Ensure no other services are using port 7070 ## Security Considerations - Bitcoin data is mounted read-only for safety - BRK runs as non-root user inside container - Only necessary ports are exposed + +## Backup and Recovery + +### Backing Up BRK Data + +```bash +# Create backup of named volume +docker run --rm -v brk_brk-data:/source -v \"$(pwd)\":/backup alpine tar czf /backup/brk-backup.tar.gz -C /source . + +# Or if using bind mount +tar czf brk-backup.tar.gz -C \"$BRK_DATA_DIR\" . +``` + +### Restoring BRK Data + +```bash +# Stop containers +docker compose down + +# Restore from backup (named volume) +docker run --rm -v brk_brk-data:/target -v \"$(pwd)\":/backup alpine tar xzf /backup/brk-backup.tar.gz -C /target + +# Start containers +docker compose up -d +``` \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f5795c75a..ca1d387f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,13 @@ -# Build stage -FROM rust:nightly AS builder +# ************* +# Builder +# ************* +FROM rustlang/rust:nightly AS builder # Install build dependencies RUN apt-get update && apt-get install -y \ pkg-config \ libssl-dev \ + build-essential \ && rm -rf /var/lib/apt/lists/* WORKDIR /app @@ -15,7 +18,9 @@ COPY . . # Build the application RUN cargo build --release --locked -# Runtime stage +# ************* +# Runtime +# ************* FROM debian:bookworm-slim # Install runtime dependencies diff --git a/crates/brk_cli/src/config.rs b/crates/brk_cli/src/config.rs index a0851c456..f67ca0850 100644 --- a/crates/brk_cli/src/config.rs +++ b/crates/brk_cli/src/config.rs @@ -201,33 +201,37 @@ impl Config { } fn check(&self) { - if !self.bitcoindir().is_dir() { - println!("{:?} isn't a valid directory", self.bitcoindir()); - println!("Please use the --bitcoindir parameter to set a valid path."); - println!("Run the program with '-h' for help."); - std::process::exit(1); - } - - if !self.blocksdir().is_dir() { - println!("{:?} isn't a valid directory", self.blocksdir()); - println!("Please use the --blocksdir parameter to set a valid path."); - println!("Run the program with '-h' for help."); - std::process::exit(1); + // Only check Bitcoin directories and RPC if we're running the processor + if self.process() { + if !self.bitcoindir().is_dir() { + println!("{:?} isn't a valid directory", self.bitcoindir()); + println!("Please use the --bitcoindir parameter to set a valid path."); + println!("Run the program with '-h' for help."); + std::process::exit(1); + } + + if !self.blocksdir().is_dir() { + println!("{:?} isn't a valid directory", self.blocksdir()); + println!("Please use the --blocksdir parameter to set a valid path."); + println!("Run the program with '-h' for help."); + std::process::exit(1); + } + + if self.rpc_auth().is_err() { + println!( + "No way found to authenticate the RPC client, please either set --rpccookiefile or --rpcuser and --rpcpassword.\nRun the program with '-h' for help." + ); + std::process::exit(1); + } } + // Always check BRK directory (needed by both processor and server) if !self.brkdir().is_dir() { println!("{:?} isn't a valid directory", self.brkdir()); println!("Please use the --brkdir parameter to set a valid path."); println!("Run the program with '-h' for help."); std::process::exit(1); } - - if self.rpc_auth().is_err() { - println!( - "No way found to authenticate the RPC client, please either set --rpccookiefile or --rpcuser and --rpcpassword.\nRun the program with '-h' for help." - ); - std::process::exit(1); - } } fn read(path: &Path) -> Self { diff --git a/crates/brk_cli/src/run.rs b/crates/brk_cli/src/run.rs index 9d42afadf..1f7ea0db3 100644 --- a/crates/brk_cli/src/run.rs +++ b/crates/brk_cli/src/run.rs @@ -12,19 +12,31 @@ use crate::config::Config; pub fn run() -> color_eyre::Result<()> { let config = Config::import()?; - let rpc = config.rpc()?; + let rpc = if config.process() { + Some(config.rpc()?) + } else { + None + }; let exit = Exit::new(); - let parser = brk_parser::Parser::new_with_outputs_dir(config.blocksdir(), config.outputsdir(), rpc); + let parser = if config.process() && rpc.is_some() { + Some(brk_parser::Parser::new_with_outputs_dir( + config.blocksdir(), + config.outputsdir(), + rpc.unwrap(), + )) + } else { + None + }; let format = config.format(); let mut indexer = Indexer::forced_import(&config.outputsdir())?; - let wait_for_synced_node = || -> color_eyre::Result<()> { + let wait_for_synced_node = |rpc_client: &bitcoincore_rpc::Client| -> color_eyre::Result<()> { let is_synced = || -> color_eyre::Result { - let info = rpc.get_blockchain_info()?; + let info = rpc_client.get_blockchain_info()?; Ok(info.headers == info.blocks) }; @@ -70,27 +82,33 @@ pub fn run() -> color_eyre::Result<()> { }; if config.process() { - loop { - wait_for_synced_node()?; + if let (Some(rpc_client), Some(parser)) = (rpc, parser) { + loop { + wait_for_synced_node(rpc_client)?; - let block_count = rpc.get_block_count()?; + let block_count = rpc_client.get_block_count()?; - info!("{} blocks found.", block_count + 1); + info!("{} blocks found.", block_count + 1); - let starting_indexes = - indexer.index(&parser, rpc, &exit, config.check_collisions())?; + let starting_indexes = + indexer.index(&parser, rpc_client, &exit, config.check_collisions())?; - computer.compute(&mut indexer, starting_indexes, &exit)?; + computer.compute(&mut indexer, starting_indexes, &exit)?; - if let Some(delay) = config.delay() { - sleep(Duration::from_secs(delay)) - } - - info!("Waiting for new blocks..."); - - while block_count == rpc.get_block_count()? { - sleep(Duration::from_secs(1)) + if let Some(delay) = config.delay() { + sleep(Duration::from_secs(delay)) + } + + info!("Waiting for new blocks..."); + + while block_count == rpc_client.get_block_count()? { + sleep(Duration::from_secs(1)) + } } + } else { + return Err(color_eyre::eyre::eyre!( + "RPC client and parser required for processing mode" + ))?; } } diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index 1e1cc3643..099f50d3a 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -28,6 +28,7 @@ jiff = { workspace = true } log = { workspace = true } minreq = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } tokio = { workspace = true } tower-http = { version = "0.6.6", features = ["compression-full", "trace"] } tracing = "0.1.41" diff --git a/crates/brk_server/src/api/mod.rs b/crates/brk_server/src/api/mod.rs index 99631973c..78c54e522 100644 --- a/crates/brk_server/src/api/mod.rs +++ b/crates/brk_server/src/api/mod.rs @@ -107,6 +107,16 @@ impl ApiRoutes for Router { }, ), ) + .route( + "/health", + get(|| async { + Json(serde_json::json!({ + "status": "healthy", + "service": "brk-server", + "timestamp": jiff::Timestamp::now().to_string() + })) + }), + ) .route( "/api", get(|| async { diff --git a/docker-compose.yml b/docker-compose.yml index b74f8357c..59edf59fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,18 +1,18 @@ +# BRK multi-container Docker Compose configuration + services: - brk: + brk-processor: build: context: . dockerfile: Dockerfile image: brk:latest - container_name: brk + container_name: brk-processor restart: unless-stopped - ports: - - 7070:3110 # Map host port 7070 to container port 3110 volumes: # Bitcoin Core data directory (read-only) # For access to raw block data - ${BITCOIN_DATA_DIR:-/path/to/bitcoin}:/bitcoin:ro - # BRK data directory for outputs and state + # BRK data directory for outputs and state (shared with server) # Option 1: Use a named volume (default) - ${BRK_DATA_VOLUME:-brk-data}:/home/brk/.brk # Option 2: Use a bind mount (uncomment and set BRK_DATA_DIR in .env) @@ -22,8 +22,7 @@ services: - BITCOINDIR=/bitcoin - BLOCKSDIR=/bitcoin/blocks - # RPC configuration - # RPC node access is for chain validation, chain sync status etc. + # RPC configuration (required for processor) - RPCCONNECT=${BTC_RPC_HOST:-localhost} - RPCPORT=${BTC_RPC_PORT:-8332} # - RPCCOOKIEFILE=/bitcoin/.cookie @@ -37,7 +36,6 @@ services: - COMPUTATION=${BRK_COMPUTATION:-lazy} - FORMAT=${BRK_FORMAT:-raw} - FETCH=${BRK_FETCH:-true} - - MCP=${BRK_MCP:-true} command: - --bitcoindir - /bitcoin @@ -50,7 +48,45 @@ services: - --rpcpassword - "${BTC_RPC_PASSWORD:-bitcoin}" - --services - - "${BRK_SERVICES:-all}" # Can be: all, processor, server + - processor + healthcheck: + test: ["CMD", "pgrep", "-f", "brk"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + brk-server: + build: + context: . + dockerfile: Dockerfile + image: brk:latest + container_name: brk-server + restart: unless-stopped + ports: + - 7070:3110 # Map host port 7070 to container port 3110 + volumes: + # BRK data directory (shared with processor) + # Option 1: Use a named volume (default) + - ${BRK_DATA_VOLUME:-brk-data}:/home/brk/.brk + # Option 2: Use a bind mount (uncomment and set BRK_DATA_DIR in .env) + # - ${BRK_DATA_DIR:-./brk-data}:/home/brk/.brk + environment: + # BRK configuration (server doesn't need Bitcoin RPC) + - BRKDIR=/home/brk/.brk + - MCP=${BRK_MCP:-true} + command: + - --brkdir + - /home/brk/.brk + - --services + - server + # Note: Server can start without processor, will serve empty data until processor indexes blocks + healthcheck: + test: ["CMD", "sh", "-c", "nc -z localhost 3110 || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s volumes: brk-data: