From ec3a2f29f0c47824356ad134f2acbc717434c880 Mon Sep 17 00:00:00 2001 From: deadmanoz Date: Thu, 26 Jun 2025 15:35:04 +0800 Subject: [PATCH] Docker functionality, change location of 'blk_index_to_blk_recap.json' --- .dockerignore | 68 ++++++++ .env.example | 42 +++++ .gitignore | 3 + Cargo.toml | 1 + DOCKER.md | 152 ++++++++++++++++++ Dockerfile | 55 +++++++ crates/brk_cli/src/run.rs | 2 +- .../brk_parser/src/blk_index_to_blk_recap.rs | 4 +- crates/brk_parser/src/lib.rs | 14 +- docker-build.sh | 60 +++++++ docker-compose.yml | 57 +++++++ 11 files changed, 453 insertions(+), 5 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 DOCKER.md create mode 100644 Dockerfile create mode 100755 docker-build.sh create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..494e86bb3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,68 @@ +# Git +.git +.gitignore + +# Build artifacts +target/ + +# Development files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS files +.DS_Store +Thumbs.db + +# Docker files +Dockerfile +docker-compose.yml +.dockerignore +docker-build.sh + +# Documentation +docs/ +LICENSE +# Keep README.md for build process +!README.md + +# CI/CD +.github/ + +# Logs and temporary files +*.log +tmp/ +temp/ + +# BRK runtime data (should be in volumes) +.brk/ + +# Example and test data +examples/ +tests/ +*.test +*.example + +# Node modules (if any frontend deps) +node_modules/ + +# Python cache (if any) +__pycache__/ +*.pyc +*.pyo + +# Rust workspace cache +**/*.rs.bk + +# macOS +.AppleDouble +.LSOverride + +# Windows +Desktop.ini +ehthumbs.db + +# Linux +.directory \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..8454693c4 --- /dev/null +++ b/.env.example @@ -0,0 +1,42 @@ +# Bitcoin Core data directory +# This should point to your Bitcoin Core data directory +BITCOIN_DATA_DIR=/path/to/bitcoin + +# Bitcoin Core RPC configuration +# If running Bitcoin Core on the same host (not in Docker), use host.docker.internal on macOS/Windows +# or the host's IP address on Linux +BTC_RPC_HOST=localhost +BTC_RPC_PORT=8332 + +# Use either cookie file authentication (recommended) or username/password +# Cookie file is automatically created by Bitcoin Core +# If using username/password, comment out RPCCOOKIEFILE in docker-compose.yml +# BTC_RPC_USER=your_rpc_username +# BTC_RPC_PASSWORD=your_rpc_password + +# BRK configuration +# Services to run: all, processor, or server +BRK_SERVICES=all + +# Computation mode: lazy (compute on demand) or eager (precompute and save) +BRK_COMPUTATION=lazy + +# Data format: raw (faster) or compressed (saves disk space) +BRK_FORMAT=raw + +# Enable price fetching from exchanges +BRK_FETCH=true + +# Enable Model Context Protocol (MCP) for AI/LLM integration +BRK_MCP=true + +# BRK data storage options +# Option 1: Use a Docker named volume (default, recommended) +# This is the default configuration - no changes needed. +# Leave this commented to use the default named volume +# BRK_DATA_VOLUME=brk-data + +# Option 2: Use a bind mount to a local directory +# Uncomment and set this to use a specific directory on your host +# Also uncomment the corresponding line in docker-compose.yml +# BRK_DATA_DIR=/path/to/brk/data \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8e8834d1e..e18eea715 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ _* # Logs .log + +# Environment variables/configs +.env diff --git a/Cargo.toml b/Cargo.toml index cdc0ec231..da2139dcf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ package.edition = "2024" package.version = "0.0.71" package.homepage = "https://bitcoinresearchkit.org" package.repository = "https://github.com/bitcoinresearchkit/brk" +package.readme = "README.md" [profile.release] lto = "fat" diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 000000000..404c44ead --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,152 @@ +# Docker Setup for BRK + +This guide explains how to run BRK using Docker and Docker Compose. + +## Prerequisites + +- Docker Engine (with buildx support) +- Docker Compose v2 +- A running Bitcoin Core node with RPC enabled +- Access to Bitcoin Core's blocks directory + +## Quick Start + +1. **Create environment file** + ```bash + cp .env.example .env + ``` + Edit `.env` and set `BITCOIN_DATA_DIR` to your Bitcoin Core data directory. + +2. **Run with Docker Compose** + ```bash + docker compose up -d + ``` + +3. **Access BRK** + - Web interface: http://localhost:7070 + - API: http://localhost:7070/api + +## Configuration + +### Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `BITCOIN_DATA_DIR` | Path to Bitcoin Core data directory | Required | +| `BTC_RPC_HOST` | Bitcoin Core RPC host | `localhost` | +| `BTC_RPC_PORT` | Bitcoin Core RPC port | `8332` | +| `BRK_SERVICES` | Services to run (`all`, `processor`, `server`) | `all` | +| `BRK_COMPUTATION` | Computation mode (`lazy`, `eager`) | `lazy` | +| `BRK_FORMAT` | Data format (`raw`, `compressed`) | `raw` | +| `BRK_FETCH` | Enable price fetching | `true` | +| `BRK_MCP` | Enable MCP for AI/LLM | `true` | + +### Service Modes + +- **`all`**: Run both processor and server (default) +- **`processor`**: Only process blockchain data +- **`server`**: Only serve API/web interface + +### Connecting to Bitcoin Core + +#### Option 1: Cookie File Authentication (Recommended) +BRK will automatically use the `.cookie` file from your Bitcoin Core directory. + +#### Option 2: Username/Password +1. Uncomment the RPC user/password lines in `docker-compose.yml` +2. Set `BTC_RPC_USER` and `BTC_RPC_PASSWORD` in your `.env` file + +#### Network Connectivity +- **Same host**: + - If Bitcoin Core is running natively (not in Docker): Use `host.docker.internal` on macOS/Windows or `172.17.0.1` on Linux + - If Bitcoin Core is also in Docker: Use the service name or container IP +- **Remote host**: Use the actual IP address or hostname + +## Building the Image + +### Using Docker Compose (Simple) +```bash +docker compose build +``` + +### Using Docker Build Script +```bash +# Build with default settings +./docker-build.sh + +# Build with custom tag +./docker-build.sh --tag v1.0.0 +``` + +## Volumes and Data Storage + +BRK supports two options for storing its data: + +### Option 1: Docker Named Volume (Default) +Uses a Docker-managed named volume called `brk-data`. This is the recommended approach for most users. + +**Advantages:** +- Managed by Docker +- Easy backup/restore +- Platform-independent + +**Usage:** This is the default configuration - no changes needed. + +### Option 2: Bind Mount +Maps a specific directory on your host to the container's data directory. + +**Advantages:** +- Direct access to files from host +- Easy to locate and manage +- Can be on specific storage devices + +**Usage:** +1. Set `BRK_DATA_DIR` in your `.env` file to your desired host directory +2. In `docker-compose.yml`, comment out the named volume line and uncomment the bind mount line + +**Example:** +```bash +# In .env file +BRK_DATA_DIR=/home/user/brk-data +``` + +```yaml +# In docker-compose.yml, uncomment and change as necessary: + # - ${BRK_DATA_VOLUME:-brk-data}:/home/brk/.brk + - ${BRK_DATA_DIR:-./brk-data}:/home/brk/.brk +``` + +### Volume Details +- **BRK data**: Stores computed datasets, indexes, and application state +- **Bitcoin data**: Mounted read-only from host (always a bind mount) + +## Monitoring + +View logs: +```bash +docker compose logs -f brk +``` + +Check status: +```bash +docker compose ps +``` + +## Troubleshooting + +### Cannot connect to Bitcoin Core +1. Ensure Bitcoin Core is running with `-server=1` +2. Check RPC credentials are correct +3. Verify network connectivity from container + +### Permission denied errors +Ensure the Bitcoin data directory is readable by the container user (UID 1000). + +### Out of memory +Increase Docker's memory limit or use `BRK_COMPUTATION=lazy` to reduce memory usage. + +## Security Considerations + +- Bitcoin data is mounted read-only for safety +- BRK runs as non-root user inside container +- Only necessary ports are exposed diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..f5795c75a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,55 @@ +# Build stage +FROM rust:nightly AS builder + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy all source files +COPY . . + +# Build the application +RUN cargo build --release --locked + +# Runtime stage +FROM debian:bookworm-slim + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + ca-certificates \ + openssl \ + && rm -rf /var/lib/apt/lists/* + +# Create non-root user +RUN useradd -m -s /bin/bash brk + +# Copy binary from builder +COPY --from=builder /app/target/release/brk /usr/local/bin/brk + +# Copy websites directory +COPY --from=builder /app/websites /app/websites + +# Set ownership +RUN chown -R brk:brk /app + +# Switch to non-root user +USER brk + +# Create directories for BRK data +RUN mkdir -p /home/brk/.brk + +# Expose API port +EXPOSE 3110 + +# Set working directory +WORKDIR /home/brk + +# Default entrypoint +ENTRYPOINT ["brk"] + +# Default command (can be overridden) +CMD ["--services", "all"] \ No newline at end of file diff --git a/crates/brk_cli/src/run.rs b/crates/brk_cli/src/run.rs index 398153356..9d42afadf 100644 --- a/crates/brk_cli/src/run.rs +++ b/crates/brk_cli/src/run.rs @@ -16,7 +16,7 @@ pub fn run() -> color_eyre::Result<()> { let exit = Exit::new(); - let parser = brk_parser::Parser::new(config.blocksdir(), rpc); + let parser = brk_parser::Parser::new_with_outputs_dir(config.blocksdir(), config.outputsdir(), rpc); let format = config.format(); diff --git a/crates/brk_parser/src/blk_index_to_blk_recap.rs b/crates/brk_parser/src/blk_index_to_blk_recap.rs index e4e619467..4f8b07862 100644 --- a/crates/brk_parser/src/blk_index_to_blk_recap.rs +++ b/crates/brk_parser/src/blk_index_to_blk_recap.rs @@ -15,11 +15,11 @@ pub struct BlkIndexToBlkRecap { impl BlkIndexToBlkRecap { pub fn import( - bitcoin_dir: &Path, + outputs_dir: &Path, blk_index_to_blk_path: &BlkIndexToBlkPath, start: Option, ) -> (Self, u16) { - let path = bitcoin_dir.join("blk_index_to_blk_recap.json"); + let path = outputs_dir.join("blk_index_to_blk_recap.json"); let tree = { if let Ok(file) = File::open(&path) { diff --git a/crates/brk_parser/src/lib.rs b/crates/brk_parser/src/lib.rs index 59b80bdb7..b5222e212 100644 --- a/crates/brk_parser/src/lib.rs +++ b/crates/brk_parser/src/lib.rs @@ -38,12 +38,22 @@ const BOUND_CAP: usize = 50; pub struct Parser { blocks_dir: PathBuf, + outputs_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client, } impl Parser { pub fn new(blocks_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self { - Self { blocks_dir, rpc } + // For backward compatibility, use blocks_dir as outputs_dir + Self { + outputs_dir: blocks_dir.clone(), + blocks_dir, + rpc + } + } + + pub fn new_with_outputs_dir(blocks_dir: PathBuf, outputs_dir: PathBuf, rpc: &'static bitcoincore_rpc::Client) -> Self { + Self { blocks_dir, outputs_dir, rpc } } pub fn get(&self, height: Height) -> Block { @@ -74,7 +84,7 @@ impl Parser { let blk_index_to_blk_path = BlkIndexToBlkPath::scan(blocks_dir); let (mut blk_index_to_blk_recap, blk_index) = - BlkIndexToBlkRecap::import(blocks_dir, &blk_index_to_blk_path, start); + BlkIndexToBlkRecap::import(&self.outputs_dir, &blk_index_to_blk_path, start); let xor_bytes = XORBytes::from(blocks_dir); diff --git a/docker-build.sh b/docker-build.sh new file mode 100755 index 000000000..359c8a72b --- /dev/null +++ b/docker-build.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Default values +IMAGE_NAME="brk" +TAG="latest" + +# Function to print colored output +print_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +print_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -t|--tag) + TAG="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -t, --tag TAG Tag for the image (default: latest)" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + print_error "Unknown option: $1" + exit 1 + ;; + esac +done + +# Build the image +print_info "Building BRK Docker image..." +print_info "Image: ${IMAGE_NAME}:${TAG}" + +# Execute the build +if docker build -t "${IMAGE_NAME}:${TAG}" .; then + print_info "Build completed successfully!" + print_info "Image built as ${IMAGE_NAME}:${TAG}" +else + print_error "Build failed!" + exit 1 +fi \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..b74f8357c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,57 @@ +services: + brk: + build: + context: . + dockerfile: Dockerfile + image: brk:latest + container_name: brk + restart: unless-stopped + ports: + - 7070:3110 # Map host port 7070 to container port 3110 + volumes: + # Bitcoin Core data directory (read-only) + # For access to raw block data + - ${BITCOIN_DATA_DIR:-/path/to/bitcoin}:/bitcoin:ro + # BRK data directory for outputs and state + # Option 1: Use a named volume (default) + - ${BRK_DATA_VOLUME:-brk-data}:/home/brk/.brk + # Option 2: Use a bind mount (uncomment and set BRK_DATA_DIR in .env) + # - ${BRK_DATA_DIR:-./brk-data}:/home/brk/.brk + environment: + # Bitcoin Core configuration + - BITCOINDIR=/bitcoin + - BLOCKSDIR=/bitcoin/blocks + + # RPC configuration + # RPC node access is for chain validation, chain sync status etc. + - RPCCONNECT=${BTC_RPC_HOST:-localhost} + - RPCPORT=${BTC_RPC_PORT:-8332} + # - RPCCOOKIEFILE=/bitcoin/.cookie + + # Username/password authentication + - RPCUSER=${BTC_RPC_USER} + - RPCPASSWORD=${BTC_RPC_PASSWORD} + + # BRK configuration + - BRKDIR=/home/brk/.brk + - COMPUTATION=${BRK_COMPUTATION:-lazy} + - FORMAT=${BRK_FORMAT:-raw} + - FETCH=${BRK_FETCH:-true} + - MCP=${BRK_MCP:-true} + command: + - --bitcoindir + - /bitcoin + - --brkdir + - /home/brk/.brk + - --rpcconnect + - "${BTC_RPC_HOST:-localhost}" + - --rpcuser + - "${BTC_RPC_USER:-bitcoin}" + - --rpcpassword + - "${BTC_RPC_PASSWORD:-bitcoin}" + - --services + - "${BRK_SERVICES:-all}" # Can be: all, processor, server + +volumes: + brk-data: + driver: local \ No newline at end of file