From 691952249babce1f24e6b1f3419076a6914420b6 Mon Sep 17 00:00:00 2001 From: nym21 Date: Sun, 2 Mar 2025 11:08:53 +0100 Subject: [PATCH] readme: update --- Cargo.lock | 38 ++++++----- Cargo.toml | 2 +- README.md | 184 +++++++++++++++++------------------------------------ publish.sh | 25 ++++---- 4 files changed, 97 insertions(+), 152 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5506bdf69..60c57795b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -324,7 +324,7 @@ checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" [[package]] name = "brk" -version = "0.0.2" +version = "0.0.3" dependencies = [ "brk_computer", "brk_core", @@ -340,7 +340,7 @@ dependencies = [ [[package]] name = "brk_cli" -version = "0.0.2" +version = "0.0.3" dependencies = [ "brk_computer", "brk_core", @@ -351,11 +351,13 @@ dependencies = [ "brk_query", "brk_server", "clap", + "color-eyre", + "log", ] [[package]] name = "brk_computer" -version = "0.0.2" +version = "0.0.3" dependencies = [ "brk_core", "brk_exit", @@ -370,7 +372,7 @@ dependencies = [ [[package]] name = "brk_core" -version = "0.0.2" +version = "0.0.3" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -387,7 +389,7 @@ dependencies = [ [[package]] name = "brk_exit" -version = "0.0.2" +version = "0.0.3" dependencies = [ "ctrlc", "log", @@ -395,7 +397,7 @@ dependencies = [ [[package]] name = "brk_fetcher" -version = "0.0.2" +version = "0.0.3" dependencies = [ "brk_core", "brk_logger", @@ -408,7 +410,7 @@ dependencies = [ [[package]] name = "brk_indexer" -version = "0.0.2" +version = "0.0.3" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -427,7 +429,7 @@ dependencies = [ [[package]] name = "brk_logger" -version = "0.0.2" +version = "0.0.3" dependencies = [ "color-eyre", "env_logger", @@ -436,7 +438,7 @@ dependencies = [ [[package]] name = "brk_parser" -version = "0.0.2" +version = "0.0.3" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -451,16 +453,21 @@ dependencies = [ [[package]] name = "brk_query" -version = "0.0.2" +version = "0.0.3" dependencies = [ "brk_computer", "brk_indexer", + "brk_vec", "clap", + "color-eyre", + "derive_deref", + "serde", + "serde_json", ] [[package]] name = "brk_server" -version = "0.0.2" +version = "0.0.3" dependencies = [ "axum", "brk_computer", @@ -468,6 +475,7 @@ dependencies = [ "brk_indexer", "brk_logger", "brk_parser", + "brk_query", "brk_vec", "color-eyre", "derive_deref", @@ -482,7 +490,7 @@ dependencies = [ [[package]] name = "brk_vec" -version = "0.0.2" +version = "0.0.3" dependencies = [ "brk_exit", "memmap2", @@ -1897,12 +1905,12 @@ dependencies = [ [[package]] name = "quick_cache" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f67cfc9c723c39f3615eb0840b00c4cb9e2b068d2fa761a30d845ec91730a59" +checksum = "0af25b4e960ffdf0dead61cf0cec0c2e44c76927bf933ab4f02e2858fb449397" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 57867baa8..58c1b9f5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["crates/*"] package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node" package.license = "MIT" package.edition = "2024" -package.version = "0.0.2" +package.version = "0.0.3" package.repository = "https://github.com/bitcoinresearchkit/brk" [workspace.dependencies] diff --git a/README.md b/README.md index 784c024fe..edc46a0a5 100644 --- a/README.md +++ b/README.md @@ -7,98 +7,20 @@ The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node. -## Crates +- [`brk`](https://crates.io/crates/brk): Wrapper around all other `brk-*` crates +- [`brk_cli`](https://crates.io/crates/brk_cli): A command line interface to interact with the Bitcoin Research Kit +- [`brk_computer`](https://crates.io/crates/brk_computer): A Bitcoin dataset computer, built on top of brk_indexer +- [`brk_core`](https://crates.io/crates/brk_core): The Core (Structs and Errors) of the Bitcoin Research Kit +- [`brk_exit`](https://crates.io/crates/brk_exit): An exit blocker built on top of ctrlc +- [`brk_fetcher`](https://crates.io/crates/brk_fetcher): A Bitcoin price fetcher +- [`brk_indexer`](https://crates.io/crates/brk_indexer): A Bitcoin Core indexer built on top of brk_parser +- [`brk_logger`](https://crates.io/crates/brk_logger): A clean logger used in the Bitcoin Research Kit. +- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust +- [`brk_query`](https://crates.io/crates/brk_query): A library that finds requested datasets. +- [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of brk_indexer, brk_fetcher and brk_computer +- [`brk_vec`](https://crates.io/crates/brk_vec): A very small, fast, efficient and simple storable Vec. -### `brk` - -Wrapper around all other `brk-*` crates. - -> Status: ⚠️ - -### `brk_cli` - -A command line interface to interact with the Bitcoin Research Kit. - -> Status: ❌ - -### `brk_computer` - -A Bitcoin dataset computer, built on top of brk_indexer. - -> Status: ⚠️ - -### `brk_core` - -The Core (Structs and Errors) of the Bitcoin Research Kit. - -> Status: ✅ - -### `brk_exit` - -An exit blocker built on top of ctrlc. - -> Status: ✅ - -### `brk_fetcher` - -A Bitcoin price fetcher. - -> Status: ✅ - -### `brk_indexer` - -A Bitcoin Core indexer built on top of brk_parser. - -> Status: ✅ - -### `brk_logger` - -A clean logger used in the Bitcoin Research Kit. - -> Status: ✅ - -### `brk_parser` - -A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust. - -> Status: ✅ - -### `brk_query` - -A library that finds requested datasets. - -> Status: ⚠️ - -### `brk_server` - -A server that serves Bitcoin data and swappable front-ends, built on top of brk_indexer, brk_fetcher and brk_computer. - -> Status: ⚠️ - -### `brk_vec` - -A very small, fast, efficient and simple storable Vec. - -> Status: ✅ - -## Old README - -[**kibō**](https://kibo.money) (_hope_ in japanese) is primarily an open source Bitcoin Core data extractor and visualizer (similar to [Glassnode](https://glassnode.com)) which goal is to empower anybody with data about Bitcoin for free. - -The project is split in 3 parts: - -- First you have the extractor (parser), which parses the block data files from your Bitcoin Core node and computes a very wide range of datasets which are stored in compressed binary files - > For the curious, it takes at the very least 24 hours to parse all the blocks and compute all datasets. After that it will wait for a new block and take between 1 and 3 minutes to be up to date -- Then there is the website on which you can view, among other things, all datasets in various charts -- Finally there is the server which serves the website and the generated data via an [API](https://github.com/kibo-money/kibo/tree/main#endpoints) - -Whether you're an enthusiast, a researcher, a miner, an analyst, a trader, a skeptic or just curious, there is something for everyone ! - -This project was created out of frustration by all the alternatives that were either very expensive and thus discriminatory and against bitcoin values or just very limited and none were open-source and verifiable. So while it's not the first tool trying to solve these problems, it's the first that is completely free, open-source and self-hostable. - -If you are a user of [mempool.space](https://mempool.space), you'll find this to be very complimentary, as it offers a macro view of the chain over time instead of a detailed one. - -## Instances +## Servers | URL | Front-end | Version | Status | Last Height | Up Time Ratio | | ------------------------------------------------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | @@ -106,52 +28,32 @@ If you are a user of [mempool.space](https://mempool.space), you'll find this to Feel free to open an issue if you want to add another instance -## Endpoints - -> If you running locally, you can replace `https://kibo.money` by `http://localhost:3110` - -- [/](https://kibo.money/): Website -- [/api](https://kibo.money/api): A JSON with all available datasets, with their respective id and endpoint, better viewed in a Firefox based browser -- /api/TIMESCALE-to-ID: `TIMESCALE` can be `date` or `height`, and `ID` is the id with `_` replaced by `-`, let's take `date-to-close` (price at the end of each day) as an example - - [/api/date-to-close](https://kibo.money/api/date-to-close): current year's values in a json format - - [/api/date-to-close?chunk=2009](https://kibo.money/api/date-to-close?chunk=2009): values from the year 2009 in a json format - - [/api/date-to-close?all=true](https://kibo.money/api/date-to-close?all=true): all values in a json format - - You can also specify the extension to download a file, either `.json` or `.csv` to get the dataset in a CSV format; like so: - - [/api/date-to-close.csv](https://kibo.money/api/date-to-close.csv) - - [/api/date-to-close.csv?chunk=2009](https://kibo.money/api/date-to-close.csv?chunk=2009) - - [/api/date-to-close.csv?all=true](https://kibo.money/api/date-to-close.csv?all=true) - ## Setup ### Hardware -- Last base model Mac mini -- External SSD +#### Recommended + +- [Latest base model Mac mini](https://www.apple.com/mac-mini/) +- [Thunderbolt 4 SSD enclosure](https://satechi.net/products/usb4-nvme-ssd-pro-enclosure/Z2lkOi8vc2hvcGlmeS9Qcm9kdWN0VmFyaWFudC80MDE4ODQ3MDA2NzI4OA==?queryID=7961465089021ee203a60db7e62e90d2) +- [2 TB NVMe SSD](https://shop.sandisk.com/products/ssd/internal-ssd/wd-black-sn850x-nvme-ssd?sku=WDS200T2X0E-00BCA0) + +#### Minimum + +To be determined ### Requirements -- At least 16 GB of RAM -- A disk with 1 TB of free space (will use between 40% to 80% depending on several things) - - Recommended: Rated at 3 GB/s (Thunderbolt 4 speed) -- A running instance of bitcoin-core - - Example: `bitcoind -datadir="$HOME/.bitcoin" -blocksonly` - Unix based operating system (Mac OS or Linux) - Ubuntu users need to install `open-ssl` via `sudo apt install libssl-dev pkg-config` +- [Bitcoin](https://bitcoin.org/en/full-node) + - Example: `bitcoind -datadir="$HOME/.bitcoin" -blocksonly` +- [Rust](https://www.rust-lang.org/tools/install) + - Install: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` + - Update: `rustup update` ### Build -First we need to install Rust (https://www.rust-lang.org/tools/install) - -```bash -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -``` - -If you already had Rust installed you could update it - -```bash -rustup update -``` - Then you need to choose a path where the project will reside and then clone it ```bash @@ -188,3 +90,35 @@ Then the easiest to let others access your server is to use `cloudflared` which [lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4](lightning:lnurl1dp68gurn8ghj7ampd3kx2ar0veekzar0wd5xjtnrdakj7tnhv4kxctttdehhwm30d3h82unvwqhkxmmww3jkuar8d35kgetj8yuq363hv4) [Geyser Fund](https://geyser.fund/project/kibo/) + +## Old README + +[**kibō**](https://kibo.money) (_hope_ in japanese) is primarily an open source Bitcoin Core data extractor and visualizer (similar to [Glassnode](https://glassnode.com)) which goal is to empower anybody with data about Bitcoin for free. + +The project is split in 3 parts: + +- First you have the extractor (parser), which parses the block data files from your Bitcoin Core node and computes a very wide range of datasets which are stored in compressed binary files + > For the curious, it takes at the very least 24 hours to parse all the blocks and compute all datasets. After that it will wait for a new block and take between 1 and 3 minutes to be up to date +- Then there is the website on which you can view, among other things, all datasets in various charts +- Finally there is the server which serves the website and the generated data via an [API](https://github.com/kibo-money/kibo/tree/main#endpoints) + +Whether you're an enthusiast, a researcher, a miner, an analyst, a trader, a skeptic or just curious, there is something for everyone ! + +This project was created out of frustration by all the alternatives that were either very expensive and thus discriminatory and against bitcoin values or just very limited and none were open-source and verifiable. So while it's not the first tool trying to solve these problems, it's the first that is completely free, open-source and self-hostable. + +If you are a user of [mempool.space](https://mempool.space), you'll find this to be very complimentary, as it offers a macro view of the chain over time instead of a detailed one. + +## Endpoints + +> If you running locally, you can replace `https://kibo.money` by `http://localhost:3110` + +- [/](https://kibo.money/): Website +- [/api](https://kibo.money/api): A JSON with all available datasets, with their respective id and endpoint, better viewed in a Firefox based browser +- /api/TIMESCALE-to-ID: `TIMESCALE` can be `date` or `height`, and `ID` is the id with `_` replaced by `-`, let's take `date-to-close` (price at the end of each day) as an example + - [/api/date-to-close](https://kibo.money/api/date-to-close): current year's values in a json format + - [/api/date-to-close?chunk=2009](https://kibo.money/api/date-to-close?chunk=2009): values from the year 2009 in a json format + - [/api/date-to-close?all=true](https://kibo.money/api/date-to-close?all=true): all values in a json format + - You can also specify the extension to download a file, either `.json` or `.csv` to get the dataset in a CSV format; like so: + - [/api/date-to-close.csv](https://kibo.money/api/date-to-close.csv) + - [/api/date-to-close.csv?chunk=2009](https://kibo.money/api/date-to-close.csv?chunk=2009) + - [/api/date-to-close.csv?all=true](https://kibo.money/api/date-to-close.csv?all=true) diff --git a/publish.sh b/publish.sh index 49cfbefd7..5af089e54 100755 --- a/publish.sh +++ b/publish.sh @@ -3,36 +3,39 @@ cd crates/brk cd ../brk_core -cargo publish --allow-dirty +cargo publish cd ../brk_exit -cargo publish --allow-dirty +cargo publish cd ../brk_vec -cargo publish --allow-dirty +cargo publish cd ../brk_logger -cargo publish --allow-dirty +cargo publish cd ../brk_indexer -cargo publish --allow-dirty +cargo publish cd ../brk_parser -cargo publish --allow-dirty +cargo publish cd ../brk_fetcher -cargo publish --allow-dirty +cargo publish cd ../brk_computer -cargo publish --allow-dirty +cargo publish + +cd ../brk_query +cargo publish cd ../brk_server -cargo publish --allow-dirty +cargo publish cd ../brk_cli -cargo publish --allow-dirty +cargo publish cd ../brk -cargo publish --allow-dirty +cargo publish cd ../..