diff --git a/.gitignore b/.gitignore index 70eb0fa00..d0bd3df09 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ target *\ copy* # Ignored -_ignore +/_* # Editors .vscode diff --git a/Cargo.lock b/Cargo.lock index e7ab13591..d764e42e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,46 +238,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" -[[package]] -name = "berver" -version = "0.1.0" -dependencies = [ - "axum", - "bindexer", - "bomputer", - "cl0g", - "color-eyre", - "derive_deref", - "jiff", - "oxc", - "serde", - "serde_json", - "storable_vec", - "tokio", - "tower-http", -] - -[[package]] -name = "bindexer" -version = "0.1.0" -dependencies = [ - "bitcoin", - "biterator", - "cl0g", - "color-eyre", - "derive_deref", - "fjall", - "hodor", - "jiff", - "rapidhash", - "rayon", - "rlimit", - "serde", - "serde_bytes", - "storable_vec", - "zerocopy 0.8.20", -] - [[package]] name = "bitcoin" version = "0.32.5" @@ -357,7 +317,71 @@ dependencies = [ ] [[package]] -name = "biterator" +name = "bitflags" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" + +[[package]] +name = "brk_cli" +version = "0.1.0" + +[[package]] +name = "brk_computer" +version = "0.1.0" +dependencies = [ + "brk_fetcher", + "brk_indexer", + "brk_parser", + "color-eyre", + "derive_deref", + "fjall", + "hodor", + "storable_vec", + "zerocopy 0.8.20", +] + +[[package]] +name = "brk_fetcher" +version = "0.1.0" +dependencies = [ + "brk_indexer", + "brk_printer", + "color-eyre", + "derive_deref", + "jiff", + "log", + "minreq", + "serde", + "serde_json", + "storable_vec", + "zerocopy 0.8.20", +] + +[[package]] +name = "brk_indexer" +version = "0.1.0" +dependencies = [ + "bitcoin", + "brk_parser", + "brk_printer", + "color-eyre", + "derive_deref", + "fjall", + "hodor", + "jiff", + "log", + "rapidhash", + "rayon", + "rlimit", + "serde", + "serde_bytes", + "storable_vec", + "zerocopy 0.8.20", +] + +[[package]] +name = "brk_parser" version = "0.2.3" dependencies = [ "bitcoin", @@ -372,44 +396,33 @@ dependencies = [ ] [[package]] -name = "bitflags" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" - -[[package]] -name = "bli" -version = "0.1.0" - -[[package]] -name = "bomputer" +name = "brk_printer" version = "0.1.0" dependencies = [ - "bindexer", - "biterator", - "bricer", "color-eyre", - "derive_deref", - "fjall", - "hodor", - "storable_vec", - "zerocopy 0.8.20", + "env_logger", + "jiff", + "log", ] [[package]] -name = "bricer" +name = "brk_server" version = "0.1.0" dependencies = [ - "bindexer", - "cl0g", + "axum", + "brk_computer", + "brk_indexer", + "brk_printer", "color-eyre", "derive_deref", "jiff", - "minreq", + "log", + "oxc", "serde", "serde_json", "storable_vec", - "zerocopy 0.8.20", + "tokio", + "tower-http", ] [[package]] @@ -492,16 +505,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" -[[package]] -name = "cl0g" -version = "0.1.0" -dependencies = [ - "color-eyre", - "env_logger", - "jiff", - "log", -] - [[package]] name = "color-eyre" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index ef5ae6f42..eff4598be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,39 +1,29 @@ [workspace] -members = [ - "cli", - "computer", - "hodor", - "indexer", - "iterator", - "logger", - "pricer", - "server", - "storable_vec", -] +members = ["crates/*"] resolver = "2" package.license = "MIT" package.edition = "2024" [workspace.dependencies] bitcoin = { version = "0.32.5", features = ["serde"] } +brk_computer = { version = "0", path = "crates/brk_computer" } +brk_fetcher = { version = "0", path = "crates/brk_fetcher" } +brk_indexer = { version = "0", path = "crates/brk_indexer" } +brk_parser = { version = "0", path = "crates/brk_parser", features = ["bytes"] } +brk_printer = { version = "0", path = "crates/brk_printer" } +brk_server = { version = "0", path = "crates/brk_server" } color-eyre = "0.6.3" -computer = { version = "0", path = "computer", package = "bomputer" } derive_deref = "1.1.1" fjall = "2.6.5" -hodor = { version = "0", path = "hodor" } -indexer = { version = "0", path = "indexer", package = "bindexer" } -iterator = { version = "0", path = "iterator", package = "biterator", features = [ - "bytes", -] } +hodor = { version = "0", path = "crates/hodor" } jiff = "0.2.1" log = { version = "0.4.26" } -logger = { version = "0", path = "logger", package = "cl0g" } minreq = { version = "2.13.2", features = ["https", "serde_json"] } -pricer = { version = "0", path = "pricer", package = "bricer" } rayon = "1.10.0" serde = { version = "1.0.218", features = ["derive"] } serde_bytes = "0.11.15" serde_json = { version = "1.0.139", features = ["float_roundtrip"] } -server = { version = "0", path = "server", package = "berver" } -storable_vec = { version = "0", path = "storable_vec", features = ["json"] } +storable_vec = { version = "0", path = "crates/storable_vec", features = [ + "json", +] } zerocopy = { version = "0.8.20", features = ["derive"] } diff --git a/README.md b/README.md index 900f36fa9..7aecf84a9 100644 --- a/README.md +++ b/README.md @@ -63,14 +63,10 @@ Please open an issue if you want to add another instance ### Requirements - At least 16 GB of RAM - - Recommended: 32 GB - A disk with 1 TB of free space (will use between 40% to 80% depending on several things) - Recommended: Rated at 3 GB/s (Thunderbolt 4 speed) -- A running instance of bitcoin-core with: - - `-txindex=1` - - `-blocksxor=0` - - RPC credentials - - Example: `bitcoind -datadir="$HOME/.bitcoin" -blocksonly -txindex=1 -blocksxor=0` +- A running instance of bitcoin-core + - Example: `bitcoind -datadir="$HOME/.bitcoin" -blocksonly` - Git - Unix based operating system (Mac OS or Linux) - Ubuntu users need to install `open-ssl` via `sudo apt install libssl-dev pkg-config` diff --git a/src/main.rs b/_src/main.rs similarity index 93% rename from src/main.rs rename to _src/main.rs index 1996a7aa5..3dfe00316 100644 --- a/src/main.rs +++ b/_src/main.rs @@ -1,6 +1,6 @@ -use iterator::bitcoincore_rpc::Client; +use brk_parser::bitcoincore_rpc::Client; use log::info; -use rlimit::{getrlimit, setrlimit, Resource}; +use rlimit::{Resource, getrlimit, setrlimit}; mod io; mod parser; @@ -8,7 +8,7 @@ mod server; mod structs; mod utils; -use parser::Datasets; +use brk_parser::Datasets; use server::api::structs::Routes; use structs::{Config, Exit}; use utils::init_log; diff --git a/src/parser/actions/export.rs b/_src/parser/actions/export.rs similarity index 100% rename from src/parser/actions/export.rs rename to _src/parser/actions/export.rs diff --git a/src/parser/actions/iter_blocks.rs b/_src/parser/actions/iter_blocks.rs similarity index 99% rename from src/parser/actions/iter_blocks.rs rename to _src/parser/actions/iter_blocks.rs index b5d9edfc0..bb8eb84be 100644 --- a/src/parser/actions/iter_blocks.rs +++ b/_src/parser/actions/iter_blocks.rs @@ -1,8 +1,8 @@ use std::{collections::BTreeSet, time::Instant}; +use brk_parser::bitcoincore_rpc::Client; use chrono::Datelike; use export::ExportedData; -use iterator::bitcoincore_rpc::Client; use itertools::Itertools; use log::info; diff --git a/src/parser/actions/min_height.rs b/_src/parser/actions/min_height.rs similarity index 100% rename from src/parser/actions/min_height.rs rename to _src/parser/actions/min_height.rs diff --git a/src/parser/actions/mod.rs b/_src/parser/actions/mod.rs similarity index 100% rename from src/parser/actions/mod.rs rename to _src/parser/actions/mod.rs diff --git a/src/parser/actions/parse.rs b/_src/parser/actions/parse.rs similarity index 99% rename from src/parser/actions/parse.rs rename to _src/parser/actions/parse.rs index b526a71a9..d601ac62c 100644 --- a/src/parser/actions/parse.rs +++ b/_src/parser/actions/parse.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, ops::ControlFlow, thread}; -use iterator::{ +use brk_parser::{ bitcoin::{Block, Txid}, bitcoincore_rpc::RpcApi, }; diff --git a/src/parser/databases/_trait.rs b/_src/parser/databases/_trait.rs similarity index 100% rename from src/parser/databases/_trait.rs rename to _src/parser/databases/_trait.rs diff --git a/src/parser/databases/address_index_to_address_data.rs b/_src/parser/databases/address_index_to_address_data.rs similarity index 100% rename from src/parser/databases/address_index_to_address_data.rs rename to _src/parser/databases/address_index_to_address_data.rs diff --git a/src/parser/databases/address_index_to_empty_address_data.rs b/_src/parser/databases/address_index_to_empty_address_data.rs similarity index 100% rename from src/parser/databases/address_index_to_empty_address_data.rs rename to _src/parser/databases/address_index_to_empty_address_data.rs diff --git a/src/parser/databases/address_to_address_index.rs b/_src/parser/databases/address_to_address_index.rs similarity index 100% rename from src/parser/databases/address_to_address_index.rs rename to _src/parser/databases/address_to_address_index.rs diff --git a/src/parser/databases/metadata.rs b/_src/parser/databases/metadata.rs similarity index 100% rename from src/parser/databases/metadata.rs rename to _src/parser/databases/metadata.rs diff --git a/src/parser/databases/mod.rs b/_src/parser/databases/mod.rs similarity index 100% rename from src/parser/databases/mod.rs rename to _src/parser/databases/mod.rs diff --git a/src/parser/databases/txid_to_tx_data.rs b/_src/parser/databases/txid_to_tx_data.rs similarity index 99% rename from src/parser/databases/txid_to_tx_data.rs rename to _src/parser/databases/txid_to_tx_data.rs index 4ca277d4a..0d3581aec 100644 --- a/src/parser/databases/txid_to_tx_data.rs +++ b/_src/parser/databases/txid_to_tx_data.rs @@ -5,7 +5,7 @@ use std::{ }; use allocative::Allocative; -use iterator::bitcoin::Txid; +use brk_parser::bitcoin::Txid; use itertools::Itertools; use snkrj::{AnyDatabase, Database as _Database}; diff --git a/src/parser/databases/txout_index_to_address_index.rs b/_src/parser/databases/txout_index_to_address_index.rs similarity index 100% rename from src/parser/databases/txout_index_to_address_index.rs rename to _src/parser/databases/txout_index_to_address_index.rs diff --git a/src/parser/databases/txout_index_to_amount.rs b/_src/parser/databases/txout_index_to_amount.rs similarity index 100% rename from src/parser/databases/txout_index_to_amount.rs rename to _src/parser/databases/txout_index_to_amount.rs diff --git a/src/parser/datasets/_traits/any_dataset.rs b/_src/parser/datasets/_traits/any_dataset.rs similarity index 100% rename from src/parser/datasets/_traits/any_dataset.rs rename to _src/parser/datasets/_traits/any_dataset.rs diff --git a/src/parser/datasets/_traits/any_dataset_group.rs b/_src/parser/datasets/_traits/any_dataset_group.rs similarity index 100% rename from src/parser/datasets/_traits/any_dataset_group.rs rename to _src/parser/datasets/_traits/any_dataset_group.rs diff --git a/src/parser/datasets/_traits/any_datasets.rs b/_src/parser/datasets/_traits/any_datasets.rs similarity index 100% rename from src/parser/datasets/_traits/any_datasets.rs rename to _src/parser/datasets/_traits/any_datasets.rs diff --git a/src/parser/datasets/_traits/min_initial_state.rs b/_src/parser/datasets/_traits/min_initial_state.rs similarity index 100% rename from src/parser/datasets/_traits/min_initial_state.rs rename to _src/parser/datasets/_traits/min_initial_state.rs diff --git a/src/parser/datasets/_traits/mod.rs b/_src/parser/datasets/_traits/mod.rs similarity index 100% rename from src/parser/datasets/_traits/mod.rs rename to _src/parser/datasets/_traits/mod.rs diff --git a/src/parser/datasets/address/all_metadata.rs b/_src/parser/datasets/address/all_metadata.rs similarity index 100% rename from src/parser/datasets/address/all_metadata.rs rename to _src/parser/datasets/address/all_metadata.rs diff --git a/src/parser/datasets/address/cohort.rs b/_src/parser/datasets/address/cohort.rs similarity index 100% rename from src/parser/datasets/address/cohort.rs rename to _src/parser/datasets/address/cohort.rs diff --git a/src/parser/datasets/address/cohort_metadata.rs b/_src/parser/datasets/address/cohort_metadata.rs similarity index 100% rename from src/parser/datasets/address/cohort_metadata.rs rename to _src/parser/datasets/address/cohort_metadata.rs diff --git a/src/parser/datasets/address/mod.rs b/_src/parser/datasets/address/mod.rs similarity index 100% rename from src/parser/datasets/address/mod.rs rename to _src/parser/datasets/address/mod.rs diff --git a/src/parser/datasets/block_metadata.rs b/_src/parser/datasets/block_metadata.rs similarity index 100% rename from src/parser/datasets/block_metadata.rs rename to _src/parser/datasets/block_metadata.rs diff --git a/src/parser/datasets/coindays.rs b/_src/parser/datasets/coindays.rs similarity index 100% rename from src/parser/datasets/coindays.rs rename to _src/parser/datasets/coindays.rs diff --git a/src/parser/datasets/cointime.rs b/_src/parser/datasets/cointime.rs similarity index 100% rename from src/parser/datasets/cointime.rs rename to _src/parser/datasets/cointime.rs diff --git a/src/parser/datasets/constant.rs b/_src/parser/datasets/constant.rs similarity index 100% rename from src/parser/datasets/constant.rs rename to _src/parser/datasets/constant.rs diff --git a/src/parser/datasets/date_metadata.rs b/_src/parser/datasets/date_metadata.rs similarity index 100% rename from src/parser/datasets/date_metadata.rs rename to _src/parser/datasets/date_metadata.rs diff --git a/src/parser/datasets/mining.rs b/_src/parser/datasets/mining.rs similarity index 100% rename from src/parser/datasets/mining.rs rename to _src/parser/datasets/mining.rs diff --git a/src/parser/datasets/mod.rs b/_src/parser/datasets/mod.rs similarity index 100% rename from src/parser/datasets/mod.rs rename to _src/parser/datasets/mod.rs diff --git a/src/parser/datasets/price.rs b/_src/parser/datasets/price.rs similarity index 100% rename from src/parser/datasets/price.rs rename to _src/parser/datasets/price.rs diff --git a/src/parser/datasets/subs/capitalization.rs b/_src/parser/datasets/subs/capitalization.rs similarity index 100% rename from src/parser/datasets/subs/capitalization.rs rename to _src/parser/datasets/subs/capitalization.rs diff --git a/src/parser/datasets/subs/input.rs b/_src/parser/datasets/subs/input.rs similarity index 100% rename from src/parser/datasets/subs/input.rs rename to _src/parser/datasets/subs/input.rs diff --git a/src/parser/datasets/subs/mod.rs b/_src/parser/datasets/subs/mod.rs similarity index 100% rename from src/parser/datasets/subs/mod.rs rename to _src/parser/datasets/subs/mod.rs diff --git a/src/parser/datasets/subs/output.rs b/_src/parser/datasets/subs/output.rs similarity index 100% rename from src/parser/datasets/subs/output.rs rename to _src/parser/datasets/subs/output.rs diff --git a/src/parser/datasets/subs/price_paid.rs b/_src/parser/datasets/subs/price_paid.rs similarity index 100% rename from src/parser/datasets/subs/price_paid.rs rename to _src/parser/datasets/subs/price_paid.rs diff --git a/src/parser/datasets/subs/ratio.rs b/_src/parser/datasets/subs/ratio.rs similarity index 100% rename from src/parser/datasets/subs/ratio.rs rename to _src/parser/datasets/subs/ratio.rs diff --git a/src/parser/datasets/subs/realized.rs b/_src/parser/datasets/subs/realized.rs similarity index 100% rename from src/parser/datasets/subs/realized.rs rename to _src/parser/datasets/subs/realized.rs diff --git a/src/parser/datasets/subs/recap.rs b/_src/parser/datasets/subs/recap.rs similarity index 100% rename from src/parser/datasets/subs/recap.rs rename to _src/parser/datasets/subs/recap.rs diff --git a/src/parser/datasets/subs/supply.rs b/_src/parser/datasets/subs/supply.rs similarity index 100% rename from src/parser/datasets/subs/supply.rs rename to _src/parser/datasets/subs/supply.rs diff --git a/src/parser/datasets/subs/unrealized.rs b/_src/parser/datasets/subs/unrealized.rs similarity index 100% rename from src/parser/datasets/subs/unrealized.rs rename to _src/parser/datasets/subs/unrealized.rs diff --git a/src/parser/datasets/subs/utxo.rs b/_src/parser/datasets/subs/utxo.rs similarity index 100% rename from src/parser/datasets/subs/utxo.rs rename to _src/parser/datasets/subs/utxo.rs diff --git a/src/parser/datasets/transaction.rs b/_src/parser/datasets/transaction.rs similarity index 100% rename from src/parser/datasets/transaction.rs rename to _src/parser/datasets/transaction.rs diff --git a/src/parser/datasets/utxo/dataset.rs b/_src/parser/datasets/utxo/dataset.rs similarity index 100% rename from src/parser/datasets/utxo/dataset.rs rename to _src/parser/datasets/utxo/dataset.rs diff --git a/src/parser/datasets/utxo/mod.rs b/_src/parser/datasets/utxo/mod.rs similarity index 100% rename from src/parser/datasets/utxo/mod.rs rename to _src/parser/datasets/utxo/mod.rs diff --git a/src/parser/mod.rs b/_src/parser/mod.rs similarity index 95% rename from src/parser/mod.rs rename to _src/parser/mod.rs index df9692a70..57fd87c40 100644 --- a/src/parser/mod.rs +++ b/_src/parser/mod.rs @@ -1,6 +1,6 @@ use std::{thread::sleep, time::Duration}; -use iterator::bitcoincore_rpc::{Client, RpcApi}; +use brk_parser::bitcoincore_rpc::{Client, RpcApi}; mod actions; mod databases; diff --git a/src/parser/price/binance.rs b/_src/parser/price/binance.rs similarity index 100% rename from src/parser/price/binance.rs rename to _src/parser/price/binance.rs diff --git a/src/parser/price/kibo.rs b/_src/parser/price/kibo.rs similarity index 100% rename from src/parser/price/kibo.rs rename to _src/parser/price/kibo.rs diff --git a/src/parser/price/kraken.rs b/_src/parser/price/kraken.rs similarity index 100% rename from src/parser/price/kraken.rs rename to _src/parser/price/kraken.rs diff --git a/src/parser/price/mod.rs b/_src/parser/price/mod.rs similarity index 100% rename from src/parser/price/mod.rs rename to _src/parser/price/mod.rs diff --git a/src/parser/states/_trait.rs b/_src/parser/states/_trait.rs similarity index 100% rename from src/parser/states/_trait.rs rename to _src/parser/states/_trait.rs diff --git a/src/parser/states/cohorts_states/address/cohort_durable_states.rs b/_src/parser/states/cohorts_states/address/cohort_durable_states.rs similarity index 100% rename from src/parser/states/cohorts_states/address/cohort_durable_states.rs rename to _src/parser/states/cohorts_states/address/cohort_durable_states.rs diff --git a/src/parser/states/cohorts_states/address/cohort_id.rs b/_src/parser/states/cohorts_states/address/cohort_id.rs similarity index 100% rename from src/parser/states/cohorts_states/address/cohort_id.rs rename to _src/parser/states/cohorts_states/address/cohort_id.rs diff --git a/src/parser/states/cohorts_states/address/cohorts_durable_states.rs b/_src/parser/states/cohorts_states/address/cohorts_durable_states.rs similarity index 100% rename from src/parser/states/cohorts_states/address/cohorts_durable_states.rs rename to _src/parser/states/cohorts_states/address/cohorts_durable_states.rs diff --git a/src/parser/states/cohorts_states/address/cohorts_input_states.rs b/_src/parser/states/cohorts_states/address/cohorts_input_states.rs similarity index 100% rename from src/parser/states/cohorts_states/address/cohorts_input_states.rs rename to _src/parser/states/cohorts_states/address/cohorts_input_states.rs diff --git a/src/parser/states/cohorts_states/address/cohorts_one_shot_states.rs b/_src/parser/states/cohorts_states/address/cohorts_one_shot_states.rs similarity index 100% rename from src/parser/states/cohorts_states/address/cohorts_one_shot_states.rs rename to _src/parser/states/cohorts_states/address/cohorts_one_shot_states.rs diff --git a/src/parser/states/cohorts_states/address/cohorts_output_states.rs b/_src/parser/states/cohorts_states/address/cohorts_output_states.rs similarity index 100% rename from src/parser/states/cohorts_states/address/cohorts_output_states.rs rename to _src/parser/states/cohorts_states/address/cohorts_output_states.rs diff --git a/src/parser/states/cohorts_states/address/cohorts_realized_states.rs b/_src/parser/states/cohorts_states/address/cohorts_realized_states.rs similarity index 100% rename from src/parser/states/cohorts_states/address/cohorts_realized_states.rs rename to _src/parser/states/cohorts_states/address/cohorts_realized_states.rs diff --git a/src/parser/states/cohorts_states/address/mod.rs b/_src/parser/states/cohorts_states/address/mod.rs similarity index 100% rename from src/parser/states/cohorts_states/address/mod.rs rename to _src/parser/states/cohorts_states/address/mod.rs diff --git a/src/parser/states/cohorts_states/address/split_by_address_cohort.rs b/_src/parser/states/cohorts_states/address/split_by_address_cohort.rs similarity index 100% rename from src/parser/states/cohorts_states/address/split_by_address_cohort.rs rename to _src/parser/states/cohorts_states/address/split_by_address_cohort.rs diff --git a/src/parser/states/cohorts_states/any/capitalization_state.rs b/_src/parser/states/cohorts_states/any/capitalization_state.rs similarity index 100% rename from src/parser/states/cohorts_states/any/capitalization_state.rs rename to _src/parser/states/cohorts_states/any/capitalization_state.rs diff --git a/src/parser/states/cohorts_states/any/durable_states.rs b/_src/parser/states/cohorts_states/any/durable_states.rs similarity index 100% rename from src/parser/states/cohorts_states/any/durable_states.rs rename to _src/parser/states/cohorts_states/any/durable_states.rs diff --git a/src/parser/states/cohorts_states/any/input_state.rs b/_src/parser/states/cohorts_states/any/input_state.rs similarity index 100% rename from src/parser/states/cohorts_states/any/input_state.rs rename to _src/parser/states/cohorts_states/any/input_state.rs diff --git a/src/parser/states/cohorts_states/any/mod.rs b/_src/parser/states/cohorts_states/any/mod.rs similarity index 100% rename from src/parser/states/cohorts_states/any/mod.rs rename to _src/parser/states/cohorts_states/any/mod.rs diff --git a/src/parser/states/cohorts_states/any/one_shot_states.rs b/_src/parser/states/cohorts_states/any/one_shot_states.rs similarity index 100% rename from src/parser/states/cohorts_states/any/one_shot_states.rs rename to _src/parser/states/cohorts_states/any/one_shot_states.rs diff --git a/src/parser/states/cohorts_states/any/output_state.rs b/_src/parser/states/cohorts_states/any/output_state.rs similarity index 100% rename from src/parser/states/cohorts_states/any/output_state.rs rename to _src/parser/states/cohorts_states/any/output_state.rs diff --git a/src/parser/states/cohorts_states/any/price_paid_state.rs b/_src/parser/states/cohorts_states/any/price_paid_state.rs similarity index 100% rename from src/parser/states/cohorts_states/any/price_paid_state.rs rename to _src/parser/states/cohorts_states/any/price_paid_state.rs diff --git a/src/parser/states/cohorts_states/any/price_to_value.rs b/_src/parser/states/cohorts_states/any/price_to_value.rs similarity index 100% rename from src/parser/states/cohorts_states/any/price_to_value.rs rename to _src/parser/states/cohorts_states/any/price_to_value.rs diff --git a/src/parser/states/cohorts_states/any/realized_state.rs b/_src/parser/states/cohorts_states/any/realized_state.rs similarity index 100% rename from src/parser/states/cohorts_states/any/realized_state.rs rename to _src/parser/states/cohorts_states/any/realized_state.rs diff --git a/src/parser/states/cohorts_states/any/supply_state.rs b/_src/parser/states/cohorts_states/any/supply_state.rs similarity index 100% rename from src/parser/states/cohorts_states/any/supply_state.rs rename to _src/parser/states/cohorts_states/any/supply_state.rs diff --git a/src/parser/states/cohorts_states/any/unrealized_state.rs b/_src/parser/states/cohorts_states/any/unrealized_state.rs similarity index 100% rename from src/parser/states/cohorts_states/any/unrealized_state.rs rename to _src/parser/states/cohorts_states/any/unrealized_state.rs diff --git a/src/parser/states/cohorts_states/any/utxo_state.rs b/_src/parser/states/cohorts_states/any/utxo_state.rs similarity index 100% rename from src/parser/states/cohorts_states/any/utxo_state.rs rename to _src/parser/states/cohorts_states/any/utxo_state.rs diff --git a/src/parser/states/cohorts_states/mod.rs b/_src/parser/states/cohorts_states/mod.rs similarity index 100% rename from src/parser/states/cohorts_states/mod.rs rename to _src/parser/states/cohorts_states/mod.rs diff --git a/src/parser/states/cohorts_states/utxo/cohort_durable_states.rs b/_src/parser/states/cohorts_states/utxo/cohort_durable_states.rs similarity index 100% rename from src/parser/states/cohorts_states/utxo/cohort_durable_states.rs rename to _src/parser/states/cohorts_states/utxo/cohort_durable_states.rs diff --git a/src/parser/states/cohorts_states/utxo/cohort_filter.rs b/_src/parser/states/cohorts_states/utxo/cohort_filter.rs similarity index 100% rename from src/parser/states/cohorts_states/utxo/cohort_filter.rs rename to _src/parser/states/cohorts_states/utxo/cohort_filter.rs diff --git a/src/parser/states/cohorts_states/utxo/cohort_filters.rs b/_src/parser/states/cohorts_states/utxo/cohort_filters.rs similarity index 100% rename from src/parser/states/cohorts_states/utxo/cohort_filters.rs rename to _src/parser/states/cohorts_states/utxo/cohort_filters.rs diff --git a/src/parser/states/cohorts_states/utxo/cohort_id.rs b/_src/parser/states/cohorts_states/utxo/cohort_id.rs similarity index 100% rename from src/parser/states/cohorts_states/utxo/cohort_id.rs rename to _src/parser/states/cohorts_states/utxo/cohort_id.rs diff --git a/src/parser/states/cohorts_states/utxo/cohorts_durable_states.rs b/_src/parser/states/cohorts_states/utxo/cohorts_durable_states.rs similarity index 100% rename from src/parser/states/cohorts_states/utxo/cohorts_durable_states.rs rename to _src/parser/states/cohorts_states/utxo/cohorts_durable_states.rs diff --git a/src/parser/states/cohorts_states/utxo/cohorts_one_shot_states.rs b/_src/parser/states/cohorts_states/utxo/cohorts_one_shot_states.rs similarity index 100% rename from src/parser/states/cohorts_states/utxo/cohorts_one_shot_states.rs rename to _src/parser/states/cohorts_states/utxo/cohorts_one_shot_states.rs diff --git a/src/parser/states/cohorts_states/utxo/cohorts_sent_states.rs b/_src/parser/states/cohorts_states/utxo/cohorts_sent_states.rs similarity index 100% rename from src/parser/states/cohorts_states/utxo/cohorts_sent_states.rs rename to _src/parser/states/cohorts_states/utxo/cohorts_sent_states.rs diff --git a/src/parser/states/cohorts_states/utxo/mod.rs b/_src/parser/states/cohorts_states/utxo/mod.rs similarity index 100% rename from src/parser/states/cohorts_states/utxo/mod.rs rename to _src/parser/states/cohorts_states/utxo/mod.rs diff --git a/src/parser/states/cohorts_states/utxo/split_by_utxo_cohort.rs b/_src/parser/states/cohorts_states/utxo/split_by_utxo_cohort.rs similarity index 100% rename from src/parser/states/cohorts_states/utxo/split_by_utxo_cohort.rs rename to _src/parser/states/cohorts_states/utxo/split_by_utxo_cohort.rs diff --git a/src/parser/states/counters.rs b/_src/parser/states/counters.rs similarity index 100% rename from src/parser/states/counters.rs rename to _src/parser/states/counters.rs diff --git a/src/parser/states/date_data_vec.rs b/_src/parser/states/date_data_vec.rs similarity index 100% rename from src/parser/states/date_data_vec.rs rename to _src/parser/states/date_data_vec.rs diff --git a/src/parser/states/mod.rs b/_src/parser/states/mod.rs similarity index 100% rename from src/parser/states/mod.rs rename to _src/parser/states/mod.rs diff --git a/src/structs/address.rs b/_src/structs/address.rs similarity index 99% rename from src/structs/address.rs rename to _src/structs/address.rs index d202e63d3..da6dc5d4a 100644 --- a/src/structs/address.rs +++ b/_src/structs/address.rs @@ -1,5 +1,5 @@ use bitcoin_hashes::hash160; -use iterator::bitcoin::TxOut; +use brk_parser::bitcoin::TxOut; use super::{AddressType, Counter, U8x19, U8x31}; diff --git a/src/structs/address_data.rs b/_src/structs/address_data.rs similarity index 100% rename from src/structs/address_data.rs rename to _src/structs/address_data.rs diff --git a/src/structs/address_liquidity.rs b/_src/structs/address_liquidity.rs similarity index 100% rename from src/structs/address_liquidity.rs rename to _src/structs/address_liquidity.rs diff --git a/src/structs/address_realized_data.rs b/_src/structs/address_realized_data.rs similarity index 100% rename from src/structs/address_realized_data.rs rename to _src/structs/address_realized_data.rs diff --git a/src/structs/address_size.rs b/_src/structs/address_size.rs similarity index 100% rename from src/structs/address_size.rs rename to _src/structs/address_size.rs diff --git a/src/structs/address_split.rs b/_src/structs/address_split.rs similarity index 100% rename from src/structs/address_split.rs rename to _src/structs/address_split.rs diff --git a/src/structs/address_type.rs b/_src/structs/address_type.rs similarity index 100% rename from src/structs/address_type.rs rename to _src/structs/address_type.rs diff --git a/src/structs/amount.rs b/_src/structs/amount.rs similarity index 96% rename from src/structs/amount.rs rename to _src/structs/amount.rs index f0d6a48db..5a29adf42 100644 --- a/src/structs/amount.rs +++ b/_src/structs/amount.rs @@ -5,15 +5,15 @@ use std::{ use allocative::{Allocative, Visitor}; use bincode::{ + BorrowDecode, Decode, Encode, de::{BorrowDecoder, Decoder}, enc::Encoder, error::{DecodeError, EncodeError}, - BorrowDecode, Decode, Encode, }; +use brk_parser::bitcoin::Amount as BitcoinAmount; use derive_deref::{Deref, DerefMut}; -use iterator::bitcoin::Amount as BitcoinAmount; use serde::{Deserialize, Serialize}; -use snkrj::{direct_repr, Storable, UnsizedStorable}; +use snkrj::{Storable, UnsizedStorable, direct_repr}; use super::Height; diff --git a/src/structs/any_map.rs b/_src/structs/any_map.rs similarity index 100% rename from src/structs/any_map.rs rename to _src/structs/any_map.rs diff --git a/src/structs/array.rs b/_src/structs/array.rs similarity index 100% rename from src/structs/array.rs rename to _src/structs/array.rs diff --git a/src/structs/bi_map.rs b/_src/structs/bi_map.rs similarity index 100% rename from src/structs/bi_map.rs rename to _src/structs/bi_map.rs diff --git a/src/structs/block_data.rs b/_src/structs/block_data.rs similarity index 100% rename from src/structs/block_data.rs rename to _src/structs/block_data.rs diff --git a/src/structs/block_path.rs b/_src/structs/block_path.rs similarity index 100% rename from src/structs/block_path.rs rename to _src/structs/block_path.rs diff --git a/src/structs/config.rs b/_src/structs/config.rs similarity index 99% rename from src/structs/config.rs rename to _src/structs/config.rs index 9b84ae495..683ec1181 100644 --- a/src/structs/config.rs +++ b/_src/structs/config.rs @@ -4,9 +4,9 @@ use std::{ path::{Path, PathBuf}, }; +use brk_parser::bitcoincore_rpc::Auth; use clap::Parser; use color_eyre::eyre::eyre; -use iterator::bitcoincore_rpc::Auth; use log::info; use serde::{Deserialize, Serialize}; diff --git a/src/structs/counter.rs b/_src/structs/counter.rs similarity index 100% rename from src/structs/counter.rs rename to _src/structs/counter.rs diff --git a/src/structs/date.rs b/_src/structs/date.rs similarity index 100% rename from src/structs/date.rs rename to _src/structs/date.rs diff --git a/src/structs/date_data.rs b/_src/structs/date_data.rs similarity index 100% rename from src/structs/date_data.rs rename to _src/structs/date_data.rs diff --git a/src/structs/date_map.rs b/_src/structs/date_map.rs similarity index 100% rename from src/structs/date_map.rs rename to _src/structs/date_map.rs diff --git a/src/structs/date_map_chunk_id.rs b/_src/structs/date_map_chunk_id.rs similarity index 100% rename from src/structs/date_map_chunk_id.rs rename to _src/structs/date_map_chunk_id.rs diff --git a/src/structs/empty_address_data.rs b/_src/structs/empty_address_data.rs similarity index 100% rename from src/structs/empty_address_data.rs rename to _src/structs/empty_address_data.rs diff --git a/src/structs/epoch.rs b/_src/structs/epoch.rs similarity index 100% rename from src/structs/epoch.rs rename to _src/structs/epoch.rs diff --git a/src/structs/exit.rs b/_src/structs/exit.rs similarity index 100% rename from src/structs/exit.rs rename to _src/structs/exit.rs diff --git a/src/structs/generic_map.rs b/_src/structs/generic_map.rs similarity index 100% rename from src/structs/generic_map.rs rename to _src/structs/generic_map.rs diff --git a/src/structs/height.rs b/_src/structs/height.rs similarity index 96% rename from src/structs/height.rs rename to _src/structs/height.rs index efdc1dc18..c788e6afb 100644 --- a/src/structs/height.rs +++ b/_src/structs/height.rs @@ -5,11 +5,11 @@ use std::{ use allocative::Allocative; use bincode::{Decode, Encode}; +use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; use derive_deref::{Deref, DerefMut}; -use iterator::NUMBER_OF_UNSAFE_BLOCKS; use serde::{Deserialize, Serialize}; -use super::{HeightMapChunkId, MapKey, HEIGHT_MAP_CHUNK_SIZE}; +use super::{HEIGHT_MAP_CHUNK_SIZE, HeightMapChunkId, MapKey}; #[derive( Debug, diff --git a/src/structs/height_map.rs b/_src/structs/height_map.rs similarity index 100% rename from src/structs/height_map.rs rename to _src/structs/height_map.rs diff --git a/src/structs/height_map_chunk_id.rs b/_src/structs/height_map_chunk_id.rs similarity index 100% rename from src/structs/height_map_chunk_id.rs rename to _src/structs/height_map_chunk_id.rs diff --git a/src/structs/instant.rs b/_src/structs/instant.rs similarity index 100% rename from src/structs/instant.rs rename to _src/structs/instant.rs diff --git a/src/structs/liquidity.rs b/_src/structs/liquidity.rs similarity index 100% rename from src/structs/liquidity.rs rename to _src/structs/liquidity.rs diff --git a/src/structs/map_path.rs b/_src/structs/map_path.rs similarity index 100% rename from src/structs/map_path.rs rename to _src/structs/map_path.rs diff --git a/src/structs/map_value.rs b/_src/structs/map_value.rs similarity index 100% rename from src/structs/map_value.rs rename to _src/structs/map_value.rs diff --git a/src/structs/mod.rs b/_src/structs/mod.rs similarity index 100% rename from src/structs/mod.rs rename to _src/structs/mod.rs diff --git a/src/structs/ohlc.rs b/_src/structs/ohlc.rs similarity index 100% rename from src/structs/ohlc.rs rename to _src/structs/ohlc.rs diff --git a/src/structs/partial_txout_data.rs b/_src/structs/partial_txout_data.rs similarity index 100% rename from src/structs/partial_txout_data.rs rename to _src/structs/partial_txout_data.rs diff --git a/src/structs/price.rs b/_src/structs/price.rs similarity index 100% rename from src/structs/price.rs rename to _src/structs/price.rs diff --git a/src/structs/ram.rs b/_src/structs/ram.rs similarity index 100% rename from src/structs/ram.rs rename to _src/structs/ram.rs diff --git a/src/structs/rpc.rs b/_src/structs/rpc.rs similarity index 90% rename from src/structs/rpc.rs rename to _src/structs/rpc.rs index 20ce2db9a..885c6bb71 100644 --- a/src/structs/rpc.rs +++ b/_src/structs/rpc.rs @@ -1,4 +1,4 @@ -use iterator::bitcoincore_rpc::Client; +use brk_parser::bitcoincore_rpc::Client; use crate::structs::Config; diff --git a/src/structs/sent_data.rs b/_src/structs/sent_data.rs similarity index 100% rename from src/structs/sent_data.rs rename to _src/structs/sent_data.rs diff --git a/src/structs/serialized_btreemap.rs b/_src/structs/serialized_btreemap.rs similarity index 100% rename from src/structs/serialized_btreemap.rs rename to _src/structs/serialized_btreemap.rs diff --git a/src/structs/serialized_vec.rs b/_src/structs/serialized_vec.rs similarity index 100% rename from src/structs/serialized_vec.rs rename to _src/structs/serialized_vec.rs diff --git a/src/structs/timestamp.rs b/_src/structs/timestamp.rs similarity index 100% rename from src/structs/timestamp.rs rename to _src/structs/timestamp.rs diff --git a/src/structs/tx_data.rs b/_src/structs/tx_data.rs similarity index 100% rename from src/structs/tx_data.rs rename to _src/structs/tx_data.rs diff --git a/src/structs/txout_index.rs b/_src/structs/txout_index.rs similarity index 100% rename from src/structs/txout_index.rs rename to _src/structs/txout_index.rs diff --git a/src/utils/consts.rs b/_src/utils/consts.rs similarity index 100% rename from src/utils/consts.rs rename to _src/utils/consts.rs diff --git a/src/utils/flamegraph.rs b/_src/utils/flamegraph.rs similarity index 100% rename from src/utils/flamegraph.rs rename to _src/utils/flamegraph.rs diff --git a/src/utils/log.rs b/_src/utils/log.rs similarity index 100% rename from src/utils/log.rs rename to _src/utils/log.rs diff --git a/src/utils/lossy.rs b/_src/utils/lossy.rs similarity index 100% rename from src/utils/lossy.rs rename to _src/utils/lossy.rs diff --git a/src/utils/mod.rs b/_src/utils/mod.rs similarity index 100% rename from src/utils/mod.rs rename to _src/utils/mod.rs diff --git a/src/utils/percentile.rs b/_src/utils/percentile.rs similarity index 100% rename from src/utils/percentile.rs rename to _src/utils/percentile.rs diff --git a/src/utils/retry.rs b/_src/utils/retry.rs similarity index 100% rename from src/utils/retry.rs rename to _src/utils/retry.rs diff --git a/src/utils/time.rs b/_src/utils/time.rs similarity index 100% rename from src/utils/time.rs rename to _src/utils/time.rs diff --git a/cli/Cargo.toml b/crates/brk_cli/Cargo.toml similarity index 90% rename from cli/Cargo.toml rename to crates/brk_cli/Cargo.toml index 121e9d824..4e38c18b1 100644 --- a/cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "bli" +name = "brk_cli" description = "A command line interface to run berver" version = "0.1.0" edition = { workspace = true } diff --git a/cli/src/main.rs b/crates/brk_cli/src/main.rs similarity index 100% rename from cli/src/main.rs rename to crates/brk_cli/src/main.rs diff --git a/computer/Cargo.toml b/crates/brk_computer/Cargo.toml similarity index 58% rename from computer/Cargo.toml rename to crates/brk_computer/Cargo.toml index 8c370f919..760fe6fbf 100644 --- a/computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -1,17 +1,17 @@ [package] -name = "bomputer" -description = "A Bitcoin dataset computer built on top of bindexer and bricer" +name = "brk_computer" +description = "A Bitcoin dataset computer built on top of brk_indexer and brk_fetcher" version = "0.1.0" edition = { workspace = true } license = { workspace = true } [dependencies] +brk_fetcher = { workspace = true } +brk_indexer = { workspace = true } +brk_parser = { workspace = true } color-eyre = { workspace = true } derive_deref = { workspace = true } fjall = { workspace = true } hodor = { workspace = true } -indexer = { workspace = true } -iterator = { workspace = true } -pricer = { workspace = true } storable_vec = { workspace = true } zerocopy = { workspace = true } diff --git a/computer/src/lib.rs b/crates/brk_computer/src/lib.rs similarity index 97% rename from computer/src/lib.rs rename to crates/brk_computer/src/lib.rs index 2eb57481d..5ae619db4 100644 --- a/computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -1,13 +1,13 @@ use std::path::{Path, PathBuf}; +use brk_indexer::Indexer; +pub use brk_parser::rpc; use hodor::Exit; -use indexer::Indexer; -pub use iterator::rpc; mod storage; mod structs; -use pricer::Date; +use brk_fetcher::Date; use storable_vec::SINGLE_THREAD; use storage::{Fjalls, StorableVecs}; pub use structs::*; diff --git a/computer/src/main.rs b/crates/brk_computer/src/main.rs similarity index 88% rename from computer/src/main.rs rename to crates/brk_computer/src/main.rs index ad810775b..acf96bc03 100644 --- a/computer/src/main.rs +++ b/crates/brk_computer/src/main.rs @@ -1,8 +1,8 @@ use std::path::Path; -use bomputer::Computer; +use brk_computer::Computer; +use brk_indexer::Indexer; use hodor::Exit; -use indexer::Indexer; mod structs; diff --git a/computer/src/storage/fjalls.rs b/crates/brk_computer/src/storage/fjalls.rs similarity index 96% rename from computer/src/storage/fjalls.rs rename to crates/brk_computer/src/storage/fjalls.rs index 304ab0bc5..38221d0be 100644 --- a/computer/src/storage/fjalls.rs +++ b/crates/brk_computer/src/storage/fjalls.rs @@ -1,6 +1,6 @@ use std::path::Path; -use indexer::Store; +use brk_indexer::Store; use storable_vec::Version; use crate::structs::{AddressindexTxoutindex, Unit}; diff --git a/computer/src/storage/mod.rs b/crates/brk_computer/src/storage/mod.rs similarity index 100% rename from computer/src/storage/mod.rs rename to crates/brk_computer/src/storage/mod.rs diff --git a/computer/src/storage/storable_vecs.rs b/crates/brk_computer/src/storage/storable_vecs.rs similarity index 97% rename from computer/src/storage/storable_vecs.rs rename to crates/brk_computer/src/storage/storable_vecs.rs index 73a1b10c4..a39ec0de4 100644 --- a/computer/src/storage/storable_vecs.rs +++ b/crates/brk_computer/src/storage/storable_vecs.rs @@ -1,7 +1,7 @@ use std::{fs, path::Path}; -use indexer::{Addressindex, Height, Sats, Timestamp, Txindex, Txinindex, Txoutindex}; -use pricer::{Date, Dateindex}; +use brk_fetcher::{Date, Dateindex}; +use brk_indexer::{Addressindex, Height, Sats, Timestamp, Txindex, Txinindex, Txoutindex}; use storable_vec::{StorableVec, Version}; use crate::structs::Feerate; diff --git a/computer/src/structs/addressindextxoutindex.rs b/crates/brk_computer/src/structs/addressindextxoutindex.rs similarity index 93% rename from computer/src/structs/addressindextxoutindex.rs rename to crates/brk_computer/src/structs/addressindextxoutindex.rs index 3902528e7..e08cbf8b4 100644 --- a/computer/src/structs/addressindextxoutindex.rs +++ b/crates/brk_computer/src/structs/addressindextxoutindex.rs @@ -1,5 +1,5 @@ +use brk_indexer::{Addressindex, Txoutindex}; use fjall::Slice; -use indexer::{Addressindex, Txoutindex}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Immutable, IntoBytes, KnownLayout, FromBytes)] diff --git a/computer/src/structs/bitcoin.rs b/crates/brk_computer/src/structs/bitcoin.rs similarity index 91% rename from computer/src/structs/bitcoin.rs rename to crates/brk_computer/src/structs/bitcoin.rs index 28f4621d5..36deb7800 100644 --- a/computer/src/structs/bitcoin.rs +++ b/crates/brk_computer/src/structs/bitcoin.rs @@ -1,4 +1,4 @@ -use indexer::Sats; +use brk_indexer::Sats; #[derive(Debug, Default, Clone, Copy)] pub struct Bitcoin(f64); diff --git a/computer/src/structs/feerate.rs b/crates/brk_computer/src/structs/feerate.rs similarity index 100% rename from computer/src/structs/feerate.rs rename to crates/brk_computer/src/structs/feerate.rs diff --git a/computer/src/structs/mod.rs b/crates/brk_computer/src/structs/mod.rs similarity index 100% rename from computer/src/structs/mod.rs rename to crates/brk_computer/src/structs/mod.rs diff --git a/computer/src/structs/ohlc.rs b/crates/brk_computer/src/structs/ohlc.rs similarity index 100% rename from computer/src/structs/ohlc.rs rename to crates/brk_computer/src/structs/ohlc.rs diff --git a/computer/src/structs/unit.rs b/crates/brk_computer/src/structs/unit.rs similarity index 100% rename from computer/src/structs/unit.rs rename to crates/brk_computer/src/structs/unit.rs diff --git a/pricer/Cargo.toml b/crates/brk_fetcher/Cargo.toml similarity index 66% rename from pricer/Cargo.toml rename to crates/brk_fetcher/Cargo.toml index 6eb27ee7c..33a9595e8 100644 --- a/pricer/Cargo.toml +++ b/crates/brk_fetcher/Cargo.toml @@ -1,16 +1,17 @@ [package] -name = "bricer" -description = "A bitcoin price fetcher built on top of bindexer" +name = "brk_fetcher" +description = "A bitcoin price fetcher built on top of brk_indexer" version = "0.1.0" edition = { workspace = true } license = { workspace = true } [dependencies] +brk_indexer = { workspace = true } +brk_printer = { workspace = true } color-eyre = { workspace = true } derive_deref = { workspace = true } -indexer = { workspace = true } jiff = { workspace = true } -logger = { workspace = true } +log = { workspace = true } minreq = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/pricer/src/fetchers/binance.rs b/crates/brk_fetcher/src/fetchers/binance.rs similarity index 98% rename from pricer/src/fetchers/binance.rs rename to crates/brk_fetcher/src/fetchers/binance.rs index 265462ca6..6bd444b60 100644 --- a/pricer/src/fetchers/binance.rs +++ b/crates/brk_fetcher/src/fetchers/binance.rs @@ -6,16 +6,16 @@ use std::{ str::FromStr, }; -use color_eyre::eyre::{eyre, ContextCompat}; -use indexer::Timestamp; -use logger::info; +use brk_indexer::Timestamp; +use color_eyre::eyre::{ContextCompat, eyre}; +use log::info; use serde_json::Value; use storable_vec::STATELESS; use crate::{ + Close, Date, Dollars, High, Low, Open, Pricer, fetchers::retry, structs::{Cents, OHLC}, - Close, Date, Dollars, High, Low, Open, Pricer, }; pub struct Binance { diff --git a/pricer/src/fetchers/kibo.rs b/crates/brk_fetcher/src/fetchers/kibo.rs similarity index 99% rename from pricer/src/fetchers/kibo.rs rename to crates/brk_fetcher/src/fetchers/kibo.rs index 847ab32a4..b5327375b 100644 --- a/pricer/src/fetchers/kibo.rs +++ b/crates/brk_fetcher/src/fetchers/kibo.rs @@ -1,14 +1,14 @@ use std::{collections::BTreeMap, str::FromStr}; +use brk_indexer::Height; use color_eyre::eyre::ContextCompat; -use indexer::Height; -use logger::info; +use log::info; use serde_json::Value; use crate::{ + Cents, Close, Dollars, High, Low, Open, fetchers::retry, structs::{Date, OHLC}, - Cents, Close, Dollars, High, Low, Open, }; #[derive(Default)] diff --git a/pricer/src/fetchers/kraken.rs b/crates/brk_fetcher/src/fetchers/kraken.rs similarity index 96% rename from pricer/src/fetchers/kraken.rs rename to crates/brk_fetcher/src/fetchers/kraken.rs index 6de067b6c..49618fcee 100644 --- a/pricer/src/fetchers/kraken.rs +++ b/crates/brk_fetcher/src/fetchers/kraken.rs @@ -1,12 +1,12 @@ use std::collections::BTreeMap; +use brk_indexer::Timestamp; use color_eyre::eyre::ContextCompat; -use indexer::Timestamp; -use logger::info; +use log::info; use serde_json::Value; use storable_vec::STATELESS; -use crate::{fetchers::retry, structs::Date, Cents, Close, Dollars, High, Low, Open, Pricer, OHLC}; +use crate::{Cents, Close, Dollars, High, Low, OHLC, Open, Pricer, fetchers::retry, structs::Date}; #[derive(Default)] pub struct Kraken { diff --git a/pricer/src/fetchers/mod.rs b/crates/brk_fetcher/src/fetchers/mod.rs similarity index 100% rename from pricer/src/fetchers/mod.rs rename to crates/brk_fetcher/src/fetchers/mod.rs diff --git a/pricer/src/fetchers/retry.rs b/crates/brk_fetcher/src/fetchers/retry.rs similarity index 96% rename from pricer/src/fetchers/retry.rs rename to crates/brk_fetcher/src/fetchers/retry.rs index 555e4a31c..3e1ab4675 100644 --- a/pricer/src/fetchers/retry.rs +++ b/crates/brk_fetcher/src/fetchers/retry.rs @@ -1,6 +1,6 @@ use std::{thread::sleep, time::Duration}; -use logger::info; +use log::info; pub fn retry( function: impl Fn(usize) -> color_eyre::Result, diff --git a/pricer/src/lib.rs b/crates/brk_fetcher/src/lib.rs similarity index 98% rename from pricer/src/lib.rs rename to crates/brk_fetcher/src/lib.rs index 60a2fcc63..7de87c79c 100644 --- a/pricer/src/lib.rs +++ b/crates/brk_fetcher/src/lib.rs @@ -9,9 +9,9 @@ use color_eyre::eyre::Error; mod fetchers; mod structs; +use brk_indexer::{Height, Indexer, Timestamp}; pub use fetchers::*; -use indexer::{Height, Indexer, Timestamp}; -use storable_vec::{AnyJsonStorableVec, AnyStorableVec, StorableVec, Version, SINGLE_THREAD}; +use storable_vec::{AnyJsonStorableVec, AnyStorableVec, SINGLE_THREAD, StorableVec, Version}; pub use structs::*; pub struct Pricer { diff --git a/pricer/src/main.rs b/crates/brk_fetcher/src/main.rs similarity index 78% rename from pricer/src/main.rs rename to crates/brk_fetcher/src/main.rs index ca945e8da..2c057cdae 100644 --- a/pricer/src/main.rs +++ b/crates/brk_fetcher/src/main.rs @@ -1,11 +1,11 @@ -use bricer::{Binance, Kibo, Kraken}; -use indexer::Height; +use brk_fetcher::{Binance, Kibo, Kraken}; +use brk_indexer::Height; use serde_json::Value; fn main() -> color_eyre::Result<()> { color_eyre::install()?; - logger::init_log(None); + brk_printer::init_log(None); dbg!(Binance::fetch_1d()?); // dbg!(Binance::fetch_1mn_prices()); diff --git a/pricer/src/structs/cents.rs b/crates/brk_fetcher/src/structs/cents.rs similarity index 100% rename from pricer/src/structs/cents.rs rename to crates/brk_fetcher/src/structs/cents.rs diff --git a/pricer/src/structs/close.rs b/crates/brk_fetcher/src/structs/close.rs similarity index 100% rename from pricer/src/structs/close.rs rename to crates/brk_fetcher/src/structs/close.rs diff --git a/pricer/src/structs/date.rs b/crates/brk_fetcher/src/structs/date.rs similarity index 94% rename from pricer/src/structs/date.rs rename to crates/brk_fetcher/src/structs/date.rs index 766b661c1..21e7f112d 100644 --- a/pricer/src/structs/date.rs +++ b/crates/brk_fetcher/src/structs/date.rs @@ -1,5 +1,5 @@ -use indexer::Timestamp; -use jiff::{civil::Date as Date_, tz::TimeZone, Span}; +use brk_indexer::Timestamp; +use jiff::{Span, civil::Date as Date_, tz::TimeZone}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use super::Dateindex; diff --git a/pricer/src/structs/dateindex.rs b/crates/brk_fetcher/src/structs/dateindex.rs similarity index 100% rename from pricer/src/structs/dateindex.rs rename to crates/brk_fetcher/src/structs/dateindex.rs diff --git a/pricer/src/structs/dollars.rs b/crates/brk_fetcher/src/structs/dollars.rs similarity index 100% rename from pricer/src/structs/dollars.rs rename to crates/brk_fetcher/src/structs/dollars.rs diff --git a/pricer/src/structs/high.rs b/crates/brk_fetcher/src/structs/high.rs similarity index 100% rename from pricer/src/structs/high.rs rename to crates/brk_fetcher/src/structs/high.rs diff --git a/pricer/src/structs/low.rs b/crates/brk_fetcher/src/structs/low.rs similarity index 100% rename from pricer/src/structs/low.rs rename to crates/brk_fetcher/src/structs/low.rs diff --git a/pricer/src/structs/mod.rs b/crates/brk_fetcher/src/structs/mod.rs similarity index 100% rename from pricer/src/structs/mod.rs rename to crates/brk_fetcher/src/structs/mod.rs diff --git a/pricer/src/structs/open.rs b/crates/brk_fetcher/src/structs/open.rs similarity index 100% rename from pricer/src/structs/open.rs rename to crates/brk_fetcher/src/structs/open.rs diff --git a/indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml similarity index 73% rename from indexer/Cargo.toml rename to crates/brk_indexer/Cargo.toml index 5ae90ad97..547518d74 100644 --- a/indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -1,19 +1,20 @@ [package] -name = "bindexer" -description = "A bitcoin-core indexer built on top of biter" +name = "brk_indexer" +description = "A bitcoin-core indexer built on top of brk_parser" version = "0.1.0" edition = { workspace = true } license = { workspace = true } [dependencies] bitcoin = { workspace = true } +brk_parser = { workspace = true } +brk_printer = { workspace = true } color-eyre = { workspace = true } derive_deref = { workspace = true } fjall = { workspace = true } hodor = { workspace = true } -iterator = { workspace = true } jiff = { workspace = true } -logger = { workspace = true } +log = { workspace = true } rapidhash = "1.4.0" rayon = { workspace = true } rlimit = { version = "0.10.2" } diff --git a/indexer/README.md b/crates/brk_indexer/README.md similarity index 100% rename from indexer/README.md rename to crates/brk_indexer/README.md diff --git a/indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs similarity index 98% rename from indexer/src/lib.rs rename to crates/brk_indexer/src/lib.rs index 48219356a..901226e81 100644 --- a/indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -6,12 +6,12 @@ use std::{ time::Duration, }; -pub use iterator::*; +pub use brk_parser::*; use bitcoin::{Transaction, TxIn, TxOut}; -use color_eyre::eyre::{eyre, ContextCompat}; +use color_eyre::eyre::{ContextCompat, eyre}; use hodor::Exit; -use logger::info; +use log::info; use rayon::prelude::*; use storable_vec::CACHED_GETS; @@ -78,7 +78,7 @@ impl Indexer { let mut idxs = starting_indexes; - iterator::new(bitcoin_dir, Some(idxs.height), None, rpc) + brk_parser::new(bitcoin_dir, Some(idxs.height), None, rpc) .iter() .try_for_each(|(height, block, blockhash)| -> color_eyre::Result<()> { info!("Indexing block {height}..."); @@ -598,11 +598,9 @@ impl Indexer { vecs.txindex_to_txid.push_if_needed(txindex, txid)?; vecs.txindex_to_height.push_if_needed(txindex, height)?; vecs.txindex_to_locktime.push_if_needed(txindex, tx.lock_time.into())?; - // tx.base_size() - // tx.total_size() - // tx.is_explicitly_rbf() - // tx.weight() - // tx.vsize() in computer as it can be computed from the weight + vecs.txindex_to_base_size.push_if_needed(txindex, tx.base_size())?; + vecs.txindex_to_total_size.push_if_needed(txindex, tx.total_size())?; + vecs.txindex_to_is_explicitly_rbf.push_if_needed(txindex, tx.is_explicitly_rbf())?; Ok(()) })?; diff --git a/indexer/src/main.rs b/crates/brk_indexer/src/main.rs similarity index 89% rename from indexer/src/main.rs rename to crates/brk_indexer/src/main.rs index b44c36880..fa7788c83 100644 --- a/indexer/src/main.rs +++ b/crates/brk_indexer/src/main.rs @@ -1,15 +1,15 @@ use std::{path::Path, thread::sleep, time::Duration}; -use bindexer::{rpc::RpcApi, Indexer}; +use brk_indexer::{Indexer, rpc::RpcApi}; +use brk_parser::rpc::{self}; use hodor::Exit; -use iterator::rpc; -use logger::info; +use log::info; use storable_vec::CACHED_GETS; fn main() -> color_eyre::Result<()> { color_eyre::install()?; - logger::init_log(None); + brk_printer::init_log(None); let data_dir = Path::new("../../bitcoin"); let rpc = Box::leak(Box::new(rpc::Client::new( diff --git a/indexer/src/storage/fjalls/base.rs b/crates/brk_indexer/src/storage/fjalls/base.rs similarity index 98% rename from indexer/src/storage/fjalls/base.rs rename to crates/brk_indexer/src/storage/fjalls/base.rs index 9b215b779..207a88853 100644 --- a/indexer/src/storage/fjalls/base.rs +++ b/crates/brk_indexer/src/storage/fjalls/base.rs @@ -4,11 +4,11 @@ use std::{ path::Path, }; +use brk_parser::Height; use fjall::{ PartitionCreateOptions, PersistMode, ReadTransaction, Result, Slice, TransactionalKeyspace, TransactionalPartitionHandle, }; -use iterator::Height; use storable_vec::{Value, Version}; use zerocopy::{Immutable, IntoBytes}; @@ -102,7 +102,7 @@ where mem::take(&mut self.puts).into_iter().for_each(|(key, value)| { if CHECK_COLLISISONS { if let Ok(Some(value)) = wtx.get(&self.part, key.as_bytes()) { - dbg!(value); + dbg!(value, &self.meta); unreachable!(); } } diff --git a/indexer/src/storage/fjalls/meta.rs b/crates/brk_indexer/src/storage/fjalls/meta.rs similarity index 99% rename from indexer/src/storage/fjalls/meta.rs rename to crates/brk_indexer/src/storage/fjalls/meta.rs index b9a79c38f..5c0bfdf56 100644 --- a/indexer/src/storage/fjalls/meta.rs +++ b/crates/brk_indexer/src/storage/fjalls/meta.rs @@ -8,6 +8,7 @@ use zerocopy::{FromBytes, IntoBytes}; use super::Height; +#[derive(Debug)] pub struct StoreMeta { pathbuf: PathBuf, version: Version, diff --git a/indexer/src/storage/fjalls/mod.rs b/crates/brk_indexer/src/storage/fjalls/mod.rs similarity index 100% rename from indexer/src/storage/fjalls/mod.rs rename to crates/brk_indexer/src/storage/fjalls/mod.rs diff --git a/indexer/src/storage/fjalls/version.rs b/crates/brk_indexer/src/storage/fjalls/version.rs similarity index 100% rename from indexer/src/storage/fjalls/version.rs rename to crates/brk_indexer/src/storage/fjalls/version.rs diff --git a/indexer/src/storage/mod.rs b/crates/brk_indexer/src/storage/mod.rs similarity index 100% rename from indexer/src/storage/mod.rs rename to crates/brk_indexer/src/storage/mod.rs diff --git a/indexer/src/storage/storable_vecs/base.rs b/crates/brk_indexer/src/storage/storable_vecs/base.rs similarity index 93% rename from indexer/src/storage/storable_vecs/base.rs rename to crates/brk_indexer/src/storage/storable_vecs/base.rs index c02b6cd7e..f762bb2b5 100644 --- a/indexer/src/storage/storable_vecs/base.rs +++ b/crates/brk_indexer/src/storage/storable_vecs/base.rs @@ -39,7 +39,7 @@ where self.vec.truncate_if_needed(index) } - pub fn height(&self) -> iterator::Result { + pub fn height(&self) -> brk_parser::Result { Height::try_from(self.path_height().as_path()) } fn path_height(&self) -> PathBuf { @@ -71,7 +71,7 @@ impl DerefMut for StorableVec { } pub trait AnyStorableVec: Send + Sync { - fn height(&self) -> iterator::Result; + fn height(&self) -> brk_parser::Result; fn flush(&mut self, height: Height) -> io::Result<()>; } @@ -80,7 +80,7 @@ where I: StoredIndex, T: StoredType, { - fn height(&self) -> iterator::Result { + fn height(&self) -> brk_parser::Result { self.height() } diff --git a/indexer/src/storage/storable_vecs/mod.rs b/crates/brk_indexer/src/storage/storable_vecs/mod.rs similarity index 94% rename from indexer/src/storage/storable_vecs/mod.rs rename to crates/brk_indexer/src/storage/storable_vecs/mod.rs index 572303bc6..481f92af2 100644 --- a/indexer/src/storage/storable_vecs/mod.rs +++ b/crates/brk_indexer/src/storage/storable_vecs/mod.rs @@ -1,10 +1,11 @@ use std::{fs, io, path::Path}; -use iterator::Height; +use brk_parser::Height; use rayon::prelude::*; -use storable_vec::{AnyJsonStorableVec, Version, CACHED_GETS}; +use storable_vec::{AnyJsonStorableVec, CACHED_GETS, Version}; use crate::{ + Indexes, structs::{ Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, LockTime, Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes, P2PK65index, P2PKHAddressBytes, P2PKHindex, @@ -12,7 +13,6 @@ use crate::{ P2WSHindex, Pushonlyindex, Sats, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight, }, - Indexes, }; mod base; @@ -56,6 +56,9 @@ pub struct StorableVecs { pub txindex_to_height: StorableVec, pub txindex_to_locktime: StorableVec, pub txindex_to_txid: StorableVec, + pub txindex_to_base_size: StorableVec, + pub txindex_to_total_size: StorableVec, + pub txindex_to_is_explicitly_rbf: StorableVec, pub txindex_to_txversion: StorableVec, pub txinindex_to_txoutindex: StorableVec, pub txoutindex_to_addressindex: StorableVec, @@ -172,6 +175,12 @@ impl StorableVecs { txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?, txindex_to_locktime: StorableVec::import(&path.join("txindex_to_locktime"), Version::from(1))?, txindex_to_txid: StorableVec::import(&path.join("txindex_to_txid"), Version::from(1))?, + txindex_to_base_size: StorableVec::import(&path.join("txindex_to_base_size"), Version::from(1))?, + txindex_to_total_size: StorableVec::import(&path.join("txindex_to_total_size"), Version::from(1))?, + txindex_to_is_explicitly_rbf: StorableVec::import( + &path.join("txindex_to_is_explicitly_rbf"), + Version::from(1), + )?, txindex_to_txversion: StorableVec::import(&path.join("txindex_to_txversion"), Version::from(1))?, txinindex_to_txoutindex: StorableVec::import(&path.join("txinindex_to_txoutindex"), Version::from(1))?, txoutindex_to_addressindex: StorableVec::import( @@ -273,6 +282,10 @@ impl StorableVecs { self.txindex_to_locktime.truncate_if_needed(txindex, saved_height)?; self.txindex_to_txid.truncate_if_needed(txindex, saved_height)?; self.txindex_to_txversion.truncate_if_needed(txindex, saved_height)?; + self.txindex_to_base_size.truncate_if_needed(txindex, saved_height)?; + self.txindex_to_total_size.truncate_if_needed(txindex, saved_height)?; + self.txindex_to_is_explicitly_rbf + .truncate_if_needed(txindex, saved_height)?; self.txinindex_to_txoutindex .truncate_if_needed(txinindex, saved_height)?; @@ -298,7 +311,7 @@ impl StorableVecs { .unwrap() } - pub fn as_any_json_vec_slice(&self) -> [&dyn AnyJsonStorableVec; 40] { + pub fn as_any_json_vec_slice(&self) -> [&dyn AnyJsonStorableVec; 43] { [ &*self.addressindex_to_addresstype as &dyn AnyJsonStorableVec, &*self.addressindex_to_addresstypeindex, @@ -336,6 +349,9 @@ impl StorableVecs { &*self.txindex_to_height, &*self.txindex_to_locktime, &*self.txindex_to_txid, + &*self.txindex_to_base_size, + &*self.txindex_to_total_size, + &*self.txindex_to_is_explicitly_rbf, &*self.txindex_to_txversion, &*self.txinindex_to_txoutindex, &*self.txoutindex_to_addressindex, @@ -343,7 +359,7 @@ impl StorableVecs { ] } - pub fn as_mut_any_vec_slice(&mut self) -> [&mut dyn AnyStorableVec; 40] { + pub fn as_mut_any_vec_slice(&mut self) -> [&mut dyn AnyStorableVec; 43] { [ &mut self.addressindex_to_addresstype as &mut dyn AnyStorableVec, &mut self.addressindex_to_addresstypeindex, @@ -381,6 +397,9 @@ impl StorableVecs { &mut self.txindex_to_height, &mut self.txindex_to_locktime, &mut self.txindex_to_txid, + &mut self.txindex_to_base_size, + &mut self.txindex_to_total_size, + &mut self.txindex_to_is_explicitly_rbf, &mut self.txindex_to_txversion, &mut self.txinindex_to_txoutindex, &mut self.txoutindex_to_addressindex, diff --git a/indexer/src/structs/addressbytes.rs b/crates/brk_indexer/src/structs/addressbytes.rs similarity index 99% rename from indexer/src/structs/addressbytes.rs rename to crates/brk_indexer/src/structs/addressbytes.rs index 70940b46d..2d679a3cd 100644 --- a/indexer/src/structs/addressbytes.rs +++ b/crates/brk_indexer/src/structs/addressbytes.rs @@ -1,6 +1,6 @@ +use brk_parser::bitcoin::ScriptBuf; use color_eyre::eyre::eyre; use derive_deref::{Deref, DerefMut}; -use iterator::bitcoin::ScriptBuf; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/indexer/src/structs/addressindex.rs b/crates/brk_indexer/src/structs/addressindex.rs similarity index 100% rename from indexer/src/structs/addressindex.rs rename to crates/brk_indexer/src/structs/addressindex.rs diff --git a/indexer/src/structs/addresstype.rs b/crates/brk_indexer/src/structs/addresstype.rs similarity index 97% rename from indexer/src/structs/addresstype.rs rename to crates/brk_indexer/src/structs/addresstype.rs index a1da2b4df..94a3e06a8 100644 --- a/indexer/src/structs/addresstype.rs +++ b/crates/brk_indexer/src/structs/addresstype.rs @@ -1,4 +1,4 @@ -use iterator::bitcoin::ScriptBuf; +use brk_parser::bitcoin::ScriptBuf; use serde::Serialize; use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; diff --git a/indexer/src/structs/addresstypeindex.rs b/crates/brk_indexer/src/structs/addresstypeindex.rs similarity index 100% rename from indexer/src/structs/addresstypeindex.rs rename to crates/brk_indexer/src/structs/addresstypeindex.rs diff --git a/indexer/src/structs/blockhash.rs b/crates/brk_indexer/src/structs/blockhash.rs similarity index 93% rename from indexer/src/structs/blockhash.rs rename to crates/brk_indexer/src/structs/blockhash.rs index f8da62225..4fd59e391 100644 --- a/indexer/src/structs/blockhash.rs +++ b/crates/brk_indexer/src/structs/blockhash.rs @@ -1,10 +1,10 @@ use std::mem; -use derive_deref::Deref; -use iterator::{ - rpc::{Client, RpcApi}, +use brk_parser::{ Height, + rpc::{Client, RpcApi}, }; +use derive_deref::Deref; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -24,7 +24,7 @@ impl From for bitcoin::BlockHash { } impl TryFrom<(&Client, Height)> for BlockHash { - type Error = iterator::rpc::Error; + type Error = brk_parser::rpc::Error; fn try_from((rpc, height): (&Client, Height)) -> Result { Ok(Self::from(rpc.get_block_hash(u64::from(height))?)) } diff --git a/indexer/src/structs/compressed.rs b/crates/brk_indexer/src/structs/compressed.rs similarity index 100% rename from indexer/src/structs/compressed.rs rename to crates/brk_indexer/src/structs/compressed.rs diff --git a/indexer/src/structs/indexes.rs b/crates/brk_indexer/src/structs/indexes.rs similarity index 98% rename from indexer/src/structs/indexes.rs rename to crates/brk_indexer/src/structs/indexes.rs index 02539dc06..5339b559d 100644 --- a/indexer/src/structs/indexes.rs +++ b/crates/brk_indexer/src/structs/indexes.rs @@ -1,6 +1,6 @@ +use brk_parser::NUMBER_OF_UNSAFE_BLOCKS; +use brk_parser::{Height, rpc::Client}; use color_eyre::eyre::ContextCompat; -use iterator::NUMBER_OF_UNSAFE_BLOCKS; -use iterator::{rpc::Client, Height}; use storable_vec::CACHED_GETS; use crate::storage::{Fjalls, StorableVecs}; diff --git a/indexer/src/structs/locktime.rs b/crates/brk_indexer/src/structs/locktime.rs similarity index 97% rename from indexer/src/structs/locktime.rs rename to crates/brk_indexer/src/structs/locktime.rs index c0ba0826f..ceaac5b46 100644 --- a/indexer/src/structs/locktime.rs +++ b/crates/brk_indexer/src/structs/locktime.rs @@ -1,4 +1,4 @@ -use iterator::Height; +use brk_parser::Height; use serde::Serialize; use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes}; diff --git a/indexer/src/structs/mod.rs b/crates/brk_indexer/src/structs/mod.rs similarity index 100% rename from indexer/src/structs/mod.rs rename to crates/brk_indexer/src/structs/mod.rs diff --git a/indexer/src/structs/sats.rs b/crates/brk_indexer/src/structs/sats.rs similarity index 97% rename from indexer/src/structs/sats.rs rename to crates/brk_indexer/src/structs/sats.rs index 238e7cc3d..9d5881440 100644 --- a/indexer/src/structs/sats.rs +++ b/crates/brk_indexer/src/structs/sats.rs @@ -3,8 +3,8 @@ use std::{ ops::{Add, AddAssign, Mul, Sub, SubAssign}, }; +use brk_parser::{Height, bitcoin::Amount}; use derive_deref::{Deref, DerefMut}; -use iterator::{bitcoin::Amount, Height}; use serde::Serialize; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/indexer/src/structs/timestamp.rs b/crates/brk_indexer/src/structs/timestamp.rs similarity index 100% rename from indexer/src/structs/timestamp.rs rename to crates/brk_indexer/src/structs/timestamp.rs diff --git a/indexer/src/structs/txid.rs b/crates/brk_indexer/src/structs/txid.rs similarity index 100% rename from indexer/src/structs/txid.rs rename to crates/brk_indexer/src/structs/txid.rs diff --git a/indexer/src/structs/txindex.rs b/crates/brk_indexer/src/structs/txindex.rs similarity index 100% rename from indexer/src/structs/txindex.rs rename to crates/brk_indexer/src/structs/txindex.rs diff --git a/indexer/src/structs/txinindex.rs b/crates/brk_indexer/src/structs/txinindex.rs similarity index 100% rename from indexer/src/structs/txinindex.rs rename to crates/brk_indexer/src/structs/txinindex.rs diff --git a/indexer/src/structs/txoutindex.rs b/crates/brk_indexer/src/structs/txoutindex.rs similarity index 100% rename from indexer/src/structs/txoutindex.rs rename to crates/brk_indexer/src/structs/txoutindex.rs diff --git a/indexer/src/structs/txversion.rs b/crates/brk_indexer/src/structs/txversion.rs similarity index 100% rename from indexer/src/structs/txversion.rs rename to crates/brk_indexer/src/structs/txversion.rs diff --git a/indexer/src/structs/vin.rs b/crates/brk_indexer/src/structs/vin.rs similarity index 100% rename from indexer/src/structs/vin.rs rename to crates/brk_indexer/src/structs/vin.rs diff --git a/indexer/src/structs/vout.rs b/crates/brk_indexer/src/structs/vout.rs similarity index 100% rename from indexer/src/structs/vout.rs rename to crates/brk_indexer/src/structs/vout.rs diff --git a/indexer/src/structs/weight.rs b/crates/brk_indexer/src/structs/weight.rs similarity index 100% rename from indexer/src/structs/weight.rs rename to crates/brk_indexer/src/structs/weight.rs diff --git a/iterator/CHANGELOG.md b/crates/brk_parser/CHANGELOG.md similarity index 100% rename from iterator/CHANGELOG.md rename to crates/brk_parser/CHANGELOG.md diff --git a/iterator/Cargo.lock b/crates/brk_parser/Cargo.lock similarity index 100% rename from iterator/Cargo.lock rename to crates/brk_parser/Cargo.lock diff --git a/iterator/Cargo.toml b/crates/brk_parser/Cargo.toml similarity index 97% rename from iterator/Cargo.toml rename to crates/brk_parser/Cargo.toml index 014a1c029..e80f382c3 100644 --- a/iterator/Cargo.toml +++ b/crates/brk_parser/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "biterator" +name = "brk_parser" description = "A very fast Bitcoin block iterator built on top of bitcoin-rust" version = "0.2.3" repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/biter" diff --git a/iterator/LICENSE.md b/crates/brk_parser/LICENSE.md similarity index 100% rename from iterator/LICENSE.md rename to crates/brk_parser/LICENSE.md diff --git a/iterator/README.md b/crates/brk_parser/README.md similarity index 100% rename from iterator/README.md rename to crates/brk_parser/README.md diff --git a/iterator/src/blk_index_to_blk_path.rs b/crates/brk_parser/src/blk_index_to_blk_path.rs similarity index 100% rename from iterator/src/blk_index_to_blk_path.rs rename to crates/brk_parser/src/blk_index_to_blk_path.rs diff --git a/iterator/src/blk_index_to_blk_recap.rs b/crates/brk_parser/src/blk_index_to_blk_recap.rs similarity index 100% rename from iterator/src/blk_index_to_blk_recap.rs rename to crates/brk_parser/src/blk_index_to_blk_recap.rs diff --git a/iterator/src/blk_metadata.rs b/crates/brk_parser/src/blk_metadata.rs similarity index 100% rename from iterator/src/blk_metadata.rs rename to crates/brk_parser/src/blk_metadata.rs diff --git a/iterator/src/blk_metadata_and_block.rs b/crates/brk_parser/src/blk_metadata_and_block.rs similarity index 100% rename from iterator/src/blk_metadata_and_block.rs rename to crates/brk_parser/src/blk_metadata_and_block.rs diff --git a/iterator/src/blk_recap.rs b/crates/brk_parser/src/blk_recap.rs similarity index 100% rename from iterator/src/blk_recap.rs rename to crates/brk_parser/src/blk_recap.rs diff --git a/iterator/src/error.rs b/crates/brk_parser/src/error.rs similarity index 100% rename from iterator/src/error.rs rename to crates/brk_parser/src/error.rs diff --git a/iterator/src/height.rs b/crates/brk_parser/src/height.rs similarity index 100% rename from iterator/src/height.rs rename to crates/brk_parser/src/height.rs diff --git a/iterator/src/lib.rs b/crates/brk_parser/src/lib.rs similarity index 100% rename from iterator/src/lib.rs rename to crates/brk_parser/src/lib.rs diff --git a/iterator/src/main.rs b/crates/brk_parser/src/main.rs similarity index 91% rename from iterator/src/main.rs rename to crates/brk_parser/src/main.rs index 4341a714d..623e79bcd 100644 --- a/iterator/src/main.rs +++ b/crates/brk_parser/src/main.rs @@ -17,7 +17,7 @@ fn main() { let start = None; let end = None; - biterator::new(data_dir, start, end, rpc) + brk_parser::new(data_dir, start, end, rpc) .iter() .for_each(|(height, _block, hash)| { println!("{height}: {hash}"); diff --git a/iterator/src/utils.rs b/crates/brk_parser/src/utils.rs similarity index 100% rename from iterator/src/utils.rs rename to crates/brk_parser/src/utils.rs diff --git a/iterator/src/xor.rs b/crates/brk_parser/src/xor.rs similarity index 100% rename from iterator/src/xor.rs rename to crates/brk_parser/src/xor.rs diff --git a/logger/Cargo.toml b/crates/brk_printer/Cargo.toml similarity index 92% rename from logger/Cargo.toml rename to crates/brk_printer/Cargo.toml index d73da97f4..5009a0d61 100644 --- a/logger/Cargo.toml +++ b/crates/brk_printer/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "cl0g" +name = "brk_printer" description = "A clean logger" version = "0.1.0" edition = { workspace = true } diff --git a/logger/src/lib.rs b/crates/brk_printer/src/lib.rs similarity index 100% rename from logger/src/lib.rs rename to crates/brk_printer/src/lib.rs diff --git a/logger/src/main.rs b/crates/brk_printer/src/main.rs similarity index 55% rename from logger/src/main.rs rename to crates/brk_printer/src/main.rs index bb874d4b5..80e3c0af9 100644 --- a/logger/src/main.rs +++ b/crates/brk_printer/src/main.rs @@ -1,7 +1,6 @@ -use cl0g::init_log; use log::info; fn main() { - init_log(None); + brk_printer::init_log(None); info!("test"); } diff --git a/server/Cargo.toml b/crates/brk_server/Cargo.toml similarity index 79% rename from server/Cargo.toml rename to crates/brk_server/Cargo.toml index 6b15f027a..5806aeea3 100644 --- a/server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "berver" +name = "brk_server" description = "A Bitcoin data server built on top of bindexer, bricer and bomputer" version = "0.1.0" edition = { workspace = true } @@ -7,12 +7,13 @@ license = { workspace = true } [dependencies] axum = "0.8.1" +brk_computer = { workspace = true } +brk_indexer = { workspace = true } +brk_printer = { workspace = true } color-eyre = { workspace = true } -computer = { workspace = true } derive_deref = { workspace = true } -indexer = { workspace = true } jiff = { workspace = true } -logger = { workspace = true } +log = { workspace = true } oxc = { version = "0.52.0", features = ["codegen", "minifier"] } serde = { workspace = true } serde_json = { workspace = true } diff --git a/server/src/api/explorer/mod.rs b/crates/brk_server/src/api/explorer/mod.rs similarity index 100% rename from server/src/api/explorer/mod.rs rename to crates/brk_server/src/api/explorer/mod.rs diff --git a/server/src/api/mod.rs b/crates/brk_server/src/api/mod.rs similarity index 100% rename from server/src/api/mod.rs rename to crates/brk_server/src/api/mod.rs diff --git a/server/src/api/vecs/format.rs b/crates/brk_server/src/api/vecs/format.rs similarity index 100% rename from server/src/api/vecs/format.rs rename to crates/brk_server/src/api/vecs/format.rs diff --git a/server/src/api/vecs/index.rs b/crates/brk_server/src/api/vecs/index.rs similarity index 100% rename from server/src/api/vecs/index.rs rename to crates/brk_server/src/api/vecs/index.rs diff --git a/server/src/api/vecs/mod.rs b/crates/brk_server/src/api/vecs/mod.rs similarity index 100% rename from server/src/api/vecs/mod.rs rename to crates/brk_server/src/api/vecs/mod.rs diff --git a/server/src/api/vecs/query.rs b/crates/brk_server/src/api/vecs/query.rs similarity index 100% rename from server/src/api/vecs/query.rs rename to crates/brk_server/src/api/vecs/query.rs diff --git a/server/src/api/vecs/tree.rs b/crates/brk_server/src/api/vecs/tree.rs similarity index 100% rename from server/src/api/vecs/tree.rs rename to crates/brk_server/src/api/vecs/tree.rs diff --git a/server/src/files/file.rs b/crates/brk_server/src/files/file.rs similarity index 98% rename from server/src/files/file.rs rename to crates/brk_server/src/files/file.rs index 4018e1776..196f39033 100644 --- a/server/src/files/file.rs +++ b/crates/brk_server/src/files/file.rs @@ -10,12 +10,11 @@ use axum::{ http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, }; -use logger::{error, info}; +use log::{error, info}; use crate::{ - log_result, + WEBSITE_DEV_PATH, log_result, traits::{HeaderMapExtended, ModifiedState, ResponseExtended}, - WEBSITE_DEV_PATH, }; use super::minify::minify_js; diff --git a/server/src/files/minify.rs b/crates/brk_server/src/files/minify.rs similarity index 100% rename from server/src/files/minify.rs rename to crates/brk_server/src/files/minify.rs diff --git a/server/src/files/mod.rs b/crates/brk_server/src/files/mod.rs similarity index 100% rename from server/src/files/mod.rs rename to crates/brk_server/src/files/mod.rs diff --git a/server/src/lib.rs b/crates/brk_server/src/lib.rs similarity index 93% rename from server/src/lib.rs rename to crates/brk_server/src/lib.rs index 66c6448ba..e2aca251f 100644 --- a/server/src/lib.rs +++ b/crates/brk_server/src/lib.rs @@ -1,12 +1,12 @@ use std::time::Instant; use api::{ApiRoutes, VecIdToIndexToVec}; -use axum::{http::StatusCode, routing::get, serve, Json, Router}; +use axum::{Json, Router, http::StatusCode, routing::get, serve}; +use brk_computer::Computer; +use brk_indexer::Indexer; use color_eyre::owo_colors::OwoColorize; -use computer::Computer; use files::FilesRoutes; -use indexer::Indexer; -use logger::{error, info}; +use log::{error, info}; use storable_vec::STATELESS; use tokio::net::TcpListener; use tower_http::compression::CompressionLayer; diff --git a/server/src/main.rs b/crates/brk_server/src/main.rs similarity index 71% rename from server/src/main.rs rename to crates/brk_server/src/main.rs index b01a47c11..fde668fdc 100644 --- a/server/src/main.rs +++ b/crates/brk_server/src/main.rs @@ -1,20 +1,20 @@ use std::path::Path; -use computer::Computer; -use indexer::Indexer; +use brk_computer::Computer; +use brk_indexer::Indexer; use storable_vec::STATELESS; #[tokio::main] pub async fn main() -> color_eyre::Result<()> { color_eyre::install()?; - logger::init_log(None); + brk_printer::init_log(None); let path = Path::new("../_outputs"); let indexer: Indexer = Indexer::import(&path.join("indexes"))?; let computer: Computer = Computer::import(&path.join("computed"))?; - berver::main(indexer, computer).await.unwrap(); + brk_server::main(indexer, computer).await.unwrap(); Ok(()) } diff --git a/server/src/traits/header_map.rs b/crates/brk_server/src/traits/header_map.rs similarity index 98% rename from server/src/traits/header_map.rs rename to crates/brk_server/src/traits/header_map.rs index 596a22cbc..923155885 100644 --- a/server/src/traits/header_map.rs +++ b/crates/brk_server/src/traits/header_map.rs @@ -1,11 +1,11 @@ use std::{path::Path, time}; use axum::http::{ - header::{self, HOST, IF_MODIFIED_SINCE}, HeaderMap, + header::{self, HOST, IF_MODIFIED_SINCE}, }; -use jiff::{civil::DateTime, fmt::strtime, tz::TimeZone, Timestamp}; -use logger::info; +use jiff::{Timestamp, civil::DateTime, fmt::strtime, tz::TimeZone}; +use log::info; const STALE_IF_ERROR: u64 = 30_000_000; // 1 Year ish const MODIFIED_SINCE_FORMAT: &str = "%a, %d %b %Y %H:%M:%S GMT"; diff --git a/server/src/traits/mod.rs b/crates/brk_server/src/traits/mod.rs similarity index 100% rename from server/src/traits/mod.rs rename to crates/brk_server/src/traits/mod.rs diff --git a/server/src/traits/response.rs b/crates/brk_server/src/traits/response.rs similarity index 100% rename from server/src/traits/response.rs rename to crates/brk_server/src/traits/response.rs diff --git a/hodor/Cargo.toml b/crates/hodor/Cargo.toml similarity index 65% rename from hodor/Cargo.toml rename to crates/hodor/Cargo.toml index d1697fb1c..0100d6fc9 100644 --- a/hodor/Cargo.toml +++ b/crates/hodor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hodor" -description = "An exit blocker, can hold the door until a task is completed" +description = "Hold the door, an exit blocker that can wait until a task is completed" version = "0.1.0" edition = { workspace = true } license = { workspace = true } diff --git a/hodor/src/lib.rs b/crates/hodor/src/lib.rs similarity index 100% rename from hodor/src/lib.rs rename to crates/hodor/src/lib.rs diff --git a/storable_vec/.gitignore b/crates/storable_vec/.gitignore similarity index 100% rename from storable_vec/.gitignore rename to crates/storable_vec/.gitignore diff --git a/storable_vec/Cargo.lock b/crates/storable_vec/Cargo.lock similarity index 100% rename from storable_vec/Cargo.lock rename to crates/storable_vec/Cargo.lock diff --git a/storable_vec/Cargo.toml b/crates/storable_vec/Cargo.toml similarity index 100% rename from storable_vec/Cargo.toml rename to crates/storable_vec/Cargo.toml diff --git a/storable_vec/LICENSE.md b/crates/storable_vec/LICENSE.md similarity index 100% rename from storable_vec/LICENSE.md rename to crates/storable_vec/LICENSE.md diff --git a/storable_vec/README.md b/crates/storable_vec/README.md similarity index 100% rename from storable_vec/README.md rename to crates/storable_vec/README.md diff --git a/storable_vec/src/enums/error.rs b/crates/storable_vec/src/enums/error.rs similarity index 100% rename from storable_vec/src/enums/error.rs rename to crates/storable_vec/src/enums/error.rs diff --git a/storable_vec/src/enums/mod.rs b/crates/storable_vec/src/enums/mod.rs similarity index 100% rename from storable_vec/src/enums/mod.rs rename to crates/storable_vec/src/enums/mod.rs diff --git a/storable_vec/src/enums/value.rs b/crates/storable_vec/src/enums/value.rs similarity index 100% rename from storable_vec/src/enums/value.rs rename to crates/storable_vec/src/enums/value.rs diff --git a/storable_vec/src/lib.rs b/crates/storable_vec/src/lib.rs similarity index 100% rename from storable_vec/src/lib.rs rename to crates/storable_vec/src/lib.rs diff --git a/storable_vec/src/main.rs b/crates/storable_vec/src/main.rs similarity index 100% rename from storable_vec/src/main.rs rename to crates/storable_vec/src/main.rs diff --git a/storable_vec/src/structs/mod.rs b/crates/storable_vec/src/structs/mod.rs similarity index 100% rename from storable_vec/src/structs/mod.rs rename to crates/storable_vec/src/structs/mod.rs diff --git a/storable_vec/src/structs/unsafe_slice.rs b/crates/storable_vec/src/structs/unsafe_slice.rs similarity index 100% rename from storable_vec/src/structs/unsafe_slice.rs rename to crates/storable_vec/src/structs/unsafe_slice.rs diff --git a/storable_vec/src/structs/version.rs b/crates/storable_vec/src/structs/version.rs similarity index 100% rename from storable_vec/src/structs/version.rs rename to crates/storable_vec/src/structs/version.rs diff --git a/storable_vec/src/traits/any.rs b/crates/storable_vec/src/traits/any.rs similarity index 100% rename from storable_vec/src/traits/any.rs rename to crates/storable_vec/src/traits/any.rs diff --git a/storable_vec/src/traits/bytes.rs b/crates/storable_vec/src/traits/bytes.rs similarity index 100% rename from storable_vec/src/traits/bytes.rs rename to crates/storable_vec/src/traits/bytes.rs diff --git a/storable_vec/src/traits/mod.rs b/crates/storable_vec/src/traits/mod.rs similarity index 100% rename from storable_vec/src/traits/mod.rs rename to crates/storable_vec/src/traits/mod.rs diff --git a/storable_vec/src/traits/stored_index.rs b/crates/storable_vec/src/traits/stored_index.rs similarity index 100% rename from storable_vec/src/traits/stored_index.rs rename to crates/storable_vec/src/traits/stored_index.rs diff --git a/storable_vec/src/traits/stored_type.rs b/crates/storable_vec/src/traits/stored_type.rs similarity index 100% rename from storable_vec/src/traits/stored_type.rs rename to crates/storable_vec/src/traits/stored_type.rs diff --git a/snkrj/Cargo.lock b/snkrj/Cargo.lock deleted file mode 100644 index 7939c3538..000000000 --- a/snkrj/Cargo.lock +++ /dev/null @@ -1,242 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "libc" -version = "0.2.168" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" - -[[package]] -name = "lock_api" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" - -[[package]] -name = "memmap2" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" -dependencies = [ - "libc", -] - -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall", - "smallvec", - "winapi", -] - -[[package]] -name = "proc-macro2" -version = "1.0.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - -[[package]] -name = "sanakirja" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81aaf70d064e2122209f04d01fd91e8908e7a327b516236e1cbc0c3f34ac6d11" -dependencies = [ - "fs2", - "log", - "memmap2", - "parking_lot", - "sanakirja-core", - "serde", - "thiserror", -] - -[[package]] -name = "sanakirja-core" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8376db34ae3eac6e7bd91168bc638450073b708ce9fb46940de676f552238bf5" - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "serde" -version = "1.0.216" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.216" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - -[[package]] -name = "snkrj" -version = "0.1.1" -dependencies = [ - "sanakirja", -] - -[[package]] -name = "syn" -version = "2.0.90" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "unicode-ident" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/snkrj/Cargo.toml b/snkrj/Cargo.toml deleted file mode 100644 index 49704555a..000000000 --- a/snkrj/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "snkrj" -description = "A very simple wrapper around Sanakirja" -version = "0.1.1" -license = "MIT" -repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/snkrj" -keywords = ["database", "sanakirja", "btreemap"] -categories = ["database"] -edition = "2021" - -[dependencies] -sanakirja = "1.4.3" diff --git a/snkrj/LICENSE.md b/snkrj/LICENSE.md deleted file mode 100644 index aac235dcb..000000000 --- a/snkrj/LICENSE.md +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2024 snkrj - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/snkrj/README.md b/snkrj/README.md deleted file mode 100644 index 1803d180c..000000000 --- a/snkrj/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# snkrj - -A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap. - -## Example - -```rust -use snkrj::{AnyDatabase, Database}; - -fn main() { - let path = std::env::temp_dir().join("./db"); - - let database: Database = Database::open(path.clone()).unwrap(); - let _ = database.destroy(); - - let mut database: Database = Database::open(path.clone()).unwrap(); - database.insert(64, 128); - database.export(false).unwrap(); - - let mut database: Database = Database::open(path).unwrap(); - database.insert(1, 2); - database.insert(128, 256); - println!("iter_ram:"); - database.iter_ram().for_each(|pair| { - println!("{:?}", pair); - }); - println!("iter_disk:"); - database.iter_disk().for_each(|pair| { - println!("{:?}", pair.unwrap()); - }); - println!("iter_ram_then_disk:"); - database.iter_ram_then_disk().for_each(|pair| { - println!("{:?}", pair); - }); - database.export(false).unwrap(); -} -``` diff --git a/snkrj/src/base.rs b/snkrj/src/base.rs deleted file mode 100644 index 113ffe77a..000000000 --- a/snkrj/src/base.rs +++ /dev/null @@ -1,174 +0,0 @@ -// https://docs.rs/sanakirja/latest/sanakirja/index.html -// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/ - -use std::{ - collections::BTreeMap, - fs::{self, File}, - io, - path::{Component, Path, PathBuf}, - result::Result, -}; - -use sanakirja::btree::{page, Db_}; -pub use sanakirja::*; - -use crate::{DatabaseKey, DatabaseValue}; - -pub type UnitDatabase = Base<(), ()>; - -/// -/// A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap. -/// -/// The state of the tree is uncommited until `.export()` is called during which it is unsafe to stop the program. -/// -pub struct Base -where - Key: DatabaseKey, - Value: DatabaseValue, -{ - pathbuf: PathBuf, - db: Db_>, - txn: MutTxn, -} - -const ROOT_DB: usize = 0; -const PAGE_SIZE: u64 = 4096; - -const DEFRAGMENT_RATIO_THRESHOLD: f64 = 0.5; - -impl Base -where - Key: DatabaseKey, - Value: DatabaseValue, -{ - const KEY_SIZE: usize = size_of::(); - const VALUE_SIZE: usize = size_of::(); - const KEY_AND_VALUE_SIZE: usize = Self::KEY_SIZE + Self::VALUE_SIZE; - - /// Open a database without a lock file where only one instance is safe to open. - pub fn open(pathbuf: PathBuf) -> Result { - fs::create_dir_all(&pathbuf)?; - - let env = unsafe { Env::new_nolock(Self::path_sanakirja_(&pathbuf), PAGE_SIZE, 1)? }; - - let mut txn = Env::mut_txn_begin(env)?; - - let db = txn - .root_db(ROOT_DB) - .unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() }); - - Ok(Self { pathbuf, db, txn }) - } - - pub fn path_sanakirja(&self) -> PathBuf { - Self::path_sanakirja_(&self.pathbuf) - } - fn path_sanakirja_(path: &Path) -> PathBuf { - path.join("sanakirja") - } - - pub fn path_self_defragmented(&self) -> PathBuf { - let defragmented_path_opt: Option = self.pathbuf.components().last(); - let folder = match defragmented_path_opt { - Some(Component::Normal(f)) => f.to_str().unwrap(), - _ => unreachable!(), - }; - let mut original_path = self.pathbuf.clone(); - original_path.pop(); - original_path.join(format!("{folder}-defragmented")) - } - - pub fn read_length(&self) -> usize { - Self::read_length_(&self.pathbuf) - } - pub fn read_length_(path: &Path) -> usize { - fs::read(Self::path_length(path)) - .map(|v| { - let mut buf = [0_u8; 8]; - v.iter().enumerate().take(8).for_each(|(i, b)| { - buf[i] = *b; - }); - usize::from_le_bytes(buf) - }) - .unwrap_or_default() - } - pub fn write_length(&self, len: usize) -> Result<(), io::Error> { - Self::write_length_(&self.pathbuf, len) - } - pub fn write_length_(path: &Path, len: usize) -> Result<(), io::Error> { - fs::write(Self::path_length(path), len.to_le_bytes()) - } - fn path_length(path: &Path) -> PathBuf { - path.join("length") - } - - #[inline] - pub fn get(&self, key: &Key) -> Result, Error> { - let option = btree::get(&self.txn, &self.db, key, None)?; - if let Some((key_found, v)) = option { - if key == key_found { - return Ok(Some(v)); - } - } - Ok(None) - } - - /// Iterate over key/value pairs from the database (disk) - #[inline] - #[allow(clippy::type_complexity)] - pub fn iter( - &self, - ) -> Result, Key, Value, page::Page>, Error> { - btree::iter(&self.txn, &self.db, None) - } - - pub fn put(&mut self, key: &Key, value: &Value) -> Result { - btree::put(&mut self.txn, &mut self.db, key, value) - } - - pub fn del(&mut self, key: &Key, value: Option<&Value>) -> Result { - btree::del(&mut self.txn, &mut self.db, key, value) - } - - fn get_file_size_to_data_size_ratio(&self, len: usize) -> Result { - let data_bytes = (len * Self::KEY_AND_VALUE_SIZE) as f64; - let file_bytes = File::open(&self.pathbuf)?.metadata()?.len() as f64; - Ok(file_bytes / data_bytes) - } - - pub fn should_defragment(&self, len: usize) -> Result { - Ok(self.get_file_size_to_data_size_ratio(len)? >= DEFRAGMENT_RATIO_THRESHOLD) - } - - pub fn iter_collect(&self) -> Result, Error> { - self.iter()?.collect::<_>() - } - - pub fn iter_collect_multi(&self) -> Result>, Error> { - let mut tree: BTreeMap<_, Vec<_>> = BTreeMap::new(); - self.iter()?.try_for_each(|res| -> Result<(), Error> { - let (key, value): (&Key, &Value) = res?; - tree.entry(key).or_default().push(value); - Ok(()) - })?; - Ok(tree) - } - - pub fn commit(mut self, len: usize) -> Result<(), Error> { - // dbg!(&self.pathbuf, len); - // panic!(); - self.write_length(len)?; - self.txn.set_root(ROOT_DB, self.db.db.into()); - self.txn.commit() - } - - pub fn destroy(self) -> io::Result<()> { - let path = self.pathbuf.to_owned(); - drop(self); - fs::remove_dir_all(&path) - } - - pub fn path(&self) -> &Path { - &self.pathbuf - } -} diff --git a/snkrj/src/lib.rs b/snkrj/src/lib.rs deleted file mode 100644 index 241bfd3b7..000000000 --- a/snkrj/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -// https://docs.rs/sanakirja/latest/sanakirja/index.html -// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/ - -pub use sanakirja::*; - -mod base; -mod multi; -mod traits; -mod unique; - -pub use base::*; -pub use multi::*; -pub use traits::*; -pub use unique::*; diff --git a/snkrj/src/main.rs b/snkrj/src/main.rs deleted file mode 100644 index 055d45f79..000000000 --- a/snkrj/src/main.rs +++ /dev/null @@ -1,29 +0,0 @@ -use snkrj::DatabaseUnique; - -fn main() { - let path = std::env::temp_dir().join("./db"); - - // let database: DatabaseUnique = DatabaseUnique::open(path.clone()).unwrap(); - // let _ = database.destroy(); - - let mut database: DatabaseUnique = DatabaseUnique::open(path.clone()).unwrap(); - database.insert(64, 128); - database.export().unwrap(); - - let mut database: DatabaseUnique = DatabaseUnique::open(path).unwrap(); - database.insert(1, 2); - database.insert(128, 256); - println!("iter_ram:"); - database.iter_ram().for_each(|pair| { - println!("{:?}", pair); - }); - println!("iter_disk:"); - database.iter_disk().unwrap().for_each(|pair| { - println!("{:?}", pair.unwrap()); - }); - println!("iter_ram_then_disk:"); - database.iter_ram_then_disk().unwrap().for_each(|pair| { - println!("{:?}", pair); - }); - database.export().unwrap(); -} diff --git a/snkrj/src/multi.rs b/snkrj/src/multi.rs deleted file mode 100644 index 24f8cbebe..000000000 --- a/snkrj/src/multi.rs +++ /dev/null @@ -1,240 +0,0 @@ -// https://docs.rs/sanakirja/latest/sanakirja/index.html -// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/ - -use core::panic; -use std::{ - collections::{BTreeMap, BTreeSet}, - fs, mem, - path::PathBuf, - result::Result, -}; - -use sanakirja::btree::page; -pub use sanakirja::*; - -use crate::{AnyDatabase, Base, DatabaseKey, DatabaseValue}; - -/// -/// A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap. -/// -/// The state of the tree is uncommited until `.export()` is called during which it is unsafe to stop the program. -/// -pub struct DatabaseMulti -where - Key: DatabaseKey, - Value: DatabaseValue, -{ - puts: BTreeMap>, - dels: BTreeSet, - len: usize, - db: Base, -} - -impl DatabaseMulti -where - Key: DatabaseKey, - Value: DatabaseValue, -{ - /// Open a database without a lock file where only one instance is safe to open. - pub fn open(pathbuf: PathBuf) -> Result { - let db = Base::open(pathbuf)?; - Ok(Self { - len: db.read_length(), - puts: BTreeMap::default(), - dels: BTreeSet::default(), - db, - }) - } - - #[inline] - pub fn get(&self, key: &Key) -> Result, Error> { - if let Some(cached_put) = self.get_uncommited(key) { - return Ok(Some(cached_put)); - } - - self.db.get(key) - } - - /// Get only from the uncommited tree (ram) without checking the database (disk) - #[inline] - pub fn get_uncommited(&self, key: &Key) -> Option<&Value> { - self.puts.get(key).and_then(|v| v.first()) - } - - /// Get mut only from the uncommited tree (ram) without checking the database (disk) - #[inline] - pub fn get_mut_uncommited(&mut self, key: &Key) -> Option<&mut Value> { - self.puts.get_mut(key).and_then(|v| v.first_mut()) - } - - #[inline] - pub fn insert(&mut self, key: Key, value: Value) -> Option { - self.dels.remove(&key); - self.unchecked_insert(key, value) - } - - /// Insert without removing the key to the dels tree, so be sure that it hasn't added to the delete set - #[inline] - pub fn unchecked_insert(&mut self, key: Key, value: Value) -> Option { - self.len += 1; - self.puts.entry(key).or_default().push(value); - None - } - - #[inline] - pub fn update(&mut self, key: Key, value: Value) -> Option { - todo!() - // self.dels.insert(key.clone()); - // self.puts.insert(key, value) - } - - #[inline] - pub fn remove(&mut self, key: &Key) -> Option { - todo!() - // self.len -= 1; - // self.puts.remove(key).or_else(|| { - // self.dels.insert(key.clone()); - // None - // }) - } - - /// Remove only from the uncommited tree (ram) without checking the database (disk) - #[inline] - pub fn remove_from_uncommited(&mut self, key: &Key) -> Option { - todo!() - // self.len -= 1; - // self.puts.remove(key) - } - - /// Add the key only to the dels tree without checking if it's present in the puts tree, only use if you are positive that you neither added nor updated an entry with this key - #[inline] - pub fn remove_later_from_disk(&mut self, key: &Key) { - todo!() - // self.len -= 1; - // self.dels.insert(key.clone()); - } - - /// Iterate over key/value pairs from the uncommited tree (ram) - #[inline] - pub fn iter_ram(&self) -> std::collections::btree_map::Iter<'_, Key, Vec> { - self.puts.iter() - } - - /// Iterate over key/value pairs from the database (disk) - #[inline] - #[allow(clippy::type_complexity)] - pub fn iter_disk( - &self, - ) -> Result, Key, Value, page::Page>, Error> { - self.db.iter() - } - - /// Iterate over key/value pairs - // #[inline] - // pub fn iter_ram_then_disk(&self) -> Result, Error> { - // todo!(); - // // Ok(self.iter_ram().chain(self.iter_disk()?.map(|r| r.unwrap()))) - // } - - /// Collect a **clone** of all uncommited key/value pairs (ram) - pub fn collect_ram(&self) -> BTreeMap { - todo!() - // self.puts.clone() - } - - #[inline] - pub fn len(&self) -> usize { - self.len - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Flush all puts and dels from the ram to disk with an option to defragment the database to save some disk space - /// - /// /!\ Do not kill the program while this function is runnning /!\ - pub fn export(mut self) -> Result<(), Error> { - if self.dels.is_empty() && self.puts.is_empty() { - return Ok(()); - } - - if self.db.should_defragment(self.len)? { - let mut btree = self.db.iter_collect_multi()?; - // TODO: - // self.dels.iter().for_each(|key| { - // btree.remove(key); - // }); - self.puts.iter().for_each(|(key, values)| { - // btree.insert(key, value); - let vec = btree.entry(key).or_default(); - vec.extend(values.iter()); - }); - - let path_self_original = self.db.path().to_owned(); - let path_self_defragmented = self.db.path_self_defragmented(); - - let len = btree.values().map(|v| v.len()).sum::(); - - if len != self.len { - dbg!(len, self.len, path_self_defragmented); - panic!("Len should be the same"); - } - - { - let mut defragmented = Self::open(path_self_defragmented.clone()).unwrap(); - - btree - .into_iter() - .try_for_each(|(key, values)| -> Result<(), Error> { - values - .into_iter() - .try_for_each(|value| -> Result<(), Error> { - defragmented.db.put(key, value)?; - Ok(()) - })?; - Ok(()) - })?; - - defragmented.len = len; - defragmented.db.commit(self.len)?; - } - - drop(self); - - fs::remove_dir_all(&path_self_original)?; - fs::rename(&path_self_defragmented, &path_self_original)?; - - Ok(()) - } else { - mem::take(&mut self.dels) - .into_iter() - .try_for_each(|key| -> Result<(), Error> { - self.db.del(&key, None)?; - Ok(()) - })?; - - mem::take(&mut self.puts).into_iter().try_for_each( - |(key, vec): (Key, Vec)| -> Result<(), Error> { - vec.into_iter().try_for_each(|value| -> Result<(), Error> { - self.db.put(&key, &value)?; - Ok(()) - }) - }, - )?; - - self.db.commit(self.len) - } - } -} - -impl AnyDatabase for DatabaseMulti -where - Key: DatabaseKey, - Value: DatabaseValue, -{ - fn export(self) -> Result<(), Error> { - self.export() - } -} diff --git a/snkrj/src/traits.rs b/snkrj/src/traits.rs deleted file mode 100644 index 758b70472..000000000 --- a/snkrj/src/traits.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::fmt::Debug; - -use sanakirja::Storable; - -pub trait AnyDatabase { - fn export(self) -> Result<(), sanakirja::Error>; - // fn destroy(self) -> io::Result<()>; -} - -pub trait DatabaseKey -where - Self: Ord + Clone + Debug + Storable + Send + Sync, -{ - const SIZE: usize = size_of::(); - const SIZE_SMALLER_THAN_TWO: bool = Self::SIZE < 2; - - fn as_ne_byte(&self) -> u8 { - let data: *const Self = self; - let data: *const u8 = data as *const u8; - let slice = unsafe { std::slice::from_raw_parts(data, Self::SIZE) }; - - *(if cfg!(target_endian = "big") { - slice.last() - } else { - slice.first() - }) - .unwrap() - } - - fn as_ne_six_bits(&self) -> u8 { - self.as_ne_byte() >> 2 - } - - fn as_ne_two_bytes(&self) -> [u8; 2] { - let data: *const Self = self; - let data: *const u8 = data as *const u8; - let slice = unsafe { std::slice::from_raw_parts(data, Self::SIZE) }; - - if Self::SIZE_SMALLER_THAN_TWO { - panic!("Doesn't make sense") - } - - if cfg!(target_endian = "big") { - let mut iter = slice.iter().rev(); - [*iter.next().unwrap(), *iter.next().unwrap()] - } else { - let mut iter = slice.iter(); - [*iter.next().unwrap(), *iter.next().unwrap()] - } - } -} -impl DatabaseKey for T where T: Ord + Clone + Debug + Storable + Send + Sync {} - -pub trait DatabaseValue -where - Self: Clone + Storable + PartialEq + Send + Sync, -{ -} -impl DatabaseValue for T where T: Clone + Storable + PartialEq + Send + Sync {} diff --git a/snkrj/src/unique.rs b/snkrj/src/unique.rs deleted file mode 100644 index f5dc6582e..000000000 --- a/snkrj/src/unique.rs +++ /dev/null @@ -1,223 +0,0 @@ -// https://docs.rs/sanakirja/latest/sanakirja/index.html -// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/ - -use core::panic; -use std::{ - collections::{BTreeMap, BTreeSet}, - fs, mem, - path::PathBuf, - result::Result, -}; - -use sanakirja::btree::page; -pub use sanakirja::*; - -use crate::{AnyDatabase, Base, DatabaseKey, DatabaseValue}; - -/// -/// A simple wrapper around Sanakirja aatabase that acts as a very fast on disk BTreeMap. -/// -/// The state of the tree is uncommited until `.export()` is called during which it is unsafe to stop the program. -/// -pub struct DatabaseUnique -where - Key: DatabaseKey, - Value: DatabaseValue, -{ - puts: BTreeMap, - dels: BTreeSet, - len: usize, - db: Base, -} - -impl DatabaseUnique -where - Key: DatabaseKey, - Value: DatabaseValue, -{ - /// Open a database without a lock file where only one instance is safe to open. - pub fn open(pathbuf: PathBuf) -> Result { - let db = Base::open(pathbuf)?; - Ok(Self { - len: db.read_length(), - puts: BTreeMap::default(), - dels: BTreeSet::default(), - db, - }) - } - - #[inline] - pub fn get(&self, key: &Key) -> Result, Error> { - if let Some(cached_put) = self.get_uncommited(key) { - return Ok(Some(cached_put)); - } - - self.db.get(key) - } - - /// Get only from the uncommited tree (ram) without checking the database (disk) - #[inline] - pub fn get_uncommited(&self, key: &Key) -> Option<&Value> { - self.puts.get(key) - } - - /// Get mut only from the uncommited tree (ram) without checking the database (disk) - #[inline] - pub fn get_mut_uncommited(&mut self, key: &Key) -> Option<&mut Value> { - self.puts.get_mut(key) - } - - #[inline] - pub fn insert(&mut self, key: Key, value: Value) -> Option { - self.dels.remove(&key); - self.unchecked_insert(key, value) - } - - /// Insert without removing the key to the dels tree, so be sure that it hasn't added to the delete set - #[inline] - pub fn unchecked_insert(&mut self, key: Key, value: Value) -> Option { - self.len += 1; - self.puts.insert(key, value) - } - - #[inline] - pub fn update(&mut self, key: Key, value: Value) -> Option { - self.dels.insert(key.clone()); - self.puts.insert(key, value) - } - - #[inline] - pub fn remove(&mut self, key: &Key) -> Option { - self.len -= 1; - self.puts.remove(key).or_else(|| { - self.dels.insert(key.clone()); - None - }) - } - - /// Remove only from the uncommited tree (ram) without checking the database (disk) - #[inline] - pub fn remove_from_uncommited(&mut self, key: &Key) -> Option { - self.len -= 1; - self.puts.remove(key) - } - - /// Add the key only to the dels tree without checking if it's present in the puts tree, only use if you are positive that you neither added nor updated an entry with this key - #[inline] - pub fn remove_later_from_disk(&mut self, key: &Key) { - self.len -= 1; - self.dels.insert(key.clone()); - } - - /// Iterate over key/value pairs from the uncommited tree (ram) - #[inline] - pub fn iter_ram(&self) -> std::collections::btree_map::Iter<'_, Key, Value> { - self.puts.iter() - } - - /// Iterate over key/value pairs from the database (disk) - #[inline] - #[allow(clippy::type_complexity)] - pub fn iter_disk( - &self, - ) -> Result, Key, Value, page::Page>, Error> { - self.db.iter() - } - - /// Iterate over key/value pairs - #[inline] - pub fn iter_ram_then_disk(&self) -> Result, Error> { - Ok(self.iter_ram().chain(self.iter_disk()?.map(|r| r.unwrap()))) - } - - /// Collect a **clone** of all uncommited key/value pairs (ram) - pub fn collect_ram(&self) -> BTreeMap { - self.puts.clone() - } - - #[inline] - pub fn len(&self) -> usize { - self.len - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Flush all puts and dels from the ram to disk with an option to defragment the database to save some disk space - /// - /// /!\ Do not kill the program while this function is runnning /!\ - pub fn export(mut self) -> Result<(), Error> { - if self.dels.is_empty() && self.puts.is_empty() { - return Ok(()); - } - - if self.db.should_defragment(self.len)? { - let mut btree = self.db.iter_collect()?; - self.dels.iter().for_each(|key| { - btree.remove(key); - }); - self.puts.iter().for_each(|(key, value)| { - btree.insert(key, value); - }); - - let path_self_original = self.db.path().to_owned(); - let path_self_defragmented = self.db.path_self_defragmented(); - - let len = btree.len(); - - if len != self.len { - dbg!(len, self.len, path_self_defragmented); - panic!("Len should be the same"); - } - - { - let mut defragmented = Self::open(path_self_defragmented.clone()).unwrap(); - - btree - .into_iter() - .try_for_each(|(key, value)| -> Result<(), Error> { - defragmented.db.put(key, value)?; - Ok(()) - })?; - - defragmented.len = len; - defragmented.db.commit(self.len)?; - } - - drop(self); - - fs::remove_dir_all(&path_self_original)?; - fs::rename(&path_self_defragmented, &path_self_original)?; - - Ok(()) - } else { - mem::take(&mut self.dels) - .into_iter() - .try_for_each(|key| -> Result<(), Error> { - self.db.del(&key, None)?; - Ok(()) - })?; - - mem::take(&mut self.puts).into_iter().try_for_each( - |(key, value)| -> Result<(), Error> { - self.db.put(&key, &value)?; - Ok(()) - }, - )?; - - self.db.commit(self.len) - } - } -} - -impl AnyDatabase for DatabaseUnique -where - Key: DatabaseKey, - Value: DatabaseValue, -{ - fn export(self) -> Result<(), Error> { - self.export() - } -}