diff --git a/Cargo.lock b/Cargo.lock index 53f693268..f7ce3629f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -127,9 +127,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" [[package]] name = "append-only-vec" @@ -494,7 +494,6 @@ dependencies = [ "brk_server", "brk_store", "brk_structs", - "vecdb", ] [[package]] @@ -547,6 +546,7 @@ dependencies = [ "brk_structs", "derive_deref", "log", + "pco", "rayon", "serde", "vecdb", @@ -669,7 +669,7 @@ dependencies = [ "serde", "serde_json", "sse-stream", - "thiserror 2.0.12", + "thiserror 2.0.14", "tokio", "tokio-stream", "tokio-util", @@ -1190,18 +1190,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.43" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50fd97c9dc2399518aa331917ac6f274280ec5eb34e555dd291899745c48ec6f" +checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.43" +version = "4.5.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c35b5830294e1fa0462034af85cc95225a4cb07092c088c55bda3147cfcd8f65" +checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8" dependencies = [ "anstream", "anstyle", @@ -1211,9 +1211,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.41" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" +checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" dependencies = [ "heck", "proc-macro2", @@ -2425,9 +2425,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.174" +version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libredox" @@ -2748,7 +2748,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.14", "time", ] @@ -2762,7 +2762,7 @@ dependencies = [ "owo-colors", "oxc-miette-derive", "textwrap", - "thiserror 2.0.12", + "thiserror 2.0.14", "unicode-width", ] @@ -3036,7 +3036,7 @@ dependencies = [ "serde", "serde_json", "simdutf8", - "thiserror 2.0.12", + "thiserror 2.0.14", "tracing", "url", ] @@ -3390,7 +3390,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.14", ] [[package]] @@ -3456,9 +3456,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "d61789d7719defeb74ea5fe81f2fdfdbd28a803847077cecce2ff14e1472f6f1" dependencies = [ "unicode-ident", ] @@ -3863,9 +3863,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "seqdb" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec6a01a38293848cd0b051176e2f77b2013b5386e3433220473d2daab91b81ae" +checksum = "47c7d8532f6feaa3a6b8ebd8d311e6adce7d71bc82ee037ec87dc271a66b112b" dependencies = [ "libc", "log", @@ -4229,11 +4229,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "0b0949c3a6c842cbde3f1686d6eea5a010516deb7085f79db747562d4102f41e" dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl 2.0.14", ] [[package]] @@ -4249,9 +4249,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "cc5b44b4ab9c2fdd0e0512e6bece8388e214c0749f5862b114cc5b7a25daf227" dependencies = [ "proc-macro2", "quote", @@ -4546,7 +4546,7 @@ version = "11.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ef1b7a6d914a34127ed8e1fa927eb7088903787bcded4fa3eef8f85ee1568be" dependencies = [ - "thiserror 2.0.12", + "thiserror 2.0.14", "ts-rs-macros", ] @@ -4650,9 +4650,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" dependencies = [ "getrandom 0.3.3", "js-sys", @@ -4690,9 +4690,9 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vecdb" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f897e9d013ba5650ee2d36843170e01d16100336af876d56f903367e9544791" +checksum = "ea2240e7d9599fe71411915ec56ce7f9e9aac3eb946cd12f5f0705d7422119a4" dependencies = [ "ctrlc", "log", @@ -4710,9 +4710,9 @@ dependencies = [ [[package]] name = "vecdb_derive" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d000b504a1c700eecaf37779b5f5d1733878b372e83fca238c7645d1a8b592e1" +checksum = "ef29d5571fcd184c6212019a7eb2808395f961686121e04e692d52bb8e007f33" dependencies = [ "quote", "syn 2.0.104", diff --git a/Cargo.toml b/Cargo.toml index 004846a0c..ce1308035 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,6 @@ bitcoincore-rpc = "0.19.0" brk_bundler = { version = "0.0.88", path = "crates/brk_bundler" } brk_cli = { version = "0.0.88", path = "crates/brk_cli" } brk_computer = { version = "0.0.88", path = "crates/brk_computer" } -brk_structs = { version = "0.0.88", path = "crates/brk_structs" } brk_error = { version = "0.0.88", path = "crates/brk_error" } brk_fetcher = { version = "0.0.88", path = "crates/brk_fetcher" } brk_indexer = { version = "0.0.88", path = "crates/brk_indexer" } @@ -39,8 +38,7 @@ brk_mcp = { version = "0.0.88", path = "crates/brk_mcp" } brk_parser = { version = "0.0.88", path = "crates/brk_parser" } brk_server = { version = "0.0.88", path = "crates/brk_server" } brk_store = { version = "0.0.88", path = "crates/brk_store" } -vecdb = { version = "0.1.0", features = ["derive"]} -# vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]} +brk_structs = { version = "0.0.88", path = "crates/brk_structs" } byteview = "=0.6.1" derive_deref = "1.1.1" fjall = "2.11.2" @@ -54,6 +52,8 @@ serde_bytes = "0.11.17" serde_derive = "1.0.219" serde_json = { version = "1.0.142", features = ["float_roundtrip"] } tokio = { version = "1.47.1", features = ["rt-multi-thread"] } +# vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]} +vecdb = { version = "0.1.1", features = ["derive"]} zerocopy = "0.8.26" zerocopy-derive = "0.8.26" @@ -68,3 +68,4 @@ cargo-dist-version = "0.29.0" ci = "github" installers = [] targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"] +rust-toolchain-version = "1.89" diff --git a/README.md b/README.md index 74c4e7ab1..06b3328fd 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,6 @@ In contrast, existing alternatives tend to be either [very costly](https://studi - [`brk_server`](https://crates.io/crates/brk_server): A server with an API for anything from BRK - [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall) - [`brk_structs`](https://crates.io/crates/brk_structs): Structs used throughout BRK -- [`vecdb`](https://crates.io/crates/vecdb): A KISS index-value storage engine -- [`vecdb_macros`](https://crates.io/crates/vecdb_macros): Macros for [`vecdb`](https://crates.io/crates/vecdb) ## Hosting as a service @@ -81,7 +79,7 @@ If you'd like to have your own instance hosted for you please contact [hosting@b - Updates delivered at your convenience - Direct communication for feature requests and support - Bitcoin Core or Knots with desired version -- Optional subdomains: `*.bitcoinresearchkit.org`, `*.kibo.money` and `*.satonomics.xyz` +- Optional subdomains: `*.bitcoinresearchkit.org`, `*.brekit.org`, `*.kibo.money` and `*.satonomics.xyz` - Logo featured in the Readme if desired Pricing: `0.01 BTC / month` *or* `0.1 BTC / year` diff --git a/crates/brk/Cargo.toml b/crates/brk/Cargo.toml index cda677ddf..7ca1c1c51 100644 --- a/crates/brk/Cargo.toml +++ b/crates/brk/Cargo.toml @@ -24,7 +24,6 @@ full = [ "server", "store", "structs", - "vecs", ] bundler = ["brk_bundler"] computer = ["brk_computer"] @@ -38,7 +37,6 @@ parser = ["brk_parser"] server = ["brk_server"] store = ["brk_store"] structs = ["brk_structs"] -vecs = ["vecdb"] [dependencies] brk_bundler = { workspace = true, optional = true } @@ -54,7 +52,6 @@ brk_parser = { workspace = true, optional = true } brk_server = { workspace = true, optional = true } brk_store = { workspace = true, optional = true } brk_structs = { workspace = true, optional = true } -vecdb = { workspace = true, optional = true } [package.metadata.docs.rs] all-features = true diff --git a/crates/brk/src/lib.rs b/crates/brk/src/lib.rs index 62f78b219..8816f1147 100644 --- a/crates/brk/src/lib.rs +++ b/crates/brk/src/lib.rs @@ -50,7 +50,3 @@ pub use brk_server as server; #[cfg(feature = "store")] #[doc(inline)] pub use brk_store as store; - -#[cfg(feature = "vecs")] -#[doc(inline)] -pub use vecdb as vecs; diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index 58126903d..45857e79f 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -20,8 +20,8 @@ brk_logger = { workspace = true } brk_parser = { workspace = true } brk_server = { workspace = true } vecdb = { workspace = true } -clap = { version = "4.5.43", features = ["string"] } -clap_derive = "4.5.41" +clap = { version = "4.5.45", features = ["string"] } +clap_derive = "4.5.45" color-eyre = "0.6.5" log = { workspace = true } minreq = { workspace = true } diff --git a/crates/brk_cli/src/config.rs b/crates/brk_cli/src/config.rs index 17aa89fe5..d978cfb7c 100644 --- a/crates/brk_cli/src/config.rs +++ b/crates/brk_cli/src/config.rs @@ -42,7 +42,7 @@ pub struct Config { #[arg(long, value_name = "BOOL")] exchanges: Option, - /// Website served by the server (if active), default: default, saved + /// Website served by the server, default: default, saved #[serde(default, deserialize_with = "default_on_error")] #[arg(short, long)] website: Option, @@ -72,24 +72,8 @@ pub struct Config { #[arg(long, value_name = "PASSWORD")] rpcpassword: Option, - /// Delay between runs, default: 0, saved - #[serde(default, deserialize_with = "default_on_error")] - #[arg(long, value_name = "SECONDS")] - delay: Option, - - /// Activate the Model Context Protocol (MCP) endpoint to give LLMs access to BRK (experimental), default: true, saved - #[serde(default, deserialize_with = "default_on_error")] - #[arg(long, value_name = "BOOL")] - mcp: Option, - - /// DEV: Activate watching the selected website's folder for changes, default: false, saved - #[serde(default, deserialize_with = "default_on_error")] - #[arg(long, value_name = "BOOL")] - watch: Option, - /// DEV: Activate checking address hashes for collisions when indexing, default: false, saved #[serde(default, deserialize_with = "default_on_error")] - #[arg(long, value_name = "BOOL")] check_collisions: Option, } @@ -150,22 +134,10 @@ impl Config { config_saved.rpcpassword = Some(rpcpassword); } - if let Some(delay) = config_args.delay.take() { - config_saved.delay = Some(delay); - } - if let Some(check_collisions) = config_args.check_collisions.take() { config_saved.check_collisions = Some(check_collisions); } - if let Some(mcp) = config_args.mcp.take() { - config_saved.mcp = Some(mcp); - } - - if let Some(watch) = config_args.watch.take() { - config_saved.watch = Some(watch); - } - if config_args != Config::default() { dbg!(config_args); panic!("Didn't consume the full config") @@ -258,10 +230,6 @@ Finally, you can run the program with '-h' for help." self.rpcport } - pub fn delay(&self) -> Option { - self.delay - } - pub fn bitcoindir(&self) -> PathBuf { self.bitcoindir .as_ref() @@ -334,14 +302,6 @@ Finally, you can run the program with '-h' for help." pub fn check_collisions(&self) -> bool { self.check_collisions.is_some_and(|b| b) } - - pub fn mcp(&self) -> bool { - self.mcp.is_none_or(|b| b) - } - - pub fn watch(&self) -> bool { - self.watch.is_some_and(|b| b) - } } fn default_on_error<'de, D, T>(deserializer: D) -> Result diff --git a/crates/brk_cli/src/lib.rs b/crates/brk_cli/src/lib.rs index 0b419b3b0..19f33e0ca 100644 --- a/crates/brk_cli/src/lib.rs +++ b/crates/brk_cli/src/lib.rs @@ -29,7 +29,7 @@ pub fn main() -> color_eyre::Result<()> { fs::create_dir_all(dot_brk_path())?; - brk_logger::init(Some(&dot_brk_log_path())); + brk_logger::init(Some(&dot_brk_log_path()))?; thread::Builder::new() .stack_size(256 * 1024 * 1024) @@ -108,9 +108,7 @@ pub fn run() -> color_eyre::Result<()> { interface.generate_bridge_file(website, websites_path.as_path())?; - let watch = config.watch(); - - Some(bundle(&websites_path, website.to_folder_name(), watch).await?) + Some(bundle(&websites_path, website.to_folder_name(), true).await?) } else { None }; @@ -120,10 +118,8 @@ pub fn run() -> color_eyre::Result<()> { bundle_path, ); - let mcp = config.mcp(); - tokio::spawn(async move { - server.serve(mcp).await.unwrap(); + server.serve(true).await.unwrap(); }); sleep(Duration::from_secs(1)); @@ -140,10 +136,6 @@ pub fn run() -> color_eyre::Result<()> { computer.compute(&indexer, starting_indexes, &exit).unwrap(); - if let Some(delay) = config.delay() { - sleep(Duration::from_secs(delay)) - } - info!("Waiting for new blocks..."); while block_count == rpc.get_block_count()? { diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index 90a34d99f..b3fa642e6 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -21,6 +21,7 @@ brk_parser = { workspace = true } vecdb = { workspace = true } derive_deref = { workspace = true } log = { workspace = true } +pco = "0.4.6" rayon = { workspace = true } serde = { workspace = true } zerocopy = { workspace = true } diff --git a/crates/brk_computer/examples/computer.rs b/crates/brk_computer/examples/computer.rs index 240acdbf8..56a3f3aa5 100644 --- a/crates/brk_computer/examples/computer.rs +++ b/crates/brk_computer/examples/computer.rs @@ -12,7 +12,7 @@ use brk_parser::Parser; use vecdb::Exit; pub fn main() -> Result<()> { - brk_logger::init(Some(Path::new(".log"))); + brk_logger::init(Some(Path::new(".log")))?; let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap()) .join("Library") diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 0710681c2..d3f8e6720 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -45,7 +45,7 @@ pub struct Computer { pub cointime: cointime::Vecs, } -const VERSION: Version = Version::ONE; +const VERSION: Version = Version::TWO; impl Computer { /// Do NOT import multiple times or things will break !!! diff --git a/crates/brk_computer/src/stateful/address_cohort.rs b/crates/brk_computer/src/stateful/address_cohort.rs index 41cc4f44b..6f288e4f2 100644 --- a/crates/brk_computer/src/stateful/address_cohort.rs +++ b/crates/brk_computer/src/stateful/address_cohort.rs @@ -53,12 +53,11 @@ impl Vecs { Ok(Self { starting_height: Height::ZERO, state: states_path.map(|states_path| { - AddressCohortState::default_and_import( + AddressCohortState::new( states_path, cohort_name.unwrap_or_default(), compute_dollars, ) - .unwrap() }), height_to_address_count: EagerVec::forced_import( db, @@ -95,9 +94,6 @@ impl Vecs { impl DynCohortVecs for Vecs { fn starting_height(&self) -> Height { [ - self.state.as_ref().map_or(Height::MAX, |state| { - state.height().map_or(Height::MAX, |h| h.incremented()) - }), self.height_to_address_count.len().into(), self.inner.starting_height(), ] @@ -106,7 +102,7 @@ impl DynCohortVecs for Vecs { .unwrap() } - fn init(&mut self, starting_height: Height) { + fn import_state_at(&mut self, starting_height: Height) -> Result<()> { if starting_height > self.starting_height() { unreachable!() } @@ -120,10 +116,10 @@ impl DynCohortVecs for Vecs { .unwrap_get_inner(prev_height); } - self.inner.init( + self.inner.import_state_at( &mut self.starting_height, &mut self.state.as_mut().unwrap().inner, - ); + ) } fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { diff --git a/crates/brk_computer/src/stateful/address_cohorts.rs b/crates/brk_computer/src/stateful/address_cohorts.rs index 561cec027..bd0e9d220 100644 --- a/crates/brk_computer/src/stateful/address_cohorts.rs +++ b/crates/brk_computer/src/stateful/address_cohorts.rs @@ -26,7 +26,7 @@ impl Vecs { pub fn forced_import( db: &Database, version: Version, - _computation: Computation, + computation: Computation, format: Format, indexes: &indexes::Vecs, price: Option<&price::Vecs>, @@ -38,7 +38,7 @@ impl Vecs { _0sats: address_cohort::Vecs::forced_import( db, Some("addrs_with_0sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -49,7 +49,7 @@ impl Vecs { _1sat_to_10sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_1sat_under_10sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -60,7 +60,7 @@ impl Vecs { _10sats_to_100sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_10sats_under_100sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -71,7 +71,7 @@ impl Vecs { _100sats_to_1k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_100sats_under_1k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -82,7 +82,7 @@ impl Vecs { _1k_sats_to_10k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_1k_sats_under_10k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -93,7 +93,7 @@ impl Vecs { _10k_sats_to_100k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_10k_sats_under_100k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -104,7 +104,7 @@ impl Vecs { _100k_sats_to_1m_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_100k_sats_under_1m_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -115,7 +115,7 @@ impl Vecs { _1m_sats_to_10m_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_1m_sats_under_10m_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -126,7 +126,7 @@ impl Vecs { _10m_sats_to_1btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_10m_sats_under_1btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -137,7 +137,7 @@ impl Vecs { _1btc_to_10btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_1btc_under_10btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -148,7 +148,7 @@ impl Vecs { _10btc_to_100btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_10btc_under_100btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -159,7 +159,7 @@ impl Vecs { _100btc_to_1k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_100btc_under_1k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -170,7 +170,7 @@ impl Vecs { _1k_btc_to_10k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_1k_btc_under_10k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -181,7 +181,7 @@ impl Vecs { _10k_btc_to_100k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_10k_btc_under_100k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -192,7 +192,7 @@ impl Vecs { _100k_btc_or_more: address_cohort::Vecs::forced_import( db, Some("addrs_above_100k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -205,7 +205,7 @@ impl Vecs { _10sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_10sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -216,7 +216,7 @@ impl Vecs { _100sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_100sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -227,7 +227,7 @@ impl Vecs { _1k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_1k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -238,7 +238,7 @@ impl Vecs { _10k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_10k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -249,7 +249,7 @@ impl Vecs { _100k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_100k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -260,7 +260,7 @@ impl Vecs { _1m_sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_1m_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -271,7 +271,7 @@ impl Vecs { _10m_sats: address_cohort::Vecs::forced_import( db, Some("addrs_under_10m_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -282,7 +282,7 @@ impl Vecs { _1btc: address_cohort::Vecs::forced_import( db, Some("addrs_under_1btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -293,7 +293,7 @@ impl Vecs { _10btc: address_cohort::Vecs::forced_import( db, Some("addrs_under_10btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -304,7 +304,7 @@ impl Vecs { _100btc: address_cohort::Vecs::forced_import( db, Some("addrs_under_100btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -315,7 +315,7 @@ impl Vecs { _1k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_under_1k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -326,7 +326,7 @@ impl Vecs { _10k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_under_10k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -337,7 +337,7 @@ impl Vecs { _100k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_under_100k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -350,7 +350,7 @@ impl Vecs { _1sat: address_cohort::Vecs::forced_import( db, Some("addrs_above_1sat"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -361,7 +361,7 @@ impl Vecs { _10sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_10sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -372,7 +372,7 @@ impl Vecs { _100sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_100sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -383,7 +383,7 @@ impl Vecs { _1k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_1k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -394,7 +394,7 @@ impl Vecs { _10k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_10k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -405,7 +405,7 @@ impl Vecs { _100k_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_100k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -416,7 +416,7 @@ impl Vecs { _1m_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_1m_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -427,7 +427,7 @@ impl Vecs { _10m_sats: address_cohort::Vecs::forced_import( db, Some("addrs_above_10m_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -438,7 +438,7 @@ impl Vecs { _1btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_1btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -449,7 +449,7 @@ impl Vecs { _10btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_10btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -460,7 +460,7 @@ impl Vecs { _100btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_100btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -471,7 +471,7 @@ impl Vecs { _1k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_1k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -482,7 +482,7 @@ impl Vecs { _10k_btc: address_cohort::Vecs::forced_import( db, Some("addrs_above_10k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, diff --git a/crates/brk_computer/src/stateful/addresstype/indexes_to_addresscount.rs b/crates/brk_computer/src/stateful/addresstype/indexes_to_addresscount.rs index 98076a67e..3e053cbe6 100644 --- a/crates/brk_computer/src/stateful/addresstype/indexes_to_addresscount.rs +++ b/crates/brk_computer/src/stateful/addresstype/indexes_to_addresscount.rs @@ -19,7 +19,6 @@ impl From>> for AddressTypeToInd impl AddressTypeToIndexesToAddressCount { pub fn compute( &mut self, - // height: Height, indexes: &indexes::Vecs, starting_indexes: &Indexes, exit: &Exit, @@ -75,7 +74,7 @@ impl AddressTypeToIndexesToAddressCount { )?; Ok(()) } - // + pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> { self.0 .as_typed_vec() diff --git a/crates/brk_computer/src/stateful/common.rs b/crates/brk_computer/src/stateful/common.rs index e0e70ce46..d3fa30ad7 100644 --- a/crates/brk_computer/src/stateful/common.rs +++ b/crates/brk_computer/src/stateful/common.rs @@ -1,4 +1,4 @@ -use brk_error::Result; +use brk_error::{Error, Result}; use brk_indexer::Indexer; use brk_structs::{ Bitcoin, DateIndex, Dollars, Height, Sats, StoredF32, StoredF64, StoredU64, Version, @@ -1111,7 +1111,11 @@ impl Vecs { .unwrap() } - pub fn init(&mut self, starting_height: &mut Height, state: &mut CohortState) { + pub fn import_state_at( + &mut self, + starting_height: &mut Height, + state: &mut CohortState, + ) -> Result<()> { if let Some(prev_height) = starting_height.decremented() { state.supply.value = self .height_to_supply @@ -1126,7 +1130,12 @@ impl Vecs { state.realized.as_mut().unwrap().cap = height_to_realized_cap .into_iter() .unwrap_get_inner(prev_height); + + state.import_at(prev_height)?; } + Ok(()) + } else { + Err(Error::Str("Unset")) } } diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index 1ae40f6fa..78c50e67f 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -44,19 +44,12 @@ const VERSION: Version = Version::new(21); pub struct Vecs { db: Database, + // --- + // States + // --- + + // Rollback: diff on stamped_flush + add rollback to stamp pub chain_state: RawVec, - - pub height_to_unspendable_supply: EagerVec, - pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight, - pub height_to_opreturn_supply: EagerVec, - pub indexes_to_opreturn_supply: ComputedValueVecsFromHeight, - pub addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount, - pub addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount, - pub addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount, - pub addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount, - pub utxo_cohorts: utxo_cohorts::Vecs, - pub address_cohorts: address_cohorts::Vecs, - pub p2pk33addressindex_to_anyaddressindex: RawVec, pub p2pk65addressindex_to_anyaddressindex: RawVec, pub p2pkhaddressindex_to_anyaddressindex: RawVec, @@ -68,6 +61,22 @@ pub struct Vecs { pub loadedaddressindex_to_loadedaddressdata: RawVec, pub emptyaddressindex_to_emptyaddressdata: RawVec, + // Rollback: inner state: save price_to_amount_STAMP instead of price_to_amount + pub utxo_cohorts: utxo_cohorts::Vecs, + pub address_cohorts: address_cohorts::Vecs, + + pub height_to_unspendable_supply: EagerVec, + pub height_to_opreturn_supply: EagerVec, + pub addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount, + pub addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount, + + // --- + // Computed + // --- + pub addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount, + pub addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount, + pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight, + pub indexes_to_opreturn_supply: ComputedValueVecsFromHeight, pub indexes_to_address_count: ComputedVecsFromHeight, pub indexes_to_empty_address_count: ComputedVecsFromHeight, } @@ -678,43 +687,57 @@ impl Vecs { }; let starting_height = starting_indexes.height.min(stateful_starting_height); - - if starting_height.is_zero() { - info!("Starting processing utxos from the start"); - - // TODO: rollback instead - - chain_state = vec![]; - chain_state_starting_height = Height::ZERO; - - self.p2pk33addressindex_to_anyaddressindex.reset()?; - self.p2pk65addressindex_to_anyaddressindex.reset()?; - self.p2pkhaddressindex_to_anyaddressindex.reset()?; - self.p2shaddressindex_to_anyaddressindex.reset()?; - self.p2traddressindex_to_anyaddressindex.reset()?; - self.p2wpkhaddressindex_to_anyaddressindex.reset()?; - self.p2wshaddressindex_to_anyaddressindex.reset()?; - self.p2aaddressindex_to_anyaddressindex.reset()?; - self.loadedaddressindex_to_loadedaddressdata.reset()?; - self.emptyaddressindex_to_emptyaddressdata.reset()?; - - info!("Resetting utxo price maps..."); - - separate_utxo_vecs - .par_iter_mut() - .flat_map(|(_, v)| v.state.as_mut()) - .try_for_each(|state| state.reset_price_to_amount())?; - - info!("Resetting address price maps..."); - - separate_address_vecs - .par_iter_mut() - .try_for_each(|(_, v)| v.state.as_mut().unwrap().reset_price_to_amount())?; - }; - let last_height = Height::from(indexer.vecs.height_to_blockhash.stamp()); - if starting_height <= last_height { + let starting_height = if separate_utxo_vecs + .par_iter_mut() + .try_for_each(|(_, v)| v.import_state_at(starting_height)) + .is_err() + || separate_address_vecs + .par_iter_mut() + .try_for_each(|(_, v)| v.import_state_at(starting_height)) + .is_err() + { + Height::ZERO + } else { + starting_height + }; + + if starting_height.is_zero() { + info!("Starting processing utxos from the start"); + + chain_state = vec![]; + chain_state_starting_height = Height::ZERO; + + self.p2pk33addressindex_to_anyaddressindex.reset()?; + self.p2pk65addressindex_to_anyaddressindex.reset()?; + self.p2pkhaddressindex_to_anyaddressindex.reset()?; + self.p2shaddressindex_to_anyaddressindex.reset()?; + self.p2traddressindex_to_anyaddressindex.reset()?; + self.p2wpkhaddressindex_to_anyaddressindex.reset()?; + self.p2wshaddressindex_to_anyaddressindex.reset()?; + self.p2aaddressindex_to_anyaddressindex.reset()?; + self.loadedaddressindex_to_loadedaddressdata.reset()?; + self.emptyaddressindex_to_emptyaddressdata.reset()?; + + info!("Resetting utxo price maps..."); + + separate_utxo_vecs + .par_iter_mut() + .flat_map(|(_, v)| v.state.as_mut()) + .try_for_each(|state| state.reset_price_to_amount_if_needed())?; + + info!("Resetting address price maps..."); + + separate_address_vecs + .par_iter_mut() + .try_for_each(|(_, v)| { + v.state.as_mut().unwrap().reset_price_to_amount_if_needed() + })?; + } + + starting_indexes.update_from_height(starting_height, indexes); + let inputindex_to_outputindex_reader = inputindex_to_outputindex.create_reader(); let outputindex_to_value_reader = outputindex_to_value.create_reader(); let outputindex_to_outputtype_reader = outputindex_to_outputtype.create_reader(); @@ -746,16 +769,6 @@ impl Vecs { let mut dateindex_to_first_height_iter = dateindex_to_first_height.into_iter(); let mut dateindex_to_height_count_iter = dateindex_to_height_count.into_iter(); - starting_indexes.update_from_height(starting_height, indexes); - - separate_utxo_vecs - .par_iter_mut() - .for_each(|(_, v)| v.init(starting_height)); - - separate_address_vecs - .par_iter_mut() - .for_each(|(_, v)| v.init(starting_height)); - let height_to_close_vec = height_to_close.map(|height_to_close| height_to_close.collect().unwrap()); diff --git a/crates/brk_computer/src/stateful/trait.rs b/crates/brk_computer/src/stateful/trait.rs index 9018d0098..4fe79803a 100644 --- a/crates/brk_computer/src/stateful/trait.rs +++ b/crates/brk_computer/src/stateful/trait.rs @@ -8,7 +8,7 @@ use crate::{Indexes, indexes, market, price}; pub trait DynCohortVecs: Send + Sync { fn starting_height(&self) -> Height; - fn init(&mut self, starting_height: Height); + fn import_state_at(&mut self, starting_height: Height) -> Result<()>; fn validate_computed_versions(&mut self, base_version: Version) -> Result<()>; diff --git a/crates/brk_computer/src/stateful/utxo_cohort.rs b/crates/brk_computer/src/stateful/utxo_cohort.rs index d0b01576f..ab7d4617d 100644 --- a/crates/brk_computer/src/stateful/utxo_cohort.rs +++ b/crates/brk_computer/src/stateful/utxo_cohort.rs @@ -15,7 +15,7 @@ use crate::{ #[derive(Clone)] pub struct Vecs { - starting_height: Height, + starting_height: Option, pub state: Option, @@ -40,15 +40,14 @@ impl Vecs { let compute_dollars = price.is_some(); Ok(Self { - starting_height: Height::ZERO, + starting_height: None, state: states_path.map(|states_path| { - UTXOCohortState::default_and_import( + UTXOCohortState::new( states_path, cohort_name.unwrap_or_default(), compute_dollars, ) - .unwrap() }), inner: common::Vecs::forced_import( @@ -69,26 +68,20 @@ impl Vecs { impl DynCohortVecs for Vecs { fn starting_height(&self) -> Height { - [ - self.state.as_ref().map_or(Height::MAX, |state| { - state.height().map_or(Height::MAX, |h| h.incremented()) - }), - self.inner.starting_height(), - ] - .into_iter() - .min() - .unwrap() + self.inner.starting_height() } - fn init(&mut self, starting_height: Height) { + fn import_state_at(&mut self, starting_height: Height) -> Result<()> { if starting_height > self.starting_height() { unreachable!() } - self.starting_height = starting_height; + self.starting_height = Some(starting_height); - self.inner - .init(&mut self.starting_height, self.state.as_mut().unwrap()); + self.inner.import_state_at( + self.starting_height.as_mut().unwrap(), + self.state.as_mut().unwrap(), + ) } fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> { @@ -96,7 +89,7 @@ impl DynCohortVecs for Vecs { } fn forced_pushed_at(&mut self, height: Height, exit: &Exit) -> Result<()> { - if self.starting_height > height { + if self.starting_height.unwrap() > height { return Ok(()); } diff --git a/crates/brk_computer/src/stateful/utxo_cohorts.rs b/crates/brk_computer/src/stateful/utxo_cohorts.rs index b32890410..4e004d429 100644 --- a/crates/brk_computer/src/stateful/utxo_cohorts.rs +++ b/crates/brk_computer/src/stateful/utxo_cohorts.rs @@ -27,7 +27,7 @@ impl Vecs { pub fn forced_import( db: &Database, version: Version, - _computation: Computation, + computation: Computation, format: Format, indexes: &indexes::Vecs, price: Option<&price::Vecs>, @@ -38,7 +38,7 @@ impl Vecs { all: utxo_cohort::Vecs::forced_import( db, None, - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -52,7 +52,7 @@ impl Vecs { short: utxo_cohort::Vecs::forced_import( db, Some("short_term_holders"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -65,7 +65,7 @@ impl Vecs { long: utxo_cohort::Vecs::forced_import( db, Some("long_term_holders"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -80,7 +80,7 @@ impl Vecs { _0: utxo_cohort::Vecs::forced_import( db, Some("epoch_0"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -93,7 +93,7 @@ impl Vecs { _1: utxo_cohort::Vecs::forced_import( db, Some("epoch_1"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -106,7 +106,7 @@ impl Vecs { _2: utxo_cohort::Vecs::forced_import( db, Some("epoch_2"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -119,7 +119,7 @@ impl Vecs { _3: utxo_cohort::Vecs::forced_import( db, Some("epoch_3"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -132,7 +132,7 @@ impl Vecs { _4: utxo_cohort::Vecs::forced_import( db, Some("epoch_4"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -147,7 +147,7 @@ impl Vecs { p2pk65: utxo_cohort::Vecs::forced_import( db, Some("p2pk65"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -160,7 +160,7 @@ impl Vecs { p2pk33: utxo_cohort::Vecs::forced_import( db, Some("p2pk33"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -173,7 +173,7 @@ impl Vecs { p2pkh: utxo_cohort::Vecs::forced_import( db, Some("p2pkh"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -186,7 +186,7 @@ impl Vecs { p2sh: utxo_cohort::Vecs::forced_import( db, Some("p2sh"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -199,7 +199,7 @@ impl Vecs { p2wpkh: utxo_cohort::Vecs::forced_import( db, Some("p2wpkh"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -212,7 +212,7 @@ impl Vecs { p2wsh: utxo_cohort::Vecs::forced_import( db, Some("p2wsh"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -225,7 +225,7 @@ impl Vecs { p2tr: utxo_cohort::Vecs::forced_import( db, Some("p2tr"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -238,7 +238,7 @@ impl Vecs { p2a: utxo_cohort::Vecs::forced_import( db, Some("p2a"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -251,7 +251,7 @@ impl Vecs { p2ms: utxo_cohort::Vecs::forced_import( db, Some("p2ms_outputs"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -264,7 +264,7 @@ impl Vecs { empty: utxo_cohort::Vecs::forced_import( db, Some("empty_outputs"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -277,7 +277,7 @@ impl Vecs { unknown: utxo_cohort::Vecs::forced_import( db, Some("unknown_outputs"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -292,7 +292,7 @@ impl Vecs { _1w: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_1w_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -305,7 +305,7 @@ impl Vecs { _1m: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_1m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -318,7 +318,7 @@ impl Vecs { _2m: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_2m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -331,7 +331,7 @@ impl Vecs { _3m: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_3m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -344,7 +344,7 @@ impl Vecs { _4m: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_4m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -357,7 +357,7 @@ impl Vecs { _5m: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_5m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -370,7 +370,7 @@ impl Vecs { _6m: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_6m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -383,7 +383,7 @@ impl Vecs { _1y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_1y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -396,7 +396,7 @@ impl Vecs { _2y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_2y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -409,7 +409,7 @@ impl Vecs { _3y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_3y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -422,7 +422,7 @@ impl Vecs { _4y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_4y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -435,7 +435,7 @@ impl Vecs { _5y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_5y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -448,7 +448,7 @@ impl Vecs { _6y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_6y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -461,7 +461,7 @@ impl Vecs { _7y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_7y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -474,7 +474,7 @@ impl Vecs { _8y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_8y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -487,7 +487,7 @@ impl Vecs { _10y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_10y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -500,7 +500,7 @@ impl Vecs { _12y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_12y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -513,7 +513,7 @@ impl Vecs { _15y: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_15y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -528,7 +528,7 @@ impl Vecs { _1d: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1d_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -541,7 +541,7 @@ impl Vecs { _1w: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1w_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -554,7 +554,7 @@ impl Vecs { _1m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -567,7 +567,7 @@ impl Vecs { _2m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_2m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -580,7 +580,7 @@ impl Vecs { _3m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_3m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -593,7 +593,7 @@ impl Vecs { _4m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_4m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -606,7 +606,7 @@ impl Vecs { _5m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_5m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -619,7 +619,7 @@ impl Vecs { _6m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_6m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -632,7 +632,7 @@ impl Vecs { _1y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -645,7 +645,7 @@ impl Vecs { _2y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_2y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -658,7 +658,7 @@ impl Vecs { _3y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_3y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -671,7 +671,7 @@ impl Vecs { _4y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_4y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -684,7 +684,7 @@ impl Vecs { _5y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_5y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -697,7 +697,7 @@ impl Vecs { _6y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_6y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -710,7 +710,7 @@ impl Vecs { _7y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_7y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -723,7 +723,7 @@ impl Vecs { _8y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_8y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -736,7 +736,7 @@ impl Vecs { _10y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_10y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -749,7 +749,7 @@ impl Vecs { _12y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_12y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -764,7 +764,7 @@ impl Vecs { up_to_1d: utxo_cohort::Vecs::forced_import( db, Some("utxos_up_to_1d_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -777,7 +777,7 @@ impl Vecs { _1d_to_1w: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1d_up_to_1w_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -790,7 +790,7 @@ impl Vecs { _1w_to_1m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1w_up_to_1m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -803,7 +803,7 @@ impl Vecs { _1m_to_2m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1m_up_to_2m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -816,7 +816,7 @@ impl Vecs { _2m_to_3m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_2m_up_to_3m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -829,7 +829,7 @@ impl Vecs { _3m_to_4m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_3m_up_to_4m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -842,7 +842,7 @@ impl Vecs { _4m_to_5m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_4m_up_to_5m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -855,7 +855,7 @@ impl Vecs { _5m_to_6m: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_5m_up_to_6m_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -868,7 +868,7 @@ impl Vecs { _6m_to_1y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_6m_up_to_1y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -881,7 +881,7 @@ impl Vecs { _1y_to_2y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_1y_up_to_2y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -894,7 +894,7 @@ impl Vecs { _2y_to_3y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_2y_up_to_3y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -907,7 +907,7 @@ impl Vecs { _3y_to_4y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_3y_up_to_4y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -920,7 +920,7 @@ impl Vecs { _4y_to_5y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_4y_up_to_5y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -933,7 +933,7 @@ impl Vecs { _5y_to_6y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_5y_up_to_6y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -946,7 +946,7 @@ impl Vecs { _6y_to_7y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_6y_up_to_7y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -959,7 +959,7 @@ impl Vecs { _7y_to_8y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_7y_up_to_8y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -972,7 +972,7 @@ impl Vecs { _8y_to_10y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_8y_up_to_10y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -985,7 +985,7 @@ impl Vecs { _10y_to_12y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_10y_up_to_12y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -998,7 +998,7 @@ impl Vecs { _12y_to_15y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_12y_up_to_15y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1011,7 +1011,7 @@ impl Vecs { from_15y: utxo_cohort::Vecs::forced_import( db, Some("utxos_at_least_15y_old"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1026,7 +1026,7 @@ impl Vecs { _0sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_with_0sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1039,7 +1039,7 @@ impl Vecs { _1sat_to_10sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1sat_under_10sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1052,7 +1052,7 @@ impl Vecs { _10sats_to_100sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10sats_under_100sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1065,7 +1065,7 @@ impl Vecs { _100sats_to_1k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100sats_under_1k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1078,7 +1078,7 @@ impl Vecs { _1k_sats_to_10k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1k_sats_under_10k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1091,7 +1091,7 @@ impl Vecs { _10k_sats_to_100k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10k_sats_under_100k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1104,7 +1104,7 @@ impl Vecs { _100k_sats_to_1m_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100k_sats_under_1m_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1117,7 +1117,7 @@ impl Vecs { _1m_sats_to_10m_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1m_sats_under_10m_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1130,7 +1130,7 @@ impl Vecs { _10m_sats_to_1btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10m_sats_under_1btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1143,7 +1143,7 @@ impl Vecs { _1btc_to_10btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1btc_under_10btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1156,7 +1156,7 @@ impl Vecs { _10btc_to_100btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10btc_under_100btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1169,7 +1169,7 @@ impl Vecs { _100btc_to_1k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100btc_under_1k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1182,7 +1182,7 @@ impl Vecs { _1k_btc_to_10k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1k_btc_under_10k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1195,7 +1195,7 @@ impl Vecs { _10k_btc_to_100k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10k_btc_under_100k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1208,7 +1208,7 @@ impl Vecs { _100k_btc_or_more: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1223,7 +1223,7 @@ impl Vecs { _10sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_10sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1236,7 +1236,7 @@ impl Vecs { _100sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_100sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1249,7 +1249,7 @@ impl Vecs { _1k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_1k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1262,7 +1262,7 @@ impl Vecs { _10k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_10k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1275,7 +1275,7 @@ impl Vecs { _100k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_100k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1288,7 +1288,7 @@ impl Vecs { _1m_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_1m_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1301,7 +1301,7 @@ impl Vecs { _10m_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_10m_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1314,7 +1314,7 @@ impl Vecs { _1btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_1btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1327,7 +1327,7 @@ impl Vecs { _10btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_10btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1340,7 +1340,7 @@ impl Vecs { _100btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_100btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1353,7 +1353,7 @@ impl Vecs { _1k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_1k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1366,7 +1366,7 @@ impl Vecs { _10k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_10k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1379,7 +1379,7 @@ impl Vecs { _100k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_under_100k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1394,7 +1394,7 @@ impl Vecs { _1sat: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1sat"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1407,7 +1407,7 @@ impl Vecs { _10sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1420,7 +1420,7 @@ impl Vecs { _100sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1433,7 +1433,7 @@ impl Vecs { _1k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1446,7 +1446,7 @@ impl Vecs { _10k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1459,7 +1459,7 @@ impl Vecs { _100k_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100k_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1472,7 +1472,7 @@ impl Vecs { _1m_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1m_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1485,7 +1485,7 @@ impl Vecs { _10m_sats: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10m_sats"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1498,7 +1498,7 @@ impl Vecs { _1btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1511,7 +1511,7 @@ impl Vecs { _10btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1524,7 +1524,7 @@ impl Vecs { _100btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_100btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1537,7 +1537,7 @@ impl Vecs { _1k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_1k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, @@ -1550,7 +1550,7 @@ impl Vecs { _10k_btc: utxo_cohort::Vecs::forced_import( db, Some("utxos_above_10k_btc"), - _computation, + computation, format, version + VERSION + Version::ZERO, indexes, diff --git a/crates/brk_computer/src/states/cohorts/address.rs b/crates/brk_computer/src/states/cohorts/address.rs index 332d6e308..0af649f08 100644 --- a/crates/brk_computer/src/states/cohorts/address.rs +++ b/crates/brk_computer/src/states/cohorts/address.rs @@ -14,19 +14,15 @@ pub struct AddressCohortState { } impl AddressCohortState { - pub fn default_and_import(path: &Path, name: &str, compute_dollars: bool) -> Result { - Ok(Self { + pub fn new(path: &Path, name: &str, compute_dollars: bool) -> Self { + Self { address_count: 0, - inner: CohortState::default_and_import(path, name, compute_dollars)?, - }) + inner: CohortState::new(path, name, compute_dollars), + } } - pub fn height(&self) -> Option { - self.inner.height() - } - - pub fn reset_price_to_amount(&mut self) -> Result<()> { - self.inner.reset_price_to_amount() + pub fn reset_price_to_amount_if_needed(&mut self) -> Result<()> { + self.inner.reset_price_to_amount_if_needed() } pub fn reset_single_iteration_values(&mut self) { diff --git a/crates/brk_computer/src/states/cohorts/common.rs b/crates/brk_computer/src/states/cohorts/common.rs index 1549908c8..36c9e6650 100644 --- a/crates/brk_computer/src/states/cohorts/common.rs +++ b/crates/brk_computer/src/states/cohorts/common.rs @@ -8,38 +8,46 @@ use crate::{PriceToAmount, RealizedState, SupplyState, UnrealizedState}; #[derive(Clone)] pub struct CohortState { pub supply: SupplyState, + pub realized: Option, pub satblocks_destroyed: Sats, pub satdays_destroyed: Sats, - price_to_amount: PriceToAmount, + price_to_amount: Option, } impl CohortState { - pub fn default_and_import(path: &Path, name: &str, compute_dollars: bool) -> Result { - Ok(Self { + pub fn new(path: &Path, name: &str, compute_dollars: bool) -> Self { + Self { supply: SupplyState::default(), realized: compute_dollars.then_some(RealizedState::NAN), satblocks_destroyed: Sats::ZERO, satdays_destroyed: Sats::ZERO, - price_to_amount: PriceToAmount::forced_import(path, name), - }) + price_to_amount: compute_dollars.then_some(PriceToAmount::create(path, name)), + } } - pub fn height(&self) -> Option { - self.price_to_amount.height() + pub fn import_at(&mut self, height: Height) -> Result<()> { + if let Some(price_to_amount) = self.price_to_amount.as_mut() { + price_to_amount.import_at(height)?; + } + Ok(()) } - pub fn reset_price_to_amount(&mut self) -> Result<()> { - self.price_to_amount.reset() + pub fn reset_price_to_amount_if_needed(&mut self) -> Result<()> { + if let Some(price_to_amount) = self.price_to_amount.as_mut() { + price_to_amount.clean()?; + price_to_amount.init(); + } + Ok(()) } pub fn price_to_amount_first_key_value(&self) -> Option<(&Dollars, &Sats)> { - self.price_to_amount.first_key_value() + self.price_to_amount.as_ref().unwrap().first_key_value() } pub fn price_to_amount_last_key_value(&self) -> Option<(&Dollars, &Sats)> { - self.price_to_amount.last_key_value() + self.price_to_amount.as_ref().unwrap().last_key_value() } pub fn reset_single_iteration_values(&mut self) { @@ -58,7 +66,10 @@ impl CohortState { { let price = price.unwrap(); realized.increment(supply_state, price); - self.price_to_amount.increment(price, supply_state); + self.price_to_amount + .as_mut() + .unwrap() + .increment(price, supply_state); } } @@ -74,7 +85,10 @@ impl CohortState { && let Some(realized) = self.realized.as_mut() { realized.increment_(realized_cap); - self.price_to_amount.increment(realized_price, supply_state); + self.price_to_amount + .as_mut() + .unwrap() + .increment(realized_price, supply_state); } } @@ -86,7 +100,10 @@ impl CohortState { { let price = price.unwrap(); realized.decrement(supply_state, price); - self.price_to_amount.decrement(price, supply_state); + self.price_to_amount + .as_mut() + .unwrap() + .decrement(price, supply_state); } } @@ -102,7 +119,10 @@ impl CohortState { && let Some(realized) = self.realized.as_mut() { realized.decrement_(realized_cap); - self.price_to_amount.decrement(realized_price, supply_state); + self.price_to_amount + .as_mut() + .unwrap() + .decrement(realized_price, supply_state); } } @@ -133,13 +153,19 @@ impl CohortState { if let Some((price, supply)) = price_to_amount_increment && supply.value.is_not_zero() { - self.price_to_amount.increment(price, supply); + self.price_to_amount + .as_mut() + .unwrap() + .increment(price, supply); } if let Some((price, supply)) = price_to_amount_decrement && supply.value.is_not_zero() { - self.price_to_amount.decrement(price, supply); + self.price_to_amount + .as_mut() + .unwrap() + .decrement(price, supply); } } } @@ -196,12 +222,18 @@ impl CohortState { if let Some((price, supply)) = price_to_amount_increment && supply.value.is_not_zero() { - self.price_to_amount.increment(price, supply); + self.price_to_amount + .as_mut() + .unwrap() + .increment(price, supply); } if let Some((price, supply)) = price_to_amount_decrement && supply.value.is_not_zero() { - self.price_to_amount.decrement(price, supply); + self.price_to_amount + .as_mut() + .unwrap() + .decrement(price, supply); } } } @@ -212,7 +244,7 @@ impl CohortState { height_price: Dollars, date_price: Option, ) -> (UnrealizedState, Option) { - if self.price_to_amount.is_empty() { + if self.price_to_amount.as_ref().unwrap().is_empty() { return ( UnrealizedState::NAN, date_price.map(|_| UnrealizedState::NAN), @@ -255,23 +287,30 @@ impl CohortState { } }; - self.price_to_amount.iter().for_each(|(&price, &sats)| { - update_state(price, height_price, sats, &mut height_unrealized_state); + self.price_to_amount + .as_ref() + .unwrap() + .iter() + .for_each(|(&price, &sats)| { + update_state(price, height_price, sats, &mut height_unrealized_state); - if let Some(date_price) = date_price { - update_state( - price, - date_price, - sats, - date_unrealized_state.as_mut().unwrap(), - ) - } - }); + if let Some(date_price) = date_price { + update_state( + price, + date_price, + sats, + date_unrealized_state.as_mut().unwrap(), + ) + } + }); (height_unrealized_state, date_unrealized_state) } pub fn commit(&mut self, height: Height) -> Result<()> { - self.price_to_amount.flush(height) + if let Some(price_to_amount) = self.price_to_amount.as_mut() { + price_to_amount.flush(height)?; + } + Ok(()) } } diff --git a/crates/brk_computer/src/states/cohorts/utxo.rs b/crates/brk_computer/src/states/cohorts/utxo.rs index d9a53068d..c633ba41f 100644 --- a/crates/brk_computer/src/states/cohorts/utxo.rs +++ b/crates/brk_computer/src/states/cohorts/utxo.rs @@ -1,7 +1,6 @@ use std::path::Path; use brk_error::Result; -use brk_structs::Height; use derive_deref::{Deref, DerefMut}; use super::CohortState; @@ -10,19 +9,11 @@ use super::CohortState; pub struct UTXOCohortState(CohortState); impl UTXOCohortState { - pub fn default_and_import(path: &Path, name: &str, compute_dollars: bool) -> Result { - Ok(Self(CohortState::default_and_import( - path, - name, - compute_dollars, - )?)) + pub fn new(path: &Path, name: &str, compute_dollars: bool) -> Self { + Self(CohortState::new(path, name, compute_dollars)) } - pub fn height(&self) -> Option { - self.0.height() - } - - pub fn reset_price_to_amount(&mut self) -> Result<()> { - self.0.reset_price_to_amount() + pub fn reset_price_to_amount_if_needed(&mut self) -> Result<()> { + self.0.reset_price_to_amount_if_needed() } } diff --git a/crates/brk_computer/src/states/price_to_amount.rs b/crates/brk_computer/src/states/price_to_amount.rs index f5bfa602a..fae4c7f29 100644 --- a/crates/brk_computer/src/states/price_to_amount.rs +++ b/crates/brk_computer/src/states/price_to_amount.rs @@ -1,75 +1,65 @@ use std::{ collections::BTreeMap, fs, - io::{Cursor, Read}, path::{Path, PathBuf}, + time::SystemTime, }; -use brk_error::{Error, Result}; +use brk_error::Result; use brk_structs::{Dollars, Height, Sats}; use derive_deref::{Deref, DerefMut}; +use pco::standalone::{simple_decompress, simpler_compress}; use serde::{Deserialize, Serialize}; -use zerocopy::{FromBytes, IntoBytes}; use crate::states::SupplyState; #[derive(Clone, Debug)] pub struct PriceToAmount { pathbuf: PathBuf, - height: Option, - state: State, + state: Option, } +const STATE_AT_: &str = "state_at_"; +const STATE_TO_KEEP: usize = 10; + impl PriceToAmount { - pub fn forced_import(path: &Path, name: &str) -> Self { - Self::import(path, name).unwrap_or_else(|_| { - // dbg!(e); - Self { - pathbuf: Self::path_(path, name), - height: None, - state: State::default(), - } - }) + pub fn create(path: &Path, name: &str) -> Self { + Self { + pathbuf: path.join(format!("{name}_price_to_amount")), + state: None, + } } - pub fn import(path: &Path, name: &str) -> Result { - let path = Self::path_(path, name); - fs::create_dir_all(&path)?; - - let state = State::deserialize(&fs::read(Self::path_state_(&path))?)?; - - Ok(Self { - height: Height::try_from(Self::path_height_(&path).as_path()).ok(), - pathbuf: path, - state, - }) + pub fn import_at(&mut self, height: Height) -> Result<()> { + self.state = Some(State::deserialize(&fs::read(self.path_state(height))?)?); + Ok(()) } pub fn iter(&self) -> impl Iterator { - self.state.iter() + self.state.as_ref().unwrap().iter() } pub fn is_empty(&self) -> bool { - self.state.is_empty() + self.state.as_ref().unwrap().is_empty() } pub fn first_key_value(&self) -> Option<(&Dollars, &Sats)> { - self.state.first_key_value() + self.state.as_ref().unwrap().first_key_value() } pub fn last_key_value(&self) -> Option<(&Dollars, &Sats)> { - self.state.last_key_value() + self.state.as_ref().unwrap().last_key_value() } pub fn increment(&mut self, price: Dollars, supply_state: &SupplyState) { - *self.state.entry(price).or_default() += supply_state.value; + *self.state.as_mut().unwrap().entry(price).or_default() += supply_state.value; } pub fn decrement(&mut self, price: Dollars, supply_state: &SupplyState) { - if let Some(amount) = self.state.get_mut(&price) { + if let Some(amount) = self.state.as_mut().unwrap().get_mut(&price) { *amount -= supply_state.value; if *amount == Sats::ZERO { - self.state.remove(&price); + self.state.as_mut().unwrap().remove(&price); } } else { dbg!(&self.state, price, &self.pathbuf); @@ -77,83 +67,89 @@ impl PriceToAmount { } } - pub fn reset(&mut self) -> Result<()> { - self.state.clear(); - self.height = None; - fs::remove_dir_all(&self.pathbuf)?; + pub fn init(&mut self) { + self.state.replace(State::default()); + } + + pub fn clean(&mut self) -> Result<()> { + let _ = fs::remove_dir_all(&self.pathbuf); fs::create_dir_all(&self.pathbuf)?; Ok(()) } pub fn flush(&mut self, height: Height) -> Result<()> { - self.height = Some(height); - height.write(&self.path_height())?; - fs::write(self.path_state(), self.state.serialize())?; + let mut files: Vec<(SystemTime, PathBuf)> = fs::read_dir(&self.pathbuf)? + .filter_map(|entry| { + let path = entry.ok()?.path(); + let name = path.file_name()?.to_str()?; + if name.starts_with(STATE_AT_) && name[STATE_AT_.len() + 1..].parse::().is_ok() + { + let modified = fs::metadata(&path).ok()?.modified().ok()?; + Some((modified, path)) + } else { + None + } + }) + .collect(); + + files.sort_unstable_by_key(|(time, _)| *time); + + for (_, path) in files.iter().take(files.len().saturating_sub(STATE_TO_KEEP)) { + fs::remove_file(path)?; + } + + fs::write( + self.path_state(height), + self.state.as_ref().unwrap().serialize()?, + )?; + Ok(()) } - pub fn height(&self) -> Option { - self.height + fn path_state(&self, height: Height) -> PathBuf { + Self::path_state_(&self.pathbuf, height) } - - fn path_(path: &Path, name: &str) -> PathBuf { - path.join(format!("{name}_price_to_amount")) - } - - fn path_state(&self) -> PathBuf { - Self::path_state_(&self.pathbuf) - } - fn path_state_(path: &Path) -> PathBuf { - path.join("state") - } - - fn path_height(&self) -> PathBuf { - Self::path_height_(&self.pathbuf) - } - fn path_height_(path: &Path) -> PathBuf { - path.join("height") + fn path_state_(path: &Path, height: Height) -> PathBuf { + path.join(format!("{STATE_AT_}{}", height)) } } #[derive(Clone, Default, Debug, Deref, DerefMut, Serialize, Deserialize)] struct State(BTreeMap); +const COMPRESSION_LEVEL: usize = 4; + impl State { - fn serialize(&self) -> Vec { - let len = self.len(); + fn serialize(&self) -> vecdb::Result> { + let keys: Vec = self.keys().cloned().map(f64::from).collect(); + let values: Vec = self.values().cloned().map(u64::from).collect(); - let mut buffer = Vec::with_capacity(8 + len * 16); + let compressed_keys = simpler_compress(&keys, COMPRESSION_LEVEL)?; + let compressed_values = simpler_compress(&values, COMPRESSION_LEVEL)?; - buffer.extend(len.as_bytes()); + let mut buffer = Vec::new(); + buffer.extend(&(keys.len() as u64).to_ne_bytes()); + buffer.extend(&(compressed_keys.len() as u64).to_ne_bytes()); + buffer.extend(compressed_keys); + buffer.extend(compressed_values); - self.iter().for_each(|(key, value)| { - buffer.extend(key.as_bytes()); - buffer.extend(value.as_bytes()); - }); - - buffer + Ok(buffer) } - fn deserialize(data: &[u8]) -> Result { - let mut cursor = Cursor::new(data); - let mut buffer = [0u8; 8]; + fn deserialize(data: &[u8]) -> vecdb::Result { + let entry_count = u64::from_ne_bytes(data[0..8].try_into().unwrap()) as usize; + let keys_len = u64::from_ne_bytes(data[8..16].try_into().unwrap()) as usize; - cursor - .read_exact(&mut buffer) - .map_err(|_| Error::Str("Failed to read entry count"))?; - let entry_count = usize::read_from_bytes(&buffer)?; + let keys: Vec = simple_decompress(&data[16..16 + keys_len])?; + let values: Vec = simple_decompress(&data[16 + keys_len..])?; - let mut map = BTreeMap::new(); + let map: BTreeMap = keys + .into_iter() + .zip(values) + .map(|(k, v)| (Dollars::from(k), Sats::from(v))) + .collect(); - for _ in 0..entry_count { - cursor.read_exact(&mut buffer)?; - let key = Dollars::read_from_bytes(&buffer)?; - - cursor.read_exact(&mut buffer)?; - let value = Sats::read_from_bytes(&buffer)?; - - map.insert(key, value); - } + assert_eq!(map.len(), entry_count); Ok(Self(map)) } diff --git a/crates/brk_fetcher/examples/main.rs b/crates/brk_fetcher/examples/main.rs index cddddc748..3e9e47c35 100644 --- a/crates/brk_fetcher/examples/main.rs +++ b/crates/brk_fetcher/examples/main.rs @@ -3,7 +3,7 @@ use brk_fetcher::{BRK, Binance, Fetcher, Kraken}; use brk_structs::{Date, Height}; fn main() -> Result<()> { - brk_logger::init(None); + brk_logger::init(None)?; let mut brk = BRK::default(); dbg!(brk.get_from_height(Height::new(900_000))?); diff --git a/crates/brk_indexer/examples/indexer.rs b/crates/brk_indexer/examples/indexer.rs index 4ce571403..8b7043068 100644 --- a/crates/brk_indexer/examples/indexer.rs +++ b/crates/brk_indexer/examples/indexer.rs @@ -11,7 +11,7 @@ use brk_parser::Parser; use vecdb::Exit; fn main() -> Result<()> { - brk_logger::init(Some(Path::new(".log"))); + brk_logger::init(Some(Path::new(".log")))?; let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap()) .join("Library") diff --git a/crates/brk_logger/examples/main.rs b/crates/brk_logger/examples/main.rs index 91b658c27..a95a5d6d3 100644 --- a/crates/brk_logger/examples/main.rs +++ b/crates/brk_logger/examples/main.rs @@ -1,10 +1,14 @@ +use std::io; + use log::{debug, error, info, trace}; -fn main() { - brk_logger::init(None); +fn main() -> io::Result<()> { + brk_logger::init(None)?; info!("info"); debug!("debug"); error!("error"); trace!("trace"); + + Ok(()) } diff --git a/crates/brk_logger/src/lib.rs b/crates/brk_logger/src/lib.rs index d7e40c76d..bcf899269 100644 --- a/crates/brk_logger/src/lib.rs +++ b/crates/brk_logger/src/lib.rs @@ -6,7 +6,7 @@ use std::{ fmt::Display, fs::{self, OpenOptions}, - io::Write, + io::{self, Write}, path::Path, }; @@ -15,7 +15,7 @@ use jiff::{Timestamp, tz}; pub use owo_colors::OwoColorize; #[inline] -pub fn init(path: Option<&Path>) { +pub fn init(path: Option<&Path>) -> io::Result<()> { let file = path.map(|path| { let _ = fs::remove_file(path); OpenOptions::new() @@ -71,6 +71,8 @@ pub fn init(path: Option<&Path>) { ) }) .init(); + + Ok(()) } fn write( diff --git a/crates/brk_parser/examples/main.rs b/crates/brk_parser/examples/main.rs index db87bb84a..613258856 100644 --- a/crates/brk_parser/examples/main.rs +++ b/crates/brk_parser/examples/main.rs @@ -1,22 +1,20 @@ use std::path::Path; -use bitcoincore_rpc::{Auth, Client}; +use bitcoincore_rpc::{Auth, Client, Result}; use brk_parser::Parser; use brk_structs::Height; -fn main() { +#[allow(clippy::needless_doctest_main)] +fn main() -> Result<()> { let i = std::time::Instant::now(); let bitcoin_dir = Path::new("").join(""); let brk_dir = Path::new("").join(""); - let rpc = Box::leak(Box::new( - Client::new( - "http://localhost:8332", - Auth::CookieFile(bitcoin_dir.join(".cookie")), - ) - .unwrap(), - )); + let rpc = Box::leak(Box::new(Client::new( + "http://localhost:8332", + Auth::CookieFile(bitcoin_dir.join(".cookie")), + )?)); let start = None; let end = None; @@ -57,4 +55,6 @@ fn main() { ); dbg!(i.elapsed()); + + Ok(()) } diff --git a/crates/brk_server/examples/main.rs b/crates/brk_server/examples/main.rs index 87f0ed4cb..c178be87d 100644 --- a/crates/brk_server/examples/main.rs +++ b/crates/brk_server/examples/main.rs @@ -12,7 +12,7 @@ use brk_server::Server; use vecdb::Exit; pub fn main() -> Result<()> { - brk_logger::init(Some(Path::new(".log"))); + brk_logger::init(Some(Path::new(".log")))?; let process = true;