diff --git a/Cargo.lock b/Cargo.lock index 017d1c938..6844ec5b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -546,6 +546,15 @@ dependencies = [ "brk_types", ] +[[package]] +name = "brk_alloc" +version = "0.1.0-alpha.1" +dependencies = [ + "libmimalloc-sys", + "log", + "mimalloc", +] + [[package]] name = "brk_bencher" version = "0.1.0-alpha.1" @@ -590,6 +599,7 @@ dependencies = [ name = "brk_cli" version = "0.1.0-alpha.1" dependencies = [ + "brk_alloc", "brk_binder", "brk_bundler", "brk_computer", @@ -606,7 +616,6 @@ dependencies = [ "clap", "color-eyre", "log", - "mimalloc", "minreq", "serde", "tokio", @@ -629,6 +638,7 @@ name = "brk_computer" version = "0.1.0-alpha.1" dependencies = [ "bitcoin", + "brk_alloc", "brk_bencher", "brk_error", "brk_fetcher", @@ -644,7 +654,6 @@ dependencies = [ "color-eyre", "derive_deref", "log", - "mimalloc", "pco", "rayon", "rustc-hash", @@ -697,6 +706,7 @@ name = "brk_indexer" version = "0.1.0-alpha.1" dependencies = [ "bitcoin", + "brk_alloc", "brk_bencher", "brk_error", "brk_grouper", @@ -710,7 +720,6 @@ dependencies = [ "color-eyre", "fjall", "log", - "mimalloc", "rayon", "rlimit", "rustc-hash", @@ -1790,6 +1799,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "cty" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" + [[package]] name = "darling" version = "0.21.3" @@ -2935,6 +2950,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870" dependencies = [ "cc", + "cty", "libc", ] @@ -4086,9 +4102,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f59e70c4aef1e55797c2e8fd94a4f2a973fc972cfde0e0b05f683667b0cd39dd" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" [[package]] name = "portable-atomic-util" diff --git a/Cargo.toml b/Cargo.toml index 828ae801d..2261562c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ aide = { version = "0.16.0-alpha.1", features = ["axum-json", "axum-query"] } axum = "0.8.8" bitcoin = { version = "0.32.8", features = ["serde"] } bitcoincore-rpc = "0.19.0" +brk_alloc = { version = "0.1.0-alpha.1", path = "crates/brk_alloc" } brk_bencher = { version = "0.1.0-alpha.1", path = "crates/brk_bencher" } brk_binder = { version = "0.1.0-alpha.1", path = "crates/brk_binder" } brk_bundler = { version = "0.1.0-alpha.1", path = "crates/brk_bundler" } @@ -69,7 +70,6 @@ derive_deref = "1.1.1" fjall = { path = "../fjall" } jiff = "0.2.17" log = "0.4.29" -mimalloc = { version = "0.1.48", features = ["v3"] } minreq = { version = "2.14.1", features = ["https", "serde_json"] } parking_lot = "0.12.5" rayon = "1.11.0" diff --git a/crates/brk_alloc/Cargo.toml b/crates/brk_alloc/Cargo.toml new file mode 100644 index 000000000..d22da7eae --- /dev/null +++ b/crates/brk_alloc/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "brk_alloc" +description = "Global allocator and memory utilities for brk" +version.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true + +[dependencies] +libmimalloc-sys = { version = "0.1.44", features = ["extended"] } +log = { workspace = true } +mimalloc = { version = "0.1.48", features = ["v3"] } diff --git a/crates/brk_alloc/src/lib.rs b/crates/brk_alloc/src/lib.rs new file mode 100644 index 000000000..344d303f2 --- /dev/null +++ b/crates/brk_alloc/src/lib.rs @@ -0,0 +1,146 @@ +//! Global allocator and memory utilities for brk. +//! +//! This crate sets mimalloc as the global allocator and provides +//! utilities for monitoring and managing memory. +//! ``` + +use std::{fmt, mem::MaybeUninit}; + +use log::info; +use mimalloc::MiMalloc as Allocator; + +#[global_allocator] +static GLOBAL: Allocator = Allocator; + +/// Mimalloc allocator utilities +pub struct Mimalloc; + +impl Mimalloc { + /// Get current mimalloc memory statistics. + /// Very fast (~100-500ns) - uses getrusage/mach syscalls, no file I/O. + #[inline] + pub fn stats() -> Stats { + let mut elapsed_msecs = MaybeUninit::uninit(); + let mut user_msecs = MaybeUninit::uninit(); + let mut system_msecs = MaybeUninit::uninit(); + let mut current_rss = MaybeUninit::uninit(); + let mut peak_rss = MaybeUninit::uninit(); + let mut current_commit = MaybeUninit::uninit(); + let mut peak_commit = MaybeUninit::uninit(); + let mut page_faults = MaybeUninit::uninit(); + + unsafe { + libmimalloc_sys::mi_process_info( + elapsed_msecs.as_mut_ptr(), + user_msecs.as_mut_ptr(), + system_msecs.as_mut_ptr(), + current_rss.as_mut_ptr(), + peak_rss.as_mut_ptr(), + current_commit.as_mut_ptr(), + peak_commit.as_mut_ptr(), + page_faults.as_mut_ptr(), + ); + + Stats { + current_rss: current_rss.assume_init(), + peak_rss: peak_rss.assume_init(), + current_commit: current_commit.assume_init(), + peak_commit: peak_commit.assume_init(), + } + } + } + + /// Eagerly free memory back to OS. + /// This is expensive - only call at natural pause points. + #[inline] + pub fn collect() { + unsafe { libmimalloc_sys::mi_collect(true) } + } + + /// Collect if wasted memory exceeds threshold (in MB). + /// Returns true if collection was triggered. + pub fn collect_if_wasted_above(threshold_mb: usize) -> bool { + let stats = Self::stats(); + + info!("Mimalloc stats: {:?}", stats); + + if stats.wasted_mb() > threshold_mb { + info!( + "Mimalloc wasted {} MB (commit: {} MB, rss: {} MB), collecting...", + stats.wasted_mb(), + stats.commit_mb(), + stats.rss_mb(), + ); + Self::collect(); + true + } else { + false + } + } + + /// Force collection and return stats before/after. + pub fn force_collect() -> (Stats, Stats) { + let before = Self::stats(); + Self::collect(); + let after = Self::stats(); + + info!( + "Mimalloc collected: {} MB -> {} MB (freed {} MB)", + before.commit_mb(), + after.commit_mb(), + before.commit_mb().saturating_sub(after.commit_mb()), + ); + + (before, after) + } +} + +/// Memory stats from mimalloc +#[derive(Debug, Clone, Copy)] +pub struct Stats { + /// Resident set size (physical memory used) + pub current_rss: usize, + pub peak_rss: usize, + /// Committed memory (virtual memory reserved) + pub current_commit: usize, + pub peak_commit: usize, +} + +impl Stats { + /// Returns wasted memory in bytes (commit - rss). + /// High values suggest fragmentation. + #[inline] + pub fn wasted(&self) -> usize { + self.current_commit.saturating_sub(self.current_rss) + } + + /// Returns wasted memory in MB. + #[inline] + pub fn wasted_mb(&self) -> usize { + self.wasted() / 1024 / 1024 + } + + /// Returns current RSS in MB. + #[inline] + pub fn rss_mb(&self) -> usize { + self.current_rss / 1024 / 1024 + } + + /// Returns current commit in MB. + #[inline] + pub fn commit_mb(&self) -> usize { + self.current_commit / 1024 / 1024 + } +} + +impl fmt::Display for Stats { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "rss: {} MB, commit: {} MB, wasted: {} MB", + self.rss_mb(), + self.commit_mb(), + self.wasted_mb() + ) + } +} diff --git a/crates/brk_binder/src/javascript.rs b/crates/brk_binder/src/javascript.rs index 118686eca..622862b3d 100644 --- a/crates/brk_binder/src/javascript.rs +++ b/crates/brk_binder/src/javascript.rs @@ -579,6 +579,9 @@ fn field_to_js_type_with_generic_value( return format!("{}<{}>", field.rust_type, type_param); } field.rust_type.clone() + } else if field.is_branch() { + // Non-pattern branch struct + field.rust_type.clone() } else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) { format!("{}<{}>", accessor.name, value_type) } else { diff --git a/crates/brk_binder/src/python.rs b/crates/brk_binder/src/python.rs index d066ce214..c13e36982 100644 --- a/crates/brk_binder/src/python.rs +++ b/crates/brk_binder/src/python.rs @@ -625,6 +625,9 @@ fn field_to_python_type_with_generic_value( return format!("{}[{}]", field.rust_type, type_param); } field.rust_type.clone() + } else if field.is_branch() { + // Non-pattern branch struct + field.rust_type.clone() } else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) { // Leaf with accessor - use value_type as the generic format!("{}[{}]", accessor.name, value_type) @@ -744,7 +747,16 @@ fn generate_tree_class( ) .unwrap(); } + } else if field.is_branch() { + // Non-pattern branch - instantiate the nested class + writeln!( + output, + " self.{}: {} = {}(client, f'{{base_path}}/{}')", + field_name_py, py_type, field.rust_type, field.name + ) + .unwrap(); } else { + // Leaf - use MetricNode let metric_path = if let TreeNode::Leaf(leaf) = child_node { format!("/{}", leaf.name()) } else { diff --git a/crates/brk_binder/src/rust.rs b/crates/brk_binder/src/rust.rs index 6a9e06a3c..98d09e4b4 100644 --- a/crates/brk_binder/src/rust.rs +++ b/crates/brk_binder/src/rust.rs @@ -417,6 +417,9 @@ fn field_to_type_annotation_with_generic( return format!("{}<{}>", field.rust_type, type_param); } field.rust_type.clone() + } else if field.is_branch() { + // Non-pattern branch struct + field.rust_type.clone() } else if let Some(accessor) = metadata.find_index_set_pattern(&field.indexes) { format!("{}<{}>", accessor.name, value_type) } else { @@ -537,7 +540,16 @@ fn generate_tree_node( ) .unwrap(); } + } else if field.is_branch() { + // Non-pattern branch - instantiate the nested struct + writeln!( + output, + " {}: {}::new(client.clone(), &format!(\"{{base_path}}/{}\")),", + field_name, field.rust_type, field.name + ) + .unwrap(); } else { + // Leaf - use MetricNode let metric_path = if let TreeNode::Leaf(leaf) = child_node { format!("/{}", leaf.name()) } else { diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index ffb1c059d..2897e73ea 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -24,8 +24,8 @@ brk_rpc = { workspace = true } brk_server = { workspace = true } clap = { version = "4.5.53", features = ["derive", "string"] } color-eyre = { workspace = true } +brk_alloc = { workspace = true } log = { workspace = true } -mimalloc = { workspace = true } minreq = { workspace = true } serde = { workspace = true } tokio = { workspace = true } diff --git a/crates/brk_cli/src/main.rs b/crates/brk_cli/src/main.rs index 0a2a91eca..9434f7aa7 100644 --- a/crates/brk_cli/src/main.rs +++ b/crates/brk_cli/src/main.rs @@ -17,9 +17,9 @@ use brk_iterator::Blocks; use brk_mempool::Mempool; use brk_query::AsyncQuery; use brk_reader::Reader; +use brk_alloc::Mimalloc; use brk_server::{Server, VERSION}; use log::info; -use mimalloc::MiMalloc; use vecdb::Exit; mod config; @@ -28,9 +28,6 @@ mod website; use crate::{config::Config, paths::*}; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - pub fn main() -> color_eyre::Result<()> { // Can't increase main thread's stack size, thus we need to use another thread thread::Builder::new() @@ -60,6 +57,16 @@ pub fn run() -> color_eyre::Result<()> { let mut indexer = Indexer::forced_import(&config.brkdir())?; + // Pre-run indexer if too far behind, then drop and reimport to reduce memory + let chain_height = client.get_last_height()?; + let indexed_height = indexer.vecs.starting_height(); + if u32::from(chain_height) - u32::from(indexed_height) > 1000 { + indexer.index(&blocks, &client, &exit)?; + drop(indexer); + Mimalloc::collect(); + indexer = Indexer::forced_import(&config.brkdir())?; + } + let mut computer = Computer::forced_import(&config.brkdir(), &indexer, config.fetcher())?; let mempool = Mempool::new(&client); @@ -155,6 +162,8 @@ pub fn run() -> color_eyre::Result<()> { indexer.index(&blocks, &client, &exit)? }; + Mimalloc::collect_if_wasted_above(500); + computer.compute(&indexer, starting_indexes, &reader, &exit)?; info!("Waiting for new blocks..."); diff --git a/crates/brk_computer/Cargo.toml b/crates/brk_computer/Cargo.toml index a70f5f5dd..8675cfc27 100644 --- a/crates/brk_computer/Cargo.toml +++ b/crates/brk_computer/Cargo.toml @@ -32,6 +32,6 @@ smallvec = { workspace = true } vecdb = { workspace = true } [dev-dependencies] +brk_alloc = { workspace = true } brk_bencher = { workspace = true } color-eyre = { workspace = true } -mimalloc = { workspace = true } diff --git a/crates/brk_computer/examples/computer.rs b/crates/brk_computer/examples/computer.rs index 6b6f5a4f7..02eab1299 100644 --- a/crates/brk_computer/examples/computer.rs +++ b/crates/brk_computer/examples/computer.rs @@ -5,6 +5,7 @@ use std::{ time::{Duration, Instant}, }; +use brk_alloc::Mimalloc; use brk_computer::Computer; use brk_error::Result; use brk_fetcher::Fetcher; @@ -12,12 +13,8 @@ use brk_indexer::Indexer; use brk_iterator::Blocks; use brk_reader::Reader; use brk_rpc::{Auth, Client}; -use mimalloc::MiMalloc; use vecdb::Exit; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - pub fn main() -> color_eyre::Result<()> { color_eyre::install()?; @@ -56,11 +53,24 @@ fn run() -> Result<()> { let exit = Exit::new(); exit.set_ctrlc_handler(); + // Pre-run indexer if too far behind, then drop and reimport to reduce memory + let chain_height = client.get_last_height()?; + let indexed_height = indexer.vecs.starting_height(); + if u32::from(chain_height) - u32::from(indexed_height) > 1000 { + indexer.checked_index(&blocks, &client, &exit)?; + drop(indexer); + Mimalloc::collect(); + indexer = Indexer::forced_import(&outputs_dir)?; + } + let mut computer = Computer::forced_import(&outputs_dir, &indexer, Some(fetcher))?; loop { let i = Instant::now(); let starting_indexes = indexer.checked_index(&blocks, &client, &exit)?; + + Mimalloc::collect_if_wasted_above(500); + computer.compute(&indexer, starting_indexes, &reader, &exit)?; dbg!(i.elapsed()); sleep(Duration::from_secs(10)); diff --git a/crates/brk_computer/examples/computer_bench.rs b/crates/brk_computer/examples/computer_bench.rs index 752396053..cd09fa007 100644 --- a/crates/brk_computer/examples/computer_bench.rs +++ b/crates/brk_computer/examples/computer_bench.rs @@ -1,5 +1,6 @@ use std::{env, path::Path, thread, time::Instant}; +use brk_alloc::Mimalloc; use brk_bencher::Bencher; use brk_computer::Computer; use brk_error::Result; @@ -9,12 +10,8 @@ use brk_iterator::Blocks; use brk_reader::Reader; use brk_rpc::{Auth, Client}; use log::{debug, info}; -use mimalloc::MiMalloc; use vecdb::Exit; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - pub fn main() -> Result<()> { // Can't increase main thread's stack size, thus we need to use another thread thread::Builder::new() @@ -65,6 +62,8 @@ fn run() -> Result<()> { let starting_indexes = indexer.index(&blocks, &client, &exit)?; info!("Done in {:?}", i.elapsed()); + Mimalloc::collect_if_wasted_above(500); + let i = Instant::now(); computer.compute(&indexer, starting_indexes, &reader, &exit)?; info!("Done in {:?}", i.elapsed()); diff --git a/crates/brk_computer/examples/computer_read.rs b/crates/brk_computer/examples/computer_read.rs index 2b3d5f262..6187fb9ff 100644 --- a/crates/brk_computer/examples/computer_read.rs +++ b/crates/brk_computer/examples/computer_read.rs @@ -4,12 +4,8 @@ use brk_computer::Computer; use brk_error::Result; use brk_fetcher::Fetcher; use brk_indexer::Indexer; -use mimalloc::MiMalloc; use vecdb::{AnyStoredVec, Exit}; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - pub fn main() -> Result<()> { // Can't increase main thread's stack size, thus we need to use another thread thread::Builder::new() diff --git a/crates/brk_computer/examples/full_bench.rs b/crates/brk_computer/examples/full_bench.rs index b990a7f87..249fd6cf2 100644 --- a/crates/brk_computer/examples/full_bench.rs +++ b/crates/brk_computer/examples/full_bench.rs @@ -5,6 +5,7 @@ use std::{ time::{Duration, Instant}, }; +use brk_alloc::Mimalloc; use brk_bencher::Bencher; use brk_computer::Computer; use brk_error::Result; @@ -14,12 +15,8 @@ use brk_iterator::Blocks; use brk_reader::Reader; use brk_rpc::{Auth, Client}; use log::{debug, info}; -use mimalloc::MiMalloc; use vecdb::Exit; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - pub fn main() -> color_eyre::Result<()> { color_eyre::install()?; @@ -76,6 +73,8 @@ fn run() -> Result<()> { let starting_indexes = indexer.index(&blocks, &client, &exit)?; info!("Done in {:?}", i.elapsed()); + Mimalloc::collect_if_wasted_above(500); + let i = Instant::now(); computer.compute(&indexer, starting_indexes, &reader, &exit)?; info!("Done in {:?}", i.elapsed()); diff --git a/crates/brk_computer/src/constants.rs b/crates/brk_computer/src/constants.rs index 46088b50d..db2686de4 100644 --- a/crates/brk_computer/src/constants.rs +++ b/crates/brk_computer/src/constants.rs @@ -1,258 +1,48 @@ -use std::path::Path; - -use brk_error::Result; use brk_traversable::Traversable; use brk_types::{StoredF32, StoredI16, StoredU16, Version}; -use vecdb::{AnyVec, Database, Exit, PAGE_SIZE}; - -use crate::grouped::Source; use super::{ - Indexes, - grouped::{ComputedVecsFromHeight, VecBuilderOptions}, + grouped::{ConstantVecs, ReturnF32Tenths, ReturnI16, ReturnU16}, indexes, }; -pub const DB_NAME: &str = "constants"; - #[derive(Clone, Traversable)] pub struct Vecs { - db: Database, - - pub constant_0: ComputedVecsFromHeight, - pub constant_1: ComputedVecsFromHeight, - pub constant_2: ComputedVecsFromHeight, - pub constant_3: ComputedVecsFromHeight, - pub constant_4: ComputedVecsFromHeight, - pub constant_38_2: ComputedVecsFromHeight, - pub constant_50: ComputedVecsFromHeight, - pub constant_61_8: ComputedVecsFromHeight, - pub constant_100: ComputedVecsFromHeight, - pub constant_600: ComputedVecsFromHeight, - pub constant_minus_1: ComputedVecsFromHeight, - pub constant_minus_2: ComputedVecsFromHeight, - pub constant_minus_3: ComputedVecsFromHeight, - pub constant_minus_4: ComputedVecsFromHeight, + pub constant_0: ConstantVecs, + pub constant_1: ConstantVecs, + pub constant_2: ConstantVecs, + pub constant_3: ConstantVecs, + pub constant_4: ConstantVecs, + pub constant_38_2: ConstantVecs, + pub constant_50: ConstantVecs, + pub constant_61_8: ConstantVecs, + pub constant_100: ConstantVecs, + pub constant_600: ConstantVecs, + pub constant_minus_1: ConstantVecs, + pub constant_minus_2: ConstantVecs, + pub constant_minus_3: ConstantVecs, + pub constant_minus_4: ConstantVecs, } impl Vecs { - pub fn forced_import( - parent_path: &Path, - parent_version: Version, - indexes: &indexes::Vecs, - ) -> Result { - let db = Database::open(&parent_path.join(DB_NAME))?; - db.set_min_len(PAGE_SIZE * 1_000_000)?; + pub fn new(version: Version, indexes: &indexes::Vecs) -> Self { + let v = version + Version::ZERO; - let version = parent_version + Version::ZERO; - - let this = Self { - constant_0: ComputedVecsFromHeight::forced_import( - &db, - "constant_0", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_1: ComputedVecsFromHeight::forced_import( - &db, - "constant_1", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_2: ComputedVecsFromHeight::forced_import( - &db, - "constant_2", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_3: ComputedVecsFromHeight::forced_import( - &db, - "constant_3", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_4: ComputedVecsFromHeight::forced_import( - &db, - "constant_4", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_38_2: ComputedVecsFromHeight::forced_import( - &db, - "constant_38_2", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_50: ComputedVecsFromHeight::forced_import( - &db, - "constant_50", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_61_8: ComputedVecsFromHeight::forced_import( - &db, - "constant_61_8", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_100: ComputedVecsFromHeight::forced_import( - &db, - "constant_100", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_600: ComputedVecsFromHeight::forced_import( - &db, - "constant_600", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_minus_1: ComputedVecsFromHeight::forced_import( - &db, - "constant_minus_1", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_minus_2: ComputedVecsFromHeight::forced_import( - &db, - "constant_minus_2", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_minus_3: ComputedVecsFromHeight::forced_import( - &db, - "constant_minus_3", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - constant_minus_4: ComputedVecsFromHeight::forced_import( - &db, - "constant_minus_4", - Source::Compute, - version + Version::ZERO, - indexes, - VecBuilderOptions::default().add_last(), - )?, - - db, - }; - - this.db.retain_regions( - this.iter_any_exportable() - .flat_map(|v| v.region_names()) - .collect(), - )?; - this.db.compact()?; - - Ok(this) - } - - pub fn compute( - &mut self, - indexes: &indexes::Vecs, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { - self.compute_(indexes, starting_indexes, exit)?; - let _lock = exit.lock(); - self.db.compact()?; - Ok(()) - } - - fn compute_( - &mut self, - indexes: &indexes::Vecs, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { - [ - (&mut self.constant_0, 0), - (&mut self.constant_1, 1), - (&mut self.constant_2, 2), - (&mut self.constant_3, 3), - (&mut self.constant_4, 4), - (&mut self.constant_50, 50), - (&mut self.constant_100, 100), - (&mut self.constant_600, 600), - ] - .into_iter() - .try_for_each(|(vec, value)| { - vec.compute_all(indexes, starting_indexes, exit, |vec| { - vec.compute_to( - starting_indexes.height, - indexes.height_to_date.len(), - indexes.height_to_date.version(), - |i| (i, StoredU16::new(value)), - exit, - )?; - Ok(()) - }) - })?; - - [ - (&mut self.constant_minus_1, -1), - (&mut self.constant_minus_2, -2), - (&mut self.constant_minus_3, 3), - (&mut self.constant_minus_4, 4), - ] - .into_iter() - .try_for_each(|(vec, value)| { - vec.compute_all(indexes, starting_indexes, exit, |vec| { - vec.compute_to( - starting_indexes.height, - indexes.height_to_date.len(), - indexes.height_to_date.version(), - |i| (i, StoredI16::new(value)), - exit, - )?; - Ok(()) - }) - })?; - - [ - (&mut self.constant_38_2, 38.2), - (&mut self.constant_61_8, 61.8), - ] - .into_iter() - .try_for_each(|(vec, value)| { - vec.compute_all(indexes, starting_indexes, exit, |vec| { - vec.compute_to( - starting_indexes.height, - indexes.height_to_date.len(), - indexes.height_to_date.version(), - |i| (i, StoredF32::from(value)), - exit, - )?; - Ok(()) - }) - })?; - - Ok(()) + Self { + constant_0: ConstantVecs::new::>("constant_0", v, indexes), + constant_1: ConstantVecs::new::>("constant_1", v, indexes), + constant_2: ConstantVecs::new::>("constant_2", v, indexes), + constant_3: ConstantVecs::new::>("constant_3", v, indexes), + constant_4: ConstantVecs::new::>("constant_4", v, indexes), + constant_38_2: ConstantVecs::new::>("constant_38_2", v, indexes), + constant_50: ConstantVecs::new::>("constant_50", v, indexes), + constant_61_8: ConstantVecs::new::>("constant_61_8", v, indexes), + constant_100: ConstantVecs::new::>("constant_100", v, indexes), + constant_600: ConstantVecs::new::>("constant_600", v, indexes), + constant_minus_1: ConstantVecs::new::>("constant_minus_1", v, indexes), + constant_minus_2: ConstantVecs::new::>("constant_minus_2", v, indexes), + constant_minus_3: ConstantVecs::new::>("constant_minus_3", v, indexes), + constant_minus_4: ConstantVecs::new::>("constant_minus_4", v, indexes), + } } } diff --git a/crates/brk_computer/src/grouped/constant.rs b/crates/brk_computer/src/grouped/constant.rs new file mode 100644 index 000000000..d20fba730 --- /dev/null +++ b/crates/brk_computer/src/grouped/constant.rs @@ -0,0 +1,85 @@ +use brk_traversable::Traversable; +use brk_types::{ + DateIndex, DecadeIndex, Height, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, + YearIndex, +}; +use schemars::JsonSchema; +use serde::Serialize; +use vecdb::{Formattable, IterableCloneableVec, LazyVecFrom1, UnaryTransform, VecValue}; + +use crate::indexes; + +/// Lazy constant vecs for all index levels. +/// Uses const generic transforms to return the same value for every index. +#[derive(Clone, Traversable)] +pub struct ConstantVecs +where + T: VecValue + Formattable + Serialize + JsonSchema, +{ + pub height: LazyVecFrom1, + pub dateindex: LazyVecFrom1, + pub weekindex: LazyVecFrom1, + pub monthindex: LazyVecFrom1, + pub quarterindex: LazyVecFrom1, + pub semesterindex: LazyVecFrom1, + pub yearindex: LazyVecFrom1, + pub decadeindex: LazyVecFrom1, +} + +impl ConstantVecs { + /// Create constant vecs using a transform that ignores input and returns a constant. + pub fn new(name: &str, version: Version, indexes: &indexes::Vecs) -> Self + where + F: UnaryTransform + + UnaryTransform + + UnaryTransform + + UnaryTransform + + UnaryTransform + + UnaryTransform + + UnaryTransform + + UnaryTransform, + { + Self { + height: LazyVecFrom1::transformed::( + name, + version, + indexes.height_to_height.boxed_clone(), + ), + dateindex: LazyVecFrom1::transformed::( + name, + version, + indexes.dateindex_to_dateindex.boxed_clone(), + ), + weekindex: LazyVecFrom1::transformed::( + name, + version, + indexes.weekindex_to_weekindex.boxed_clone(), + ), + monthindex: LazyVecFrom1::transformed::( + name, + version, + indexes.monthindex_to_monthindex.boxed_clone(), + ), + quarterindex: LazyVecFrom1::transformed::( + name, + version, + indexes.quarterindex_to_quarterindex.boxed_clone(), + ), + semesterindex: LazyVecFrom1::transformed::( + name, + version, + indexes.semesterindex_to_semesterindex.boxed_clone(), + ), + yearindex: LazyVecFrom1::transformed::( + name, + version, + indexes.yearindex_to_yearindex.boxed_clone(), + ), + decadeindex: LazyVecFrom1::transformed::( + name, + version, + indexes.decadeindex_to_decadeindex.boxed_clone(), + ), + } + } +} diff --git a/crates/brk_computer/src/grouped/lazy_from_dateindex.rs b/crates/brk_computer/src/grouped/lazy_from_dateindex.rs index d93330e3c..839152fa2 100644 --- a/crates/brk_computer/src/grouped/lazy_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/lazy_from_dateindex.rs @@ -9,7 +9,6 @@ use super::{ComputedVecValue, ComputedVecsFromDateIndex, LazyTransformBuilder}; const VERSION: Version = Version::ZERO; -/// Fully lazy version of `ComputedVecsFromDateIndex` where all vecs are lazy transforms. #[derive(Clone)] pub struct LazyVecsFromDateIndex where @@ -42,11 +41,19 @@ where let v = version + VERSION; Self { dateindex: dateindex_source.map(|s| LazyVecFrom1::transformed::(name, v, s)), - dateindex_extra: LazyTransformBuilder::from_eager::(name, v, &source.dateindex_extra), + dateindex_extra: LazyTransformBuilder::from_eager::( + name, + v, + &source.dateindex_extra, + ), weekindex: LazyTransformBuilder::from_lazy::(name, v, &source.weekindex), monthindex: LazyTransformBuilder::from_lazy::(name, v, &source.monthindex), quarterindex: LazyTransformBuilder::from_lazy::(name, v, &source.quarterindex), - semesterindex: LazyTransformBuilder::from_lazy::(name, v, &source.semesterindex), + semesterindex: LazyTransformBuilder::from_lazy::( + name, + v, + &source.semesterindex, + ), yearindex: LazyTransformBuilder::from_lazy::(name, v, &source.yearindex), decadeindex: LazyTransformBuilder::from_lazy::(name, v, &source.decadeindex), } diff --git a/crates/brk_computer/src/grouped/lazy_value_from_dateindex.rs b/crates/brk_computer/src/grouped/lazy_value_from_dateindex.rs new file mode 100644 index 000000000..7e6fa14cf --- /dev/null +++ b/crates/brk_computer/src/grouped/lazy_value_from_dateindex.rs @@ -0,0 +1,65 @@ +use brk_traversable::Traversable; +use brk_types::{Bitcoin, Dollars, Sats, Version}; +use vecdb::{IterableCloneableVec, UnaryTransform}; + +use super::{ComputedValueVecsFromDateIndex, LazyVecsFromDateIndex}; + +const VERSION: Version = Version::ZERO; + +/// Fully lazy version of `ComputedValueVecsFromDateIndex` where all fields are lazy transforms. +/// Used for computed values like supply_half where sources are stored sats and dollars vecs. +#[derive(Clone, Traversable)] +pub struct LazyValueVecsFromDateIndex { + pub sats: LazyVecsFromDateIndex, + pub bitcoin: LazyVecsFromDateIndex, + pub dollars: Option>, +} + +impl LazyValueVecsFromDateIndex { + /// Create lazy dateindex value vecs from source vecs via transforms. + /// + /// - `SatsTransform`: Transform from Sats -> Sats (e.g., HalveSats) + /// - `BitcoinTransform`: Transform from Sats -> Bitcoin (e.g., HalveSatsToBitcoin) + /// - `DollarsTransform`: Transform from Dollars -> Dollars (e.g., HalveDollars) + pub fn from_source( + name: &str, + source: &ComputedValueVecsFromDateIndex, + version: Version, + ) -> Self + where + SatsTransform: UnaryTransform, + BitcoinTransform: UnaryTransform, + DollarsTransform: UnaryTransform, + { + let v = version + VERSION; + + let sats = LazyVecsFromDateIndex::from_computed::( + name, + v + Version::ZERO, + source.sats.dateindex.as_ref().map(|v| v.boxed_clone()), + &source.sats, + ); + + let bitcoin = LazyVecsFromDateIndex::from_computed::( + &format!("{name}_btc"), + v + Version::ZERO, + source.sats.dateindex.as_ref().map(|v| v.boxed_clone()), + &source.sats, + ); + + let dollars = source.dollars.as_ref().map(|dollars_source| { + LazyVecsFromDateIndex::from_computed::( + &format!("{name}_usd"), + v + Version::ZERO, + dollars_source.dateindex.as_ref().map(|v| v.boxed_clone()), + dollars_source, + ) + }); + + Self { + sats, + bitcoin, + dollars, + } + } +} diff --git a/crates/brk_computer/src/grouped/lazy_value_height.rs b/crates/brk_computer/src/grouped/lazy_value_height.rs new file mode 100644 index 000000000..c028b84dd --- /dev/null +++ b/crates/brk_computer/src/grouped/lazy_value_height.rs @@ -0,0 +1,63 @@ +use brk_traversable::Traversable; +use brk_types::{Bitcoin, Close, Dollars, Height, Sats, Version}; +use vecdb::{BinaryTransform, IterableBoxedVec, LazyVecFrom1, LazyVecFrom2, UnaryTransform}; + +const VERSION: Version = Version::ZERO; + +/// Fully lazy version of `ComputedHeightValueVecs` where all fields are lazy transforms. +/// Used for computed values like supply_half where sources are stored sats and dollars vecs. +#[derive(Clone, Traversable)] +pub struct LazyHeightValueVecs { + pub sats: LazyVecFrom1, + pub bitcoin: LazyVecFrom1, + pub dollars: Option, Height, Sats>>, +} + +impl LazyHeightValueVecs { + /// Create lazy height value vecs from sats and price sources via transforms. + /// + /// - `SatsTransform`: Transform from Sats -> Sats (e.g., HalveSats) + /// - `BitcoinTransform`: Transform from Sats -> Bitcoin (e.g., HalveSatsToBitcoin) + /// - `DollarsTransform`: Binary transform from (Close, Sats) -> Dollars (e.g., HalfClosePriceTimesSats) + pub fn from_sources( + name: &str, + sats_source: IterableBoxedVec, + price_source: Option>>, + version: Version, + ) -> Self + where + SatsTransform: UnaryTransform, + BitcoinTransform: UnaryTransform, + DollarsTransform: BinaryTransform, Sats, Dollars>, + { + let v = version + VERSION; + + let sats = LazyVecFrom1::transformed::( + name, + v + Version::ZERO, + sats_source.clone(), + ); + + let bitcoin = LazyVecFrom1::transformed::( + &format!("{name}_btc"), + v + Version::ZERO, + sats_source.clone(), + ); + + // dollars is binary transform: price × sats (with optional halving etc) + let dollars = price_source.map(|price| { + LazyVecFrom2::transformed::( + &format!("{name}_usd"), + v + Version::ZERO, + price, + sats_source, + ) + }); + + Self { + sats, + bitcoin, + dollars, + } + } +} diff --git a/crates/brk_computer/src/grouped/mod.rs b/crates/brk_computer/src/grouped/mod.rs index 9de20bb23..6d4123416 100644 --- a/crates/brk_computer/src/grouped/mod.rs +++ b/crates/brk_computer/src/grouped/mod.rs @@ -3,6 +3,7 @@ mod builder_lazy; mod builder_transform; mod builder_transform2; mod computed; +mod constant; mod computed_from_dateindex; mod computed_from_height; mod computed_from_height_strict; @@ -10,6 +11,8 @@ mod computed_from_txindex; mod lazy2_from_dateindex; mod lazy_from_dateindex; mod lazy_from_height; +mod lazy_value_from_dateindex; +mod lazy_value_height; // mod lazy_from_height_strict; // mod lazy_from_txindex; mod price_percentiles; @@ -27,12 +30,15 @@ pub use builder_lazy::*; pub use builder_transform::*; pub use builder_transform2::*; use computed::*; +pub use constant::*; pub use computed_from_dateindex::*; pub use computed_from_height::*; pub use computed_from_height_strict::*; pub use computed_from_txindex::*; pub use lazy_from_dateindex::*; pub use lazy_from_height::*; +pub use lazy_value_from_dateindex::*; +pub use lazy_value_height::*; pub use lazy2_from_dateindex::*; // pub use lazy_from_height_strict::*; // pub use lazy_from_txindex::*; diff --git a/crates/brk_computer/src/grouped/ratio_from_dateindex.rs b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs index 94849abb6..8d2fbcf7b 100644 --- a/crates/brk_computer/src/grouped/ratio_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/ratio_from_dateindex.rs @@ -1,18 +1,19 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Date, DateIndex, Dollars, StoredF32, Version}; -use vecdb::{PcoVec, +use vecdb::{ AnyStoredVec, AnyVec, CollectableVec, Database, EagerVec, Exit, GenericStoredVec, IterableVec, - TypedVecIterator, VecIndex, + PcoVec, VecIndex, }; use crate::{ Indexes, grouped::{ - ComputedStandardDeviationVecsFromDateIndex, StandardDeviationVecsOptions, source::Source, + ComputedStandardDeviationVecsFromDateIndex, LazyVecsFrom2FromDateIndex, PriceTimesRatio, + StandardDeviationVecsOptions, source::Source, }, indexes, price, - utils::{get_percentile, OptionExt}, + utils::{OptionExt, get_percentile}, }; use super::{ComputedVecsFromDateIndex, VecBuilderOptions}; @@ -30,12 +31,12 @@ pub struct ComputedRatioVecsFromDateIndex { pub ratio_pct5: Option>, pub ratio_pct2: Option>, pub ratio_pct1: Option>, - pub ratio_pct99_usd: Option>, - pub ratio_pct98_usd: Option>, - pub ratio_pct95_usd: Option>, - pub ratio_pct5_usd: Option>, - pub ratio_pct2_usd: Option>, - pub ratio_pct1_usd: Option>, + pub ratio_pct99_usd: Option>, + pub ratio_pct98_usd: Option>, + pub ratio_pct95_usd: Option>, + pub ratio_pct5_usd: Option>, + pub ratio_pct2_usd: Option>, + pub ratio_pct1_usd: Option>, pub ratio_sd: Option, pub ratio_4y_sd: Option, @@ -61,23 +62,60 @@ impl ComputedRatioVecsFromDateIndex { macro_rules! import { ($suffix:expr) => { ComputedVecsFromDateIndex::forced_import( - db, &format!("{name}_{}", $suffix), Source::Compute, v, indexes, opts, - ).unwrap() + db, + &format!("{name}_{}", $suffix), + Source::Compute, + v, + indexes, + opts, + ) + .unwrap() }; } + // Create sources first so lazy vecs can reference them + let price = source.is_compute().then(|| { + ComputedVecsFromDateIndex::forced_import(db, name, Source::Compute, v, indexes, opts) + .unwrap() + }); + macro_rules! import_sd { ($suffix:expr, $days:expr) => { ComputedStandardDeviationVecsFromDateIndex::forced_import( - db, &format!("{name}_{}", $suffix), $days, Source::Compute, v, indexes, + db, + &format!("{name}_{}", $suffix), + $days, + Source::Compute, + v, + indexes, StandardDeviationVecsOptions::default().add_all(), - ).unwrap() + price.as_ref(), + ) + .unwrap() + }; + } + + let ratio_pct99 = extended.then(|| import!("ratio_pct99")); + let ratio_pct98 = extended.then(|| import!("ratio_pct98")); + let ratio_pct95 = extended.then(|| import!("ratio_pct95")); + let ratio_pct5 = extended.then(|| import!("ratio_pct5")); + let ratio_pct2 = extended.then(|| import!("ratio_pct2")); + let ratio_pct1 = extended.then(|| import!("ratio_pct1")); + + // Create lazy usd vecs from price and ratio sources + macro_rules! lazy_usd { + ($ratio:expr, $suffix:expr) => { + price.as_ref().zip($ratio.as_ref()).map(|(p, r)| { + LazyVecsFrom2FromDateIndex::from_computed::( + &format!("{name}_{}", $suffix), + v, + p, + r, + ) + }) }; } Ok(Self { - price: source.is_compute().then(|| { - ComputedVecsFromDateIndex::forced_import(db, name, Source::Compute, v, indexes, opts).unwrap() - }), ratio: import!("ratio"), ratio_1w_sma: extended.then(|| import!("ratio_1w_sma")), ratio_1m_sma: extended.then(|| import!("ratio_1m_sma")), @@ -85,18 +123,19 @@ impl ComputedRatioVecsFromDateIndex { ratio_1y_sd: extended.then(|| import_sd!("ratio_1y", 365)), ratio_2y_sd: extended.then(|| import_sd!("ratio_2y", 2 * 365)), ratio_4y_sd: extended.then(|| import_sd!("ratio_4y", 4 * 365)), - ratio_pct99: extended.then(|| import!("ratio_pct99")), - ratio_pct98: extended.then(|| import!("ratio_pct98")), - ratio_pct95: extended.then(|| import!("ratio_pct95")), - ratio_pct5: extended.then(|| import!("ratio_pct5")), - ratio_pct2: extended.then(|| import!("ratio_pct2")), - ratio_pct1: extended.then(|| import!("ratio_pct1")), - ratio_pct99_usd: extended.then(|| import!("ratio_pct99_usd")), - ratio_pct98_usd: extended.then(|| import!("ratio_pct98_usd")), - ratio_pct95_usd: extended.then(|| import!("ratio_pct95_usd")), - ratio_pct5_usd: extended.then(|| import!("ratio_pct5_usd")), - ratio_pct2_usd: extended.then(|| import!("ratio_pct2_usd")), - ratio_pct1_usd: extended.then(|| import!("ratio_pct1_usd")), + ratio_pct99_usd: lazy_usd!(&ratio_pct99, "ratio_pct99_usd"), + ratio_pct98_usd: lazy_usd!(&ratio_pct98, "ratio_pct98_usd"), + ratio_pct95_usd: lazy_usd!(&ratio_pct95, "ratio_pct95_usd"), + ratio_pct5_usd: lazy_usd!(&ratio_pct5, "ratio_pct5_usd"), + ratio_pct2_usd: lazy_usd!(&ratio_pct2, "ratio_pct2_usd"), + ratio_pct1_usd: lazy_usd!(&ratio_pct1, "ratio_pct1_usd"), + price, + ratio_pct99, + ratio_pct98, + ratio_pct95, + ratio_pct5, + ratio_pct2, + ratio_pct1, }) } @@ -128,9 +167,8 @@ impl ComputedRatioVecsFromDateIndex { ) -> Result<()> { let closes = price.timeindexes_to_price_close.dateindex.u(); - let price = price_opt.unwrap_or_else(|| unsafe { - std::mem::transmute(&self.price.u().dateindex) - }); + let price = + price_opt.unwrap_or_else(|| unsafe { std::mem::transmute(&self.price.u().dateindex) }); self.ratio.compute_all(starting_indexes, exit, |v| { v.compute_transform2( @@ -283,83 +321,18 @@ impl ComputedRatioVecsFromDateIndex { None as Option<&EagerVec>>, )?; - let date_to_price = price_opt.unwrap_or_else(|| unsafe { - std::mem::transmute(&self.price.u().dateindex) - }); - - self.ratio_pct99_usd - .as_mut() - .unwrap() - .compute_all(starting_indexes, exit, |vec| { - let mut iter = self - .ratio_pct99 - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .into_iter(); - vec.compute_transform( - starting_indexes.dateindex, - date_to_price, - |(i, price, ..)| { - let multiplier = iter.get_unwrap(i); - (i, price * multiplier) - }, - exit, - )?; - Ok(()) - })?; - - let compute_usd = - |usd: Option<&mut ComputedVecsFromDateIndex>, - source: Option<&ComputedVecsFromDateIndex>| { - usd.unwrap().compute_all(starting_indexes, exit, |vec| { - let mut iter = source.unwrap().dateindex.u().into_iter(); - vec.compute_transform( - starting_indexes.dateindex, - date_to_price, - |(i, price, ..)| { - let multiplier = iter.get_unwrap(i); - (i, price * multiplier) - }, - exit, - )?; - Ok(()) - }) - }; - - compute_usd(self.ratio_pct1_usd.as_mut(), self.ratio_pct1.as_ref())?; - compute_usd(self.ratio_pct2_usd.as_mut(), self.ratio_pct2.as_ref())?; - compute_usd(self.ratio_pct5_usd.as_mut(), self.ratio_pct5.as_ref())?; - compute_usd(self.ratio_pct95_usd.as_mut(), self.ratio_pct95.as_ref())?; - compute_usd(self.ratio_pct98_usd.as_mut(), self.ratio_pct98.as_ref())?; - compute_usd(self.ratio_pct99_usd.as_mut(), self.ratio_pct99.as_ref())?; - - self.ratio_sd.um().compute_all( - starting_indexes, - exit, - self.ratio.dateindex.u(), - Some(date_to_price), - )?; - self.ratio_4y_sd.um().compute_all( - starting_indexes, - exit, - self.ratio.dateindex.u(), - Some(date_to_price), - )?; - self.ratio_2y_sd.um().compute_all( - starting_indexes, - exit, - self.ratio.dateindex.u(), - Some(date_to_price), - )?; - self.ratio_1y_sd.um().compute_all( - starting_indexes, - exit, - self.ratio.dateindex.u(), - Some(date_to_price), - )?; + self.ratio_sd + .um() + .compute_all(starting_indexes, exit, self.ratio.dateindex.u())?; + self.ratio_4y_sd + .um() + .compute_all(starting_indexes, exit, self.ratio.dateindex.u())?; + self.ratio_2y_sd + .um() + .compute_all(starting_indexes, exit, self.ratio.dateindex.u())?; + self.ratio_1y_sd + .um() + .compute_all(starting_indexes, exit, self.ratio.dateindex.u())?; Ok(()) } diff --git a/crates/brk_computer/src/grouped/sd_from_dateindex.rs b/crates/brk_computer/src/grouped/sd_from_dateindex.rs index c41218fe7..ecfa4056d 100644 --- a/crates/brk_computer/src/grouped/sd_from_dateindex.rs +++ b/crates/brk_computer/src/grouped/sd_from_dateindex.rs @@ -4,13 +4,15 @@ use brk_error::Result; use brk_traversable::Traversable; use brk_types::{Date, DateIndex, Dollars, StoredF32, Version}; use vecdb::{ - AnyStoredVec, AnyVec, BoxedVecIterator, CollectableVec, Database, EagerVec, Exit, - GenericStoredVec, IterableVec, PcoVec, VecIndex, + AnyStoredVec, AnyVec, CollectableVec, Database, EagerVec, Exit, GenericStoredVec, IterableVec, + PcoVec, VecIndex, }; use crate::{Indexes, grouped::source::Source, indexes, utils::OptionExt}; -use super::{ComputedVecsFromDateIndex, VecBuilderOptions}; +use super::{ + ComputedVecsFromDateIndex, LazyVecsFrom2FromDateIndex, PriceTimesRatio, VecBuilderOptions, +}; #[derive(Clone, Traversable)] pub struct ComputedStandardDeviationVecsFromDateIndex { @@ -35,19 +37,19 @@ pub struct ComputedStandardDeviationVecsFromDateIndex { pub m2_5sd: Option>, pub m3sd: Option>, - pub _0sd_usd: Option>, - pub p0_5sd_usd: Option>, - pub p1sd_usd: Option>, - pub p1_5sd_usd: Option>, - pub p2sd_usd: Option>, - pub p2_5sd_usd: Option>, - pub p3sd_usd: Option>, - pub m0_5sd_usd: Option>, - pub m1sd_usd: Option>, - pub m1_5sd_usd: Option>, - pub m2sd_usd: Option>, - pub m2_5sd_usd: Option>, - pub m3sd_usd: Option>, + pub _0sd_usd: Option>, + pub p0_5sd_usd: Option>, + pub p1sd_usd: Option>, + pub p1_5sd_usd: Option>, + pub p2sd_usd: Option>, + pub p2_5sd_usd: Option>, + pub p3sd_usd: Option>, + pub m0_5sd_usd: Option>, + pub m1sd_usd: Option>, + pub m1_5sd_usd: Option>, + pub m2sd_usd: Option>, + pub m2_5sd_usd: Option>, + pub m3sd_usd: Option>, } #[derive(Debug, Default)] @@ -104,6 +106,7 @@ impl ComputedStandardDeviationVecsFromDateIndex { parent_version: Version, indexes: &indexes::Vecs, options: StandardDeviationVecsOptions, + price: Option<&ComputedVecsFromDateIndex>, ) -> Result { let opts = VecBuilderOptions::default().add_last(); let version = parent_version + Version::ONE; @@ -122,36 +125,70 @@ impl ComputedStandardDeviationVecsFromDateIndex { }; } + // Create sources first so lazy vecs can reference them + let sma_vec = sma.is_compute().then(|| import!("sma")); + let p0_5sd = options.bands().then(|| import!("p0_5sd")); + let p1sd = options.bands().then(|| import!("p1sd")); + let p1_5sd = options.bands().then(|| import!("p1_5sd")); + let p2sd = options.bands().then(|| import!("p2sd")); + let p2_5sd = options.bands().then(|| import!("p2_5sd")); + let p3sd = options.bands().then(|| import!("p3sd")); + let m0_5sd = options.bands().then(|| import!("m0_5sd")); + let m1sd = options.bands().then(|| import!("m1sd")); + let m1_5sd = options.bands().then(|| import!("m1_5sd")); + let m2sd = options.bands().then(|| import!("m2sd")); + let m2_5sd = options.bands().then(|| import!("m2_5sd")); + let m3sd = options.bands().then(|| import!("m3sd")); + + // Create lazy USD vecs from price and band sources + macro_rules! lazy_usd { + ($band:expr, $suffix:expr) => { + price + .zip($band.as_ref()) + .filter(|_| options.price_bands()) + .map(|(p, b)| { + LazyVecsFrom2FromDateIndex::from_computed::( + &format!("{name}_{}", $suffix), + version, + p, + b, + ) + }) + }; + } + Ok(Self { days, - sma: sma.is_compute().then(|| import!("sma")), sd: import!("sd"), - p0_5sd: options.bands().then(|| import!("p0_5sd")), - p1sd: options.bands().then(|| import!("p1sd")), - p1_5sd: options.bands().then(|| import!("p1_5sd")), - p2sd: options.bands().then(|| import!("p2sd")), - p2_5sd: options.bands().then(|| import!("p2_5sd")), - p3sd: options.bands().then(|| import!("p3sd")), - m0_5sd: options.bands().then(|| import!("m0_5sd")), - m1sd: options.bands().then(|| import!("m1sd")), - m1_5sd: options.bands().then(|| import!("m1_5sd")), - m2sd: options.bands().then(|| import!("m2sd")), - m2_5sd: options.bands().then(|| import!("m2_5sd")), - m3sd: options.bands().then(|| import!("m3sd")), - _0sd_usd: options.price_bands().then(|| import!("0sd_usd")), - p0_5sd_usd: options.price_bands().then(|| import!("p0_5sd_usd")), - p1sd_usd: options.price_bands().then(|| import!("p1sd_usd")), - p1_5sd_usd: options.price_bands().then(|| import!("p1_5sd_usd")), - p2sd_usd: options.price_bands().then(|| import!("p2sd_usd")), - p2_5sd_usd: options.price_bands().then(|| import!("p2_5sd_usd")), - p3sd_usd: options.price_bands().then(|| import!("p3sd_usd")), - m0_5sd_usd: options.price_bands().then(|| import!("m0_5sd_usd")), - m1sd_usd: options.price_bands().then(|| import!("m1sd_usd")), - m1_5sd_usd: options.price_bands().then(|| import!("m1_5sd_usd")), - m2sd_usd: options.price_bands().then(|| import!("m2sd_usd")), - m2_5sd_usd: options.price_bands().then(|| import!("m2_5sd_usd")), - m3sd_usd: options.price_bands().then(|| import!("m3sd_usd")), zscore: options.zscore().then(|| import!("zscore")), + // Lazy USD vecs + _0sd_usd: lazy_usd!(&sma_vec, "0sd_usd"), + p0_5sd_usd: lazy_usd!(&p0_5sd, "p0_5sd_usd"), + p1sd_usd: lazy_usd!(&p1sd, "p1sd_usd"), + p1_5sd_usd: lazy_usd!(&p1_5sd, "p1_5sd_usd"), + p2sd_usd: lazy_usd!(&p2sd, "p2sd_usd"), + p2_5sd_usd: lazy_usd!(&p2_5sd, "p2_5sd_usd"), + p3sd_usd: lazy_usd!(&p3sd, "p3sd_usd"), + m0_5sd_usd: lazy_usd!(&m0_5sd, "m0_5sd_usd"), + m1sd_usd: lazy_usd!(&m1sd, "m1sd_usd"), + m1_5sd_usd: lazy_usd!(&m1_5sd, "m1_5sd_usd"), + m2sd_usd: lazy_usd!(&m2sd, "m2sd_usd"), + m2_5sd_usd: lazy_usd!(&m2_5sd, "m2_5sd_usd"), + m3sd_usd: lazy_usd!(&m3sd, "m3sd_usd"), + // Stored band sources + sma: sma_vec, + p0_5sd, + p1sd, + p1_5sd, + p2sd, + p2_5sd, + p3sd, + m0_5sd, + m1sd, + m1_5sd, + m2sd, + m2_5sd, + m3sd, }) } @@ -160,7 +197,6 @@ impl ComputedStandardDeviationVecsFromDateIndex { starting_indexes: &Indexes, exit: &Exit, source: &impl CollectableVec, - price_opt: Option<&impl IterableVec>, ) -> Result<()> { let min_date = DateIndex::try_from(Date::MIN_RATIO).unwrap(); @@ -179,17 +215,15 @@ impl ComputedStandardDeviationVecsFromDateIndex { })?; let sma_opt: Option<&EagerVec>> = None; - self.compute_rest(starting_indexes, exit, sma_opt, source, price_opt) + self.compute_rest(starting_indexes, exit, sma_opt, source) } - #[allow(clippy::too_many_arguments)] pub fn compute_rest( &mut self, starting_indexes: &Indexes, exit: &Exit, sma_opt: Option<&impl IterableVec>, source: &impl CollectableVec, - price_opt: Option<&impl IterableVec>, ) -> Result<()> { let sma = sma_opt.unwrap_or_else(|| unsafe { mem::transmute(&self.sma.u().dateindex) }); @@ -371,153 +405,6 @@ impl ComputedStandardDeviationVecsFromDateIndex { })?; } - let Some(price) = price_opt else { - return Ok(()); - }; - - let compute_usd = - |usd: &mut ComputedVecsFromDateIndex, - mut iter: BoxedVecIterator| { - usd.compute_all(starting_indexes, exit, |vec| { - vec.compute_transform( - starting_indexes.dateindex, - price, - |(i, price, ..)| { - let multiplier = iter.get_unwrap(i); - (i, price * multiplier) - }, - exit, - )?; - Ok(()) - }) - }; - - if self._0sd_usd.is_none() { - return Ok(()); - } - - compute_usd(self._0sd_usd.um(), sma.iter())?; - compute_usd( - self.p0_5sd_usd.um(), - self.p0_5sd - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .iter(), - )?; - compute_usd( - self.p1sd_usd.um(), - self.p1sd - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .iter(), - )?; - compute_usd( - self.p1_5sd_usd.um(), - self.p1_5sd - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .iter(), - )?; - compute_usd( - self.p2sd_usd.um(), - self.p2sd - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .iter(), - )?; - compute_usd( - self.p2_5sd_usd.um(), - self.p2_5sd - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .iter(), - )?; - compute_usd( - self.p3sd_usd.um(), - self.p3sd - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .iter(), - )?; - compute_usd( - self.m0_5sd_usd.um(), - self.m0_5sd - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .iter(), - )?; - compute_usd( - self.m1sd_usd.um(), - self.m1sd - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .iter(), - )?; - compute_usd( - self.m1_5sd_usd.um(), - self.m1_5sd - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .iter(), - )?; - compute_usd( - self.m2sd_usd.um(), - self.m2sd - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .iter(), - )?; - compute_usd( - self.m2_5sd_usd.um(), - self.m2_5sd - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .iter(), - )?; - compute_usd( - self.m3sd_usd.um(), - self.m3sd - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap() - .iter(), - )?; - Ok(()) } diff --git a/crates/brk_computer/src/grouped/transforms.rs b/crates/brk_computer/src/grouped/transforms.rs index 5c9f5d673..4ed53e815 100644 --- a/crates/brk_computer/src/grouped/transforms.rs +++ b/crates/brk_computer/src/grouped/transforms.rs @@ -1,4 +1,4 @@ -use brk_types::{Bitcoin, Dollars, Sats, StoredF32, StoredF64}; +use brk_types::{Bitcoin, Close, Dollars, Sats, StoredF32}; use vecdb::{BinaryTransform, UnaryTransform}; /// (Dollars, Dollars) -> Dollars addition @@ -57,16 +57,6 @@ impl UnaryTransform for SatsToBitcoin { } } -/// Sats -> StoredF64 via Bitcoin (for coinblocks/coindays) -pub struct SatsToStoredF64; - -impl UnaryTransform for SatsToStoredF64 { - #[inline(always)] - fn apply(sats: Sats) -> StoredF64 { - StoredF64::from(Bitcoin::from(sats)) - } -} - /// Sats -> Sats/2 (for supply_half) pub struct HalveSats; @@ -76,3 +66,100 @@ impl UnaryTransform for HalveSats { sats / 2 } } + +/// Sats -> Bitcoin/2 (halve then convert to bitcoin) +/// Avoids lazy-from-lazy by combining both transforms +pub struct HalveSatsToBitcoin; + +impl UnaryTransform for HalveSatsToBitcoin { + #[inline(always)] + fn apply(sats: Sats) -> Bitcoin { + Bitcoin::from(sats / 2) + } +} + +/// Dollars -> Dollars/2 (for supply_half_usd) +pub struct HalveDollars; + +impl UnaryTransform for HalveDollars { + #[inline(always)] + fn apply(dollars: Dollars) -> Dollars { + dollars.halved() + } +} + +/// Dollars * StoredF32 -> Dollars (price × ratio) +pub struct PriceTimesRatio; + +impl BinaryTransform for PriceTimesRatio { + #[inline(always)] + fn apply(price: Dollars, ratio: StoredF32) -> Dollars { + price * ratio + } +} + +/// Close * Sats -> Dollars (price × sats / 1e8) +/// Same as PriceTimesSats but accepts Close price source. +pub struct ClosePriceTimesSats; + +impl BinaryTransform, Sats, Dollars> for ClosePriceTimesSats { + #[inline(always)] + fn apply(price: Close, sats: Sats) -> Dollars { + *price * Bitcoin::from(sats) + } +} + +/// Close * Sats -> Dollars/2 (price × sats / 1e8 / 2) +/// Computes halved dollars directly from sats, avoiding lazy-from-lazy chains. +pub struct HalfClosePriceTimesSats; + +impl BinaryTransform, Sats, Dollars> for HalfClosePriceTimesSats { + #[inline(always)] + fn apply(price: Close, sats: Sats) -> Dollars { + (*price * Bitcoin::from(sats)).halved() + } +} + +// === Constant Transforms (using const generics) === + +use brk_types::{StoredI16, StoredU16}; + +/// Returns a constant u16 value, ignoring the input. +pub struct ReturnU16; + +impl UnaryTransform for ReturnU16 { + #[inline(always)] + fn apply(_: S) -> StoredU16 { + StoredU16::new(V) + } +} + +/// Returns a constant i16 value, ignoring the input. +pub struct ReturnI16; + +impl UnaryTransform for ReturnI16 { + #[inline(always)] + fn apply(_: S) -> StoredI16 { + StoredI16::new(V) + } +} + +/// Returns a constant f32 value from tenths (V=382 -> 38.2), ignoring the input. +pub struct ReturnF32Tenths; + +impl UnaryTransform for ReturnF32Tenths { + #[inline(always)] + fn apply(_: S) -> StoredF32 { + StoredF32::from(V as f32 / 10.0) + } +} + +/// Dollars * (V/10) -> Dollars (e.g., V=8 -> * 0.8, V=24 -> * 2.4) +pub struct DollarsTimesTenths; + +impl UnaryTransform for DollarsTimesTenths { + #[inline(always)] + fn apply(d: Dollars) -> Dollars { + d * (V as f64 / 10.0) + } +} \ No newline at end of file diff --git a/crates/brk_computer/src/grouped/value_from_txindex.rs b/crates/brk_computer/src/grouped/value_from_txindex.rs index 6aaaab463..c230ac892 100644 --- a/crates/brk_computer/src/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/grouped/value_from_txindex.rs @@ -111,45 +111,6 @@ impl ComputedValueVecsFromTxindex { }) } - // pub fn compute_all( - // &mut self, - // indexer: &Indexer, - // indexes: &indexes::Vecs, - // price: Option<&marketprice::Vecs>, - // starting_indexes: &Indexes, - // exit: &Exit, - // mut compute: F, - // ) -> Result<()> - // where - // F: FnMut( - // &mut EagerVec>, - // &Indexer, - // &indexes::Vecs, - // &Indexes, - // &Exit, - // ) -> Result<()>, - // { - // compute( - // self.sats.txindex.um(), - // indexer, - // indexes, - // starting_indexes, - // exit, - // )?; - - // let txindex: Option<&PcoVec> = None; - // self.compute_rest( - // indexer, - // indexes, - // fetched, - // starting_indexes, - // exit, - // txindex, - // )?; - - // Ok(()) - // } - pub fn compute_rest( &mut self, indexer: &Indexer, diff --git a/crates/brk_computer/src/grouped/value_height.rs b/crates/brk_computer/src/grouped/value_height.rs index a5adb4a18..ca6096ab7 100644 --- a/crates/brk_computer/src/grouped/value_height.rs +++ b/crates/brk_computer/src/grouped/value_height.rs @@ -1,21 +1,18 @@ use brk_error::Result; use brk_traversable::Traversable; -use brk_types::{Bitcoin, Dollars, Height, Sats, Version}; -use vecdb::{Database, EagerVec, Exit, ImportableVec, IterableCloneableVec, LazyVecFrom1, PcoVec}; - -use crate::{ - Indexes, - grouped::{SatsToBitcoin, Source}, - price, - traits::ComputeFromBitcoin, - utils::OptionExt, +use brk_types::{Bitcoin, Close, Dollars, Height, Sats, Version}; +use vecdb::{ + Database, EagerVec, ImportableVec, IterableBoxedVec, IterableCloneableVec, LazyVecFrom1, + LazyVecFrom2, PcoVec, }; +use crate::grouped::{ClosePriceTimesSats, SatsToBitcoin, Source}; + #[derive(Clone, Traversable)] pub struct ComputedHeightValueVecs { pub sats: Option>>, pub bitcoin: LazyVecFrom1, - pub dollars: Option>>, + pub dollars: Option, Height, Sats>>, } const VERSION: Version = Version::ZERO; @@ -26,74 +23,35 @@ impl ComputedHeightValueVecs { name: &str, source: Source, version: Version, - compute_dollars: bool, + price_source: Option>>, ) -> Result { let sats = source .is_compute() .then(|| EagerVec::forced_import(db, name, version + VERSION + Version::ZERO).unwrap()); - let bitcoin = match &source { - Source::Compute => LazyVecFrom1::transformed::( - &format!("{name}_btc"), + let sats_source: IterableBoxedVec = source + .vec() + .unwrap_or_else(|| sats.as_ref().unwrap().boxed_clone()); + + let bitcoin = LazyVecFrom1::transformed::( + &format!("{name}_btc"), + version + VERSION + Version::ZERO, + sats_source.clone(), + ); + + let dollars = price_source.map(|price| { + LazyVecFrom2::transformed::( + &format!("{name}_usd"), version + VERSION + Version::ZERO, - sats.as_ref().unwrap().boxed_clone(), - ), - Source::Vec(boxed) => LazyVecFrom1::transformed::( - &format!("{name}_btc"), - version + VERSION + Version::ZERO, - boxed.clone(), - ), - Source::None => { - panic!("Source::None not supported for lazy bitcoin - use Source::Vec instead") - } - }; + price, + sats_source.clone(), + ) + }); Ok(Self { sats, bitcoin, - dollars: compute_dollars.then(|| { - EagerVec::forced_import( - db, - &format!("{name}_usd"), - version + VERSION + Version::ZERO, - ) - .unwrap() - }), + dollars, }) } - - pub fn compute_all( - &mut self, - price: Option<&price::Vecs>, - starting_indexes: &Indexes, - exit: &Exit, - mut compute: F, - ) -> Result<()> - where - F: FnMut(&mut EagerVec>) -> Result<()>, - { - compute(self.sats.um())?; - - self.compute_rest(price, starting_indexes, exit)?; - - Ok(()) - } - - pub fn compute_rest( - &mut self, - price: Option<&price::Vecs>, - starting_indexes: &Indexes, - exit: &Exit, - ) -> Result<()> { - if let Some(dollars) = self.dollars.as_mut() { - dollars.compute_from_bitcoin( - starting_indexes.height, - &self.bitcoin, - &price.u().chainindexes_to_price_close.height, - exit, - )?; - } - - Ok(()) - } } diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index b3380a518..7b2c744a6 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -96,11 +96,8 @@ impl Computer { ); let i = Instant::now(); - let (price, constants, market) = thread::scope(|s| -> Result<_> { - let constants_handle = big_thread().spawn_scoped(s, || { - constants::Vecs::forced_import(&computed_path, VERSION, &indexes) - })?; - + let constants = constants::Vecs::new(VERSION, &indexes); + let (price, market) = thread::scope(|s| -> Result<_> { let market_handle = big_thread().spawn_scoped(s, || { market::Vecs::forced_import(&computed_path, VERSION, &indexes) })?; @@ -109,10 +106,9 @@ impl Computer { .is_some() .then(|| price::Vecs::forced_import(&computed_path, VERSION, &indexes).unwrap()); - let constants = constants_handle.join().unwrap()?; let market = market_handle.join().unwrap()?; - Ok((price, constants, market)) + Ok((price, market)) })?; info!("Imported price/constants/market in {:?}", i.elapsed()); @@ -176,7 +172,6 @@ impl Computer { blks::DB_NAME, chain::DB_NAME, cointime::DB_NAME, - constants::DB_NAME, indexes::DB_NAME, market::DB_NAME, pools::DB_NAME, @@ -246,15 +241,6 @@ impl Computer { Ok(()) }); - let constants = scope.spawn(|| -> Result<()> { - info!("Computing constants..."); - let i = Instant::now(); - self.constants - .compute(&self.indexes, &starting_indexes, exit)?; - info!("Computed constants in {:?}", i.elapsed()); - Ok(()) - }); - // Txins must complete before txouts (txouts needs txinindex_to_txoutindex) // and before chain (chain needs txinindex_to_value) info!("Computing txins..."); @@ -291,7 +277,6 @@ impl Computer { } blks.join().unwrap()?; - constants.join().unwrap()?; txouts.join().unwrap()?; Ok(()) })?; diff --git a/crates/brk_computer/src/market/compute.rs b/crates/brk_computer/src/market/compute.rs index 4ccfcc94e..cc86a5ac0 100644 --- a/crates/brk_computer/src/market/compute.rs +++ b/crates/brk_computer/src/market/compute.rs @@ -1,14 +1,13 @@ use std::thread; use brk_error::Result; -use brk_types::{Date, DateIndex, Dollars, StoredF32, StoredU16}; -use vecdb::{EagerVec, Exit, GenericStoredVec, PcoVec, TypedVecIterator, VecIndex}; +use brk_types::{Date, DateIndex, StoredF32, StoredU16}; +use vecdb::{Exit, GenericStoredVec, TypedVecIterator, VecIndex}; use crate::{ - price, + Indexes, price, traits::{ComputeDCAAveragePriceViaLen, ComputeDCAStackViaLen, ComputeDrawdown}, utils::OptionExt, - Indexes, }; use super::Vecs; @@ -576,57 +575,20 @@ impl Vecs { Ok(()) })?; - self.indexes_to_price_200d_sma_x0_8 - .compute_all(starting_indexes, exit, |v| { - v.compute_transform( - starting_indexes.dateindex, - self.indexes_to_price_200d_sma - .price - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap(), - |(i, v, ..)| (i, v * 0.8), - exit, - )?; - Ok(()) - })?; - - self.indexes_to_price_200d_sma_x2_4 - .compute_all(starting_indexes, exit, |v| { - v.compute_transform( - starting_indexes.dateindex, - self.indexes_to_price_200d_sma - .price - .as_ref() - .unwrap() - .dateindex - .as_ref() - .unwrap(), - |(i, v, ..)| (i, v * 2.4), - exit, - )?; - Ok(()) - })?; - self.indexes_to_1d_returns_1w_sd.compute_all( starting_indexes, exit, self._1d_price_returns.dateindex.u(), - None as Option<&EagerVec>>, )?; self.indexes_to_1d_returns_1m_sd.compute_all( starting_indexes, exit, self._1d_price_returns.dateindex.u(), - None as Option<&EagerVec>>, )?; self.indexes_to_1d_returns_1y_sd.compute_all( starting_indexes, exit, self._1d_price_returns.dateindex.u(), - None as Option<&EagerVec>>, )?; self.indexes_to_price_1w_volatility diff --git a/crates/brk_computer/src/market/import.rs b/crates/brk_computer/src/market/import.rs index d31c00459..9af3db537 100644 --- a/crates/brk_computer/src/market/import.rs +++ b/crates/brk_computer/src/market/import.rs @@ -3,12 +3,13 @@ use std::path::Path; use brk_error::Result; use brk_traversable::Traversable; use brk_types::Version; -use vecdb::{Database, EagerVec, ImportableVec, PAGE_SIZE}; +use vecdb::{Database, EagerVec, ImportableVec, IterableCloneableVec, PAGE_SIZE}; use crate::{ grouped::{ ComputedRatioVecsFromDateIndex, ComputedStandardDeviationVecsFromDateIndex, - ComputedVecsFromDateIndex, Source, StandardDeviationVecsOptions, VecBuilderOptions, + ComputedVecsFromDateIndex, DollarsTimesTenths, LazyVecsFromDateIndex, Source, + StandardDeviationVecsOptions, VecBuilderOptions, }, indexes, }; @@ -74,6 +75,7 @@ impl Vecs { version + $v, indexes, StandardDeviationVecsOptions::default(), + None, // No USD conversion for market returns )? }; } @@ -88,6 +90,25 @@ impl Vecs { }; } + let indexes_to_price_200d_sma = ratio_di!("price_200d_sma"); + let price_200d_sma_source = indexes_to_price_200d_sma.price.as_ref().unwrap(); + let indexes_to_price_200d_sma_x2_4 = LazyVecsFromDateIndex::from_computed::< + DollarsTimesTenths<24>, + >( + "price_200d_sma_x2_4", + version + v0, + price_200d_sma_source.dateindex.as_ref().map(|v| v.boxed_clone()), + price_200d_sma_source, + ); + let indexes_to_price_200d_sma_x0_8 = LazyVecsFromDateIndex::from_computed::< + DollarsTimesTenths<8>, + >( + "price_200d_sma_x0_8", + version + v0, + price_200d_sma_source.dateindex.as_ref().map(|v| v.boxed_clone()), + price_200d_sma_source, + ); + let this = Self { height_to_price_ath: eager_h!("price_ath", v0), height_to_price_drawdown: eager_h!("price_drawdown", v0), @@ -112,7 +133,7 @@ impl Vecs { indexes_to_price_55d_sma: ratio_di!("price_55d_sma"), indexes_to_price_89d_sma: ratio_di!("price_89d_sma"), indexes_to_price_144d_sma: ratio_di!("price_144d_sma"), - indexes_to_price_200d_sma: ratio_di!("price_200d_sma"), + indexes_to_price_200d_sma, indexes_to_price_1y_sma: ratio_di!("price_1y_sma"), indexes_to_price_2y_sma: ratio_di!("price_2y_sma"), indexes_to_price_200w_sma: ratio_di!("price_200w_sma"), @@ -247,8 +268,8 @@ impl Vecs { dca_class_2016_returns: computed_di!("dca_class_2016_returns"), dca_class_2015_returns: computed_di!("dca_class_2015_returns"), - indexes_to_price_200d_sma_x2_4: computed_di!("price_200d_sma_x2_4"), - indexes_to_price_200d_sma_x0_8: computed_di!("price_200d_sma_x0_8"), + indexes_to_price_200d_sma_x2_4, + indexes_to_price_200d_sma_x0_8, dateindex_to_price_true_range: eager_di!("price_true_range", v0), dateindex_to_price_true_range_2w_sum: eager_di!("price_true_range_2w_sum", v0), indexes_to_price_1w_min: computed_di!("price_1w_min", v1), diff --git a/crates/brk_computer/src/market/mod.rs b/crates/brk_computer/src/market/mod.rs index e21dc3a1a..e7580eaca 100644 --- a/crates/brk_computer/src/market/mod.rs +++ b/crates/brk_computer/src/market/mod.rs @@ -7,7 +7,7 @@ use vecdb::{Database, EagerVec, PcoVec}; use crate::grouped::{ ComputedRatioVecsFromDateIndex, ComputedStandardDeviationVecsFromDateIndex, - ComputedVecsFromDateIndex, + ComputedVecsFromDateIndex, LazyVecsFromDateIndex, }; pub const DB_NAME: &str = "market"; @@ -74,8 +74,8 @@ pub struct Vecs { pub indexes_to_price_200w_ema: ComputedRatioVecsFromDateIndex, pub indexes_to_price_4y_ema: ComputedRatioVecsFromDateIndex, - pub indexes_to_price_200d_sma_x2_4: ComputedVecsFromDateIndex, - pub indexes_to_price_200d_sma_x0_8: ComputedVecsFromDateIndex, + pub indexes_to_price_200d_sma_x2_4: LazyVecsFromDateIndex, + pub indexes_to_price_200d_sma_x0_8: LazyVecsFromDateIndex, pub price_1d_ago: ComputedVecsFromDateIndex, pub price_1w_ago: ComputedVecsFromDateIndex, diff --git a/crates/brk_computer/src/stateful/metrics/supply.rs b/crates/brk_computer/src/stateful/metrics/supply.rs index 0736348e7..01b18eb8f 100644 --- a/crates/brk_computer/src/stateful/metrics/supply.rs +++ b/crates/brk_computer/src/stateful/metrics/supply.rs @@ -10,8 +10,9 @@ use vecdb::{ use crate::{ Indexes, grouped::{ - ComputedHeightValueVecs, ComputedValueVecsFromDateIndex, ComputedVecsFromHeight, Source, - VecBuilderOptions, + ComputedHeightValueVecs, ComputedValueVecsFromDateIndex, ComputedVecsFromHeight, + HalfClosePriceTimesSats, HalveDollars, HalveSats, HalveSatsToBitcoin, LazyHeightValueVecs, + LazyValueVecsFromDateIndex, Source, VecBuilderOptions, }, indexes, price, stateful::states::SupplyState, @@ -37,11 +38,11 @@ pub struct SupplyMetrics { /// UTXO count indexed by various dimensions pub indexes_to_utxo_count: ComputedVecsFromHeight, - /// Half of supply value (used for computing median) - pub height_to_supply_half_value: ComputedHeightValueVecs, + /// Half of supply value (used for computing median) - lazy from supply_value + pub height_to_supply_half_value: LazyHeightValueVecs, - /// Half of supply indexed by date - pub indexes_to_supply_half: ComputedValueVecsFromDateIndex, + /// Half of supply indexed by date - lazy from indexes_to_supply + pub indexes_to_supply_half: LazyValueVecsFromDateIndex, } impl SupplyMetrics { @@ -55,27 +56,48 @@ impl SupplyMetrics { let height_to_supply: EagerVec> = EagerVec::forced_import(cfg.db, &cfg.name("supply"), cfg.version + v0)?; + let price_source = cfg + .price + .map(|p| p.chainindexes_to_price_close.height.boxed_clone()); + let height_to_supply_value = ComputedHeightValueVecs::forced_import( cfg.db, &cfg.name("supply"), Source::Vec(height_to_supply.boxed_clone()), cfg.version + v0, - compute_dollars, + price_source.clone(), )?; + let indexes_to_supply = ComputedValueVecsFromDateIndex::forced_import( + cfg.db, + &cfg.name("supply"), + Source::Compute, + cfg.version + v1, + last, + compute_dollars, + cfg.indexes, + )?; + + // Create lazy supply_half from supply sources + let height_to_supply_half_value = + LazyHeightValueVecs::from_sources::( + &cfg.name("supply_half"), + height_to_supply.boxed_clone(), + price_source, + cfg.version + v0, + ); + + let indexes_to_supply_half = + LazyValueVecsFromDateIndex::from_source::( + &cfg.name("supply_half"), + &indexes_to_supply, + cfg.version + v0, + ); + Ok(Self { height_to_supply, height_to_supply_value, - - indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import( - cfg.db, - &cfg.name("supply"), - Source::Compute, - cfg.version + v1, - last, - compute_dollars, - cfg.indexes, - )?, + indexes_to_supply, height_to_utxo_count: EagerVec::forced_import( cfg.db, @@ -92,23 +114,8 @@ impl SupplyMetrics { last, )?, - height_to_supply_half_value: ComputedHeightValueVecs::forced_import( - cfg.db, - &cfg.name("supply_half"), - Source::Compute, - cfg.version + v0, - compute_dollars, - )?, - - indexes_to_supply_half: ComputedValueVecsFromDateIndex::forced_import( - cfg.db, - &cfg.name("supply_half"), - Source::Compute, - cfg.version + v0, - last, - compute_dollars, - cfg.indexes, - )?, + height_to_supply_half_value, + indexes_to_supply_half, }) } @@ -183,9 +190,6 @@ impl SupplyMetrics { starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { - self.height_to_supply_value - .compute_rest(price, starting_indexes, exit)?; - self.indexes_to_supply .compute_all(price, starting_indexes, exit, |v| { let mut dateindex_to_height_count_iter = @@ -214,28 +218,6 @@ impl SupplyMetrics { Some(&self.height_to_utxo_count), )?; - self.height_to_supply_half_value - .compute_all(price, starting_indexes, exit, |v| { - v.compute_transform( - starting_indexes.height, - &self.height_to_supply, - |(h, v, ..)| (h, v / 2), - exit, - )?; - Ok(()) - })?; - - self.indexes_to_supply_half - .compute_all(price, starting_indexes, exit, |v| { - v.compute_transform( - starting_indexes.dateindex, - self.indexes_to_supply.sats.dateindex.as_ref().unwrap(), - |(i, sats, ..)| (i, sats / 2), - exit, - )?; - Ok(()) - })?; - Ok(()) } } diff --git a/crates/brk_computer/src/stateful/metrics/unrealized.rs b/crates/brk_computer/src/stateful/metrics/unrealized.rs index a6d3a6b62..a4229ac3d 100644 --- a/crates/brk_computer/src/stateful/metrics/unrealized.rs +++ b/crates/brk_computer/src/stateful/metrics/unrealized.rs @@ -142,19 +142,23 @@ impl UnrealizedMetrics { let height_to_supply_in_loss: EagerVec> = EagerVec::forced_import(cfg.db, &cfg.name("supply_in_loss"), cfg.version + v0)?; + let price_source = cfg + .price + .map(|p| p.chainindexes_to_price_close.height.boxed_clone()); + let height_to_supply_in_profit_value = ComputedHeightValueVecs::forced_import( cfg.db, &cfg.name("supply_in_profit"), Source::Vec(height_to_supply_in_profit.boxed_clone()), cfg.version + v0, - compute_dollars, + price_source.clone(), )?; let height_to_supply_in_loss_value = ComputedHeightValueVecs::forced_import( cfg.db, &cfg.name("supply_in_loss"), Source::Vec(height_to_supply_in_loss.boxed_clone()), cfg.version + v0, - compute_dollars, + price_source, )?; Ok(Self { @@ -341,13 +345,6 @@ impl UnrealizedMetrics { starting_indexes: &Indexes, exit: &Exit, ) -> Result<()> { - // Compute supply value from sats - self.height_to_supply_in_profit_value - .compute_rest(price, starting_indexes, exit)?; - self.height_to_supply_in_loss_value - .compute_rest(price, starting_indexes, exit)?; - - // Compute indexes from dateindex sources self.indexes_to_supply_in_profit.compute_rest( price, starting_indexes, diff --git a/crates/brk_computer/src/stateful/states/supply.rs b/crates/brk_computer/src/stateful/states/supply.rs index 5f3ff0fe4..1a3794861 100644 --- a/crates/brk_computer/src/stateful/states/supply.rs +++ b/crates/brk_computer/src/stateful/states/supply.rs @@ -2,11 +2,11 @@ use std::ops::{Add, AddAssign, SubAssign}; use brk_types::{CheckedSub, LoadedAddressData, Sats}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{Bytes, Formattable}; /// Current supply state tracking UTXO count and total value -#[derive(Debug, Default, Clone, Serialize, JsonSchema)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)] pub struct SupplyState { /// Number of unspent transaction outputs pub utxo_count: u64, diff --git a/crates/brk_indexer/Cargo.toml b/crates/brk_indexer/Cargo.toml index 6c5bb54cd..250d0b68d 100644 --- a/crates/brk_indexer/Cargo.toml +++ b/crates/brk_indexer/Cargo.toml @@ -27,6 +27,6 @@ rustc-hash = { workspace = true } vecdb = { workspace = true } [dev-dependencies] +brk_alloc = { workspace = true } brk_bencher = { workspace = true } color-eyre = { workspace = true } -mimalloc = { workspace = true } diff --git a/crates/brk_indexer/examples/indexer.rs b/crates/brk_indexer/examples/indexer.rs index e5a6df413..5f4f96acb 100644 --- a/crates/brk_indexer/examples/indexer.rs +++ b/crates/brk_indexer/examples/indexer.rs @@ -5,17 +5,14 @@ use std::{ time::{Duration, Instant}, }; +use brk_alloc::Mimalloc; use brk_indexer::Indexer; use brk_iterator::Blocks; use brk_reader::Reader; use brk_rpc::{Auth, Client}; use log::{debug, info}; -use mimalloc::MiMalloc; use vecdb::Exit; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - fn main() -> color_eyre::Result<()> { color_eyre::install()?; @@ -52,6 +49,8 @@ fn main() -> color_eyre::Result<()> { indexer.checked_index(&blocks, &client, &exit)?; info!("Done in {:?}", i.elapsed()); + Mimalloc::collect_if_wasted_above(500); + sleep(Duration::from_secs(60)); } } diff --git a/crates/brk_indexer/examples/indexer_bench.rs b/crates/brk_indexer/examples/indexer_bench.rs index 643518862..02440ce07 100644 --- a/crates/brk_indexer/examples/indexer_bench.rs +++ b/crates/brk_indexer/examples/indexer_bench.rs @@ -5,6 +5,7 @@ use std::{ time::{Duration, Instant}, }; +use brk_alloc::Mimalloc; use brk_bencher::Bencher; use brk_error::Result; use brk_indexer::Indexer; @@ -12,12 +13,8 @@ use brk_iterator::Blocks; use brk_reader::Reader; use brk_rpc::{Auth, Client}; use log::{debug, info}; -use mimalloc::MiMalloc; use vecdb::Exit; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - fn main() -> Result<()> { brk_logger::init(None)?; @@ -61,5 +58,9 @@ fn main() -> Result<()> { sleep(Duration::from_secs(10)); + Mimalloc::collect_if_wasted_above(500); + + sleep(Duration::from_secs(10)); + Ok(()) } diff --git a/crates/brk_indexer/examples/indexer_bench2.rs b/crates/brk_indexer/examples/indexer_bench2.rs index 2533332e2..7d2b739e1 100644 --- a/crates/brk_indexer/examples/indexer_bench2.rs +++ b/crates/brk_indexer/examples/indexer_bench2.rs @@ -5,6 +5,7 @@ use std::{ time::{Duration, Instant}, }; +use brk_alloc::Mimalloc; use brk_bencher::Bencher; use brk_error::Result; use brk_indexer::Indexer; @@ -12,12 +13,8 @@ use brk_iterator::Blocks; use brk_reader::Reader; use brk_rpc::{Auth, Client}; use log::{debug, info}; -use mimalloc::MiMalloc; use vecdb::Exit; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - fn main() -> Result<()> { brk_logger::init(None)?; @@ -59,6 +56,8 @@ fn main() -> Result<()> { indexer.index(&blocks, &client, &exit)?; info!("Done in {:?}", i.elapsed()); + Mimalloc::collect_if_wasted_above(500); + sleep(Duration::from_secs(60)); } } diff --git a/crates/brk_indexer/examples/indexer_read.rs b/crates/brk_indexer/examples/indexer_read.rs index 2577a2b17..e168b7a6c 100644 --- a/crates/brk_indexer/examples/indexer_read.rs +++ b/crates/brk_indexer/examples/indexer_read.rs @@ -1,12 +1,8 @@ use brk_error::Result; use brk_indexer::Indexer; -use mimalloc::MiMalloc; // use brk_types::Sats; use std::{fs, path::Path}; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - fn main() -> Result<()> { brk_logger::init(Some(Path::new(".log")))?; diff --git a/crates/brk_indexer/examples/indexer_read_speed.rs b/crates/brk_indexer/examples/indexer_read_speed.rs index 437c2d772..3a1b6f948 100644 --- a/crates/brk_indexer/examples/indexer_read_speed.rs +++ b/crates/brk_indexer/examples/indexer_read_speed.rs @@ -3,10 +3,6 @@ use std::{fs, path::Path, time::Instant}; use brk_error::Result; use brk_indexer::Indexer; use brk_types::Sats; -use mimalloc::MiMalloc; - -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; fn run_benchmark(indexer: &Indexer) -> (Sats, std::time::Duration, usize) { let start = Instant::now(); diff --git a/crates/brk_query/src/impl/metrics.rs b/crates/brk_query/src/impl/metrics.rs index 92e43859a..855a66b7e 100644 --- a/crates/brk_query/src/impl/metrics.rs +++ b/crates/brk_query/src/impl/metrics.rs @@ -236,9 +236,13 @@ impl Query { } pub fn metric_count(&self) -> MetricCount { + let total = self.total_metric_count(); + let lazy = self.lazy_metric_count(); MetricCount { distinct_metrics: self.distinct_metric_count(), - total_endpoints: self.total_metric_count(), + total_endpoints: total, + lazy_endpoints: lazy, + stored_endpoints: total - lazy, } } @@ -250,6 +254,10 @@ impl Query { self.vecs().total_metric_count } + pub fn lazy_metric_count(&self) -> usize { + self.vecs().lazy_metric_count + } + pub fn indexes(&self) -> &[IndexInfo] { &self.vecs().indexes } diff --git a/crates/brk_query/src/vecs.rs b/crates/brk_query/src/vecs.rs index 02814dadb..e91d0370a 100644 --- a/crates/brk_query/src/vecs.rs +++ b/crates/brk_query/src/vecs.rs @@ -16,6 +16,7 @@ pub struct Vecs<'a> { pub indexes: Vec, pub distinct_metric_count: usize, pub total_metric_count: usize, + pub lazy_metric_count: usize, catalog: Option, matcher: Option>, metric_to_indexes: BTreeMap<&'a str, Vec>, @@ -61,6 +62,12 @@ impl<'a> Vecs<'a> { .values() .map(|tree| tree.len()) .sum::(); + this.lazy_metric_count = this + .index_to_metric_to_vec + .values() + .flat_map(|tree| tree.values()) + .filter(|vec| vec.region_names().is_empty()) + .count(); this.indexes = this .index_to_metric_to_vec .keys() diff --git a/crates/brk_types/src/anyaddressindex.rs b/crates/brk_types/src/anyaddressindex.rs index 3b2bae74c..7290662e7 100644 --- a/crates/brk_types/src/anyaddressindex.rs +++ b/crates/brk_types/src/anyaddressindex.rs @@ -1,5 +1,7 @@ +use std::fmt; + use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{Bytes, Formattable}; use crate::{EmptyAddressIndex, LoadedAddressIndex, TypeIndex}; @@ -42,8 +44,18 @@ impl Serialize for AnyAddressIndex { } } -impl std::fmt::Display for AnyAddressIndex { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl<'de> Deserialize<'de> for AnyAddressIndex { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let variant = AnyAddressDataIndexEnum::deserialize(deserializer)?; + Ok(Self::from(variant)) + } +} + +impl fmt::Display for AnyAddressIndex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) } } @@ -55,7 +67,7 @@ impl Formattable for AnyAddressIndex { } } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum AnyAddressDataIndexEnum { Loaded(LoadedAddressIndex), @@ -73,3 +85,13 @@ impl From for AnyAddressDataIndexEnum { } } } + +impl From for AnyAddressIndex { + #[inline] + fn from(value: AnyAddressDataIndexEnum) -> Self { + match value { + AnyAddressDataIndexEnum::Loaded(idx) => Self::from(idx), + AnyAddressDataIndexEnum::Empty(idx) => Self::from(idx), + } + } +} diff --git a/crates/brk_types/src/blockrewardsentry.rs b/crates/brk_types/src/blockrewardsentry.rs index c7dd0e03e..01405cf4f 100644 --- a/crates/brk_types/src/blockrewardsentry.rs +++ b/crates/brk_types/src/blockrewardsentry.rs @@ -1,8 +1,8 @@ use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; /// A single block rewards data point. -#[derive(Debug, Serialize, JsonSchema)] +#[derive(Debug, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct BlockRewardsEntry { pub avg_height: u32, diff --git a/crates/brk_types/src/blocksizeentry.rs b/crates/brk_types/src/blocksizeentry.rs index 8e3a2f3d3..36031506c 100644 --- a/crates/brk_types/src/blocksizeentry.rs +++ b/crates/brk_types/src/blocksizeentry.rs @@ -1,8 +1,8 @@ use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; /// A single block size data point. -#[derive(Debug, Serialize, JsonSchema)] +#[derive(Debug, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct BlockSizeEntry { pub avg_height: u32, diff --git a/crates/brk_types/src/blocksizesweights.rs b/crates/brk_types/src/blocksizesweights.rs index 9b01410ca..d52d15641 100644 --- a/crates/brk_types/src/blocksizesweights.rs +++ b/crates/brk_types/src/blocksizesweights.rs @@ -1,10 +1,10 @@ use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use super::{BlockSizeEntry, BlockWeightEntry}; /// Combined block sizes and weights response. -#[derive(Debug, Serialize, JsonSchema)] +#[derive(Debug, Serialize, Deserialize, JsonSchema)] pub struct BlockSizesWeights { pub sizes: Vec, pub weights: Vec, diff --git a/crates/brk_types/src/blocktimestamp.rs b/crates/brk_types/src/blocktimestamp.rs index c4d9c3aa1..9c9b312c9 100644 --- a/crates/brk_types/src/blocktimestamp.rs +++ b/crates/brk_types/src/blocktimestamp.rs @@ -1,10 +1,10 @@ use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::{BlockHash, Height}; /// Block information returned for timestamp queries -#[derive(Debug, Clone, Serialize, JsonSchema)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct BlockTimestamp { /// Block height pub height: Height, diff --git a/crates/brk_types/src/blockweightentry.rs b/crates/brk_types/src/blockweightentry.rs index 0f12bbacb..559b8f90f 100644 --- a/crates/brk_types/src/blockweightentry.rs +++ b/crates/brk_types/src/blockweightentry.rs @@ -1,8 +1,8 @@ use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; /// A single block weight data point. -#[derive(Debug, Serialize, JsonSchema)] +#[derive(Debug, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct BlockWeightEntry { pub avg_height: u32, diff --git a/crates/brk_types/src/bytes.rs b/crates/brk_types/src/bytes.rs index caa94866c..94b81f686 100644 --- a/crates/brk_types/src/bytes.rs +++ b/crates/brk_types/src/bytes.rs @@ -1,6 +1,6 @@ use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::Bytes; #[derive( @@ -13,6 +13,7 @@ use vecdb::Bytes; PartialOrd, Ord, Serialize, + Deserialize, Bytes, Hash, JsonSchema, @@ -37,6 +38,7 @@ impl From<&[u8]> for U8x2 { PartialOrd, Ord, Serialize, + Deserialize, Bytes, Hash, JsonSchema, @@ -61,6 +63,7 @@ impl From<&[u8]> for U8x20 { PartialOrd, Ord, Serialize, + Deserialize, Bytes, Hash, JsonSchema, @@ -75,7 +78,20 @@ impl From<&[u8]> for U8x32 { } } -#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Bytes, Hash, Serialize)] +#[derive( + Debug, + Clone, + Deref, + DerefMut, + PartialEq, + Eq, + PartialOrd, + Ord, + Bytes, + Hash, + Serialize, + Deserialize, +)] pub struct U8x33(#[serde(with = "serde_bytes")] [u8; 33]); impl JsonSchema for U8x33 { @@ -98,7 +114,20 @@ impl From<&[u8]> for U8x33 { } } -#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Bytes, Hash, Serialize)] +#[derive( + Debug, + Clone, + Deref, + DerefMut, + PartialEq, + Eq, + PartialOrd, + Ord, + Bytes, + Hash, + Serialize, + Deserialize, +)] pub struct U8x65(#[serde(with = "serde_bytes")] [u8; 65]); impl JsonSchema for U8x65 { diff --git a/crates/brk_types/src/cents.rs b/crates/brk_types/src/cents.rs index f78efa089..a1504579a 100644 --- a/crates/brk_types/src/cents.rs +++ b/crates/brk_types/src/cents.rs @@ -1,13 +1,24 @@ use std::ops::{Add, Div, Mul, Sub}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco}; use super::Dollars; #[derive( - Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Pco, JsonSchema, + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Pco, + JsonSchema, )] pub struct Cents(i64); diff --git a/crates/brk_types/src/date.rs b/crates/brk_types/src/date.rs index 872959fe0..b5ef36a22 100644 --- a/crates/brk_types/src/date.rs +++ b/crates/brk_types/src/date.rs @@ -1,6 +1,8 @@ +use std::fmt; + use jiff::{Span, Zoned, civil::Date as Date_, tz::TimeZone}; use schemars::JsonSchema; -use serde::{Serialize, Serializer}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Visitor}; use vecdb::{Formattable, Pco}; use crate::ONE_DAY_IN_SEC_F64; @@ -118,8 +120,49 @@ impl Serialize for Date { } } -impl std::fmt::Display for Date { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl<'de> Deserialize<'de> for Date { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct DateVisitor; + + impl Visitor<'_> for DateVisitor { + type Value = Date; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a date string in YYYY-MM-DD format") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + // Parse YYYY-MM-DD format + if v.len() != 10 { + return Err(E::invalid_length(v.len(), &self)); + } + + let year: u16 = v[0..4] + .parse() + .map_err(|_| E::invalid_value(serde::de::Unexpected::Str(v), &self))?; + let month: u8 = v[5..7] + .parse() + .map_err(|_| E::invalid_value(serde::de::Unexpected::Str(v), &self))?; + let day: u8 = v[8..10] + .parse() + .map_err(|_| E::invalid_value(serde::de::Unexpected::Str(v), &self))?; + + Ok(Date::new(year, month, day)) + } + } + + deserializer.deserialize_str(DateVisitor) + } +} + +impl fmt::Display for Date { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut buf = itoa::Buffer::new(); f.write_str(buf.format(self.year()))?; diff --git a/crates/brk_types/src/dateindex.rs b/crates/brk_types/src/dateindex.rs index fc172d9f1..f8c0e7919 100644 --- a/crates/brk_types/src/dateindex.rs +++ b/crates/brk_types/src/dateindex.rs @@ -1,9 +1,12 @@ -use std::ops::{Add, Rem}; +use std::{ + fmt, + ops::{Add, Rem}, +}; use brk_error::Error; use jiff::Span; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, FromCoarserIndex, Pco, PrintableIndex}; use crate::{DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, WeekIndex, YearIndex}; @@ -11,7 +14,18 @@ use crate::{DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, WeekIndex, Yea use super::Date; #[derive( - Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Pco, JsonSchema, + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Pco, + JsonSchema, )] pub struct DateIndex(u16); @@ -227,8 +241,8 @@ impl PrintableIndex for DateIndex { } } -impl std::fmt::Display for DateIndex { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for DateIndex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut buf = itoa::Buffer::new(); let str = buf.format(self.0); f.write_str(str) diff --git a/crates/brk_types/src/dollars.rs b/crates/brk_types/src/dollars.rs index 7a50d8d66..f853d04af 100644 --- a/crates/brk_types/src/dollars.rs +++ b/crates/brk_types/src/dollars.rs @@ -48,6 +48,10 @@ impl Dollars { pub fn is_zero(&self) -> bool { self.0 == 0.0 } + + pub fn halved(self) -> Self { + Self(self.0 / 2.0) + } } impl From for Dollars { diff --git a/crates/brk_types/src/emptyaddressindex.rs b/crates/brk_types/src/emptyaddressindex.rs index 96e8a123b..cd89e5d0b 100644 --- a/crates/brk_types/src/emptyaddressindex.rs +++ b/crates/brk_types/src/emptyaddressindex.rs @@ -2,13 +2,25 @@ use std::ops::Add; use derive_deref::Deref; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; #[derive( - Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref, Serialize, Pco, JsonSchema, + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Deref, + Serialize, + Deserialize, + Pco, + JsonSchema, )] pub struct EmptyAddressIndex(TypeIndex); diff --git a/crates/brk_types/src/emptyoutputindex.rs b/crates/brk_types/src/emptyoutputindex.rs index 4878082e7..75d856881 100644 --- a/crates/brk_types/src/emptyoutputindex.rs +++ b/crates/brk_types/src/emptyoutputindex.rs @@ -1,14 +1,27 @@ -use std::ops::Add; +use std::{fmt, ops::Add}; use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; #[derive( - Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default, Serialize, Pco, JsonSchema, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + Deref, + DerefMut, + Default, + Serialize, + Deserialize, + Pco, + JsonSchema, )] pub struct EmptyOutputIndex(TypeIndex); impl From for EmptyOutputIndex { @@ -58,8 +71,8 @@ impl PrintableIndex for EmptyOutputIndex { } } -impl std::fmt::Display for EmptyOutputIndex { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for EmptyOutputIndex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) } } diff --git a/crates/brk_types/src/feerate.rs b/crates/brk_types/src/feerate.rs index 740a0f019..865d045a3 100644 --- a/crates/brk_types/src/feerate.rs +++ b/crates/brk_types/src/feerate.rs @@ -4,13 +4,13 @@ use std::{ }; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{Formattable, Pco}; use super::{Sats, VSize}; /// Fee rate in sats/vB -#[derive(Debug, Default, Clone, Copy, Serialize, Pco, JsonSchema)] +#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, Pco, JsonSchema)] pub struct FeeRate(f64); impl FeeRate { diff --git a/crates/brk_types/src/loadedaddressindex.rs b/crates/brk_types/src/loadedaddressindex.rs index 01789d712..3bb475d12 100644 --- a/crates/brk_types/src/loadedaddressindex.rs +++ b/crates/brk_types/src/loadedaddressindex.rs @@ -2,13 +2,25 @@ use std::ops::Add; use derive_deref::Deref; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; #[derive( - Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, Default, Serialize, Pco, JsonSchema, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + Deref, + Default, + Serialize, + Deserialize, + Pco, + JsonSchema, )] pub struct LoadedAddressIndex(TypeIndex); diff --git a/crates/brk_types/src/metriccount.rs b/crates/brk_types/src/metriccount.rs index 249f5806b..24de55578 100644 --- a/crates/brk_types/src/metriccount.rs +++ b/crates/brk_types/src/metriccount.rs @@ -10,4 +10,10 @@ pub struct MetricCount { /// Total number of metric-index combinations across all timeframes #[schemars(example = 21000)] pub total_endpoints: usize, + /// Number of lazy (computed on-the-fly) metric-index combinations + #[schemars(example = 5000)] + pub lazy_endpoints: usize, + /// Number of eager (stored on disk) metric-index combinations + #[schemars(example = 16000)] + pub stored_endpoints: usize, } diff --git a/crates/brk_types/src/ohlc.rs b/crates/brk_types/src/ohlc.rs index 84b1455b4..0a5ea3927 100644 --- a/crates/brk_types/src/ohlc.rs +++ b/crates/brk_types/src/ohlc.rs @@ -6,7 +6,11 @@ use std::{ use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::{Serialize, Serializer, ser::SerializeTuple}; +use serde::{ + Deserialize, Deserializer, Serialize, Serializer, + de::{SeqAccess, Visitor}, + ser::SerializeTuple, +}; use vecdb::{Bytes, Formattable, Pco, TransparentPco}; use crate::StoredF64; @@ -61,6 +65,58 @@ impl Serialize for OHLCCents { } } +macro_rules! impl_ohlc_deserialize { + ($ohlc_type:ty, $inner_type:ty) => { + impl<'de> Deserialize<'de> for $ohlc_type { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct OHLCVisitor; + + impl<'de> Visitor<'de> for OHLCVisitor { + type Value = $ohlc_type; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a tuple of 4 elements (open, high, low, close)") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let open = seq + .next_element::<$inner_type>()? + .ok_or_else(|| serde::de::Error::invalid_length(0, &self))?; + let high = seq + .next_element::<$inner_type>()? + .ok_or_else(|| serde::de::Error::invalid_length(1, &self))?; + let low = seq + .next_element::<$inner_type>()? + .ok_or_else(|| serde::de::Error::invalid_length(2, &self))?; + let close = seq + .next_element::<$inner_type>()? + .ok_or_else(|| serde::de::Error::invalid_length(3, &self))?; + + Ok(Self::Value { + open: Open::new(open), + high: High::new(high), + low: Low::new(low), + close: Close::new(close), + }) + } + } + + deserializer.deserialize_tuple(4, OHLCVisitor) + } + } + }; +} + +impl_ohlc_deserialize!(OHLCCents, Cents); +impl_ohlc_deserialize!(OHLCDollars, Dollars); +impl_ohlc_deserialize!(OHLCSats, Sats); + impl Display for OHLCCents { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( diff --git a/crates/brk_types/src/opreturnindex.rs b/crates/brk_types/src/opreturnindex.rs index 788efbe78..cd2a42801 100644 --- a/crates/brk_types/src/opreturnindex.rs +++ b/crates/brk_types/src/opreturnindex.rs @@ -2,7 +2,7 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; @@ -19,6 +19,7 @@ use crate::TypeIndex; DerefMut, Default, Serialize, + Deserialize, Pco, JsonSchema, )] diff --git a/crates/brk_types/src/outpoint.rs b/crates/brk_types/src/outpoint.rs index a5aea1fe4..8b3f9322e 100644 --- a/crates/brk_types/src/outpoint.rs +++ b/crates/brk_types/src/outpoint.rs @@ -1,12 +1,10 @@ use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use vecdb::{Formattable, Pco}; use crate::{TxIndex, Vout}; -#[derive( - Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize, JsonSchema, Hash, Pco, -)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, JsonSchema, Hash, Pco)] pub struct OutPoint(u64); impl OutPoint { @@ -54,3 +52,31 @@ impl Formattable for OutPoint { true } } + +impl Serialize for OutPoint { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + use serde::ser::SerializeStruct; + let mut state = serializer.serialize_struct("OutPoint", 2)?; + state.serialize_field("txindex", &self.txindex())?; + state.serialize_field("vout", &self.vout())?; + state.end() + } +} + +impl<'de> Deserialize<'de> for OutPoint { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper { + txindex: TxIndex, + vout: Vout, + } + let h = Helper::deserialize(deserializer)?; + Ok(Self::new(h.txindex, h.vout)) + } +} diff --git a/crates/brk_types/src/outputtype.rs b/crates/brk_types/src/outputtype.rs index e25edb03d..0879b8f72 100644 --- a/crates/brk_types/src/outputtype.rs +++ b/crates/brk_types/src/outputtype.rs @@ -1,13 +1,26 @@ use bitcoin::{AddressType, ScriptBuf, opcodes::all::OP_PUSHBYTES_2}; use brk_error::Error; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use strum::Display; use vecdb::{Bytes, Formattable, Pco, TransparentPco}; use crate::AddressBytes; -#[derive(Debug, Clone, Copy, Display, PartialEq, Eq, PartialOrd, Ord, Serialize, JsonSchema, Hash)] +#[derive( + Debug, + Clone, + Copy, + Display, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + JsonSchema, + Hash, +)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] #[repr(u16)] diff --git a/crates/brk_types/src/p2aaddressindex.rs b/crates/brk_types/src/p2aaddressindex.rs index 4d4eb8ef2..ee74d7418 100644 --- a/crates/brk_types/src/p2aaddressindex.rs +++ b/crates/brk_types/src/p2aaddressindex.rs @@ -2,7 +2,7 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; @@ -19,6 +19,7 @@ use crate::TypeIndex; DerefMut, Default, Serialize, + Deserialize, Pco, JsonSchema, )] diff --git a/crates/brk_types/src/p2abytes.rs b/crates/brk_types/src/p2abytes.rs index 5c1722060..6d8f02470 100644 --- a/crates/brk_types/src/p2abytes.rs +++ b/crates/brk_types/src/p2abytes.rs @@ -2,12 +2,25 @@ use std::fmt; use derive_deref::Deref; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{Bytes, Formattable}; use crate::U8x2; -#[derive(Debug, Clone, Deref, PartialEq, Eq, PartialOrd, Ord, Serialize, Bytes, Hash, JsonSchema)] +#[derive( + Debug, + Clone, + Deref, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Bytes, + Hash, + JsonSchema, +)] pub struct P2ABytes(U8x2); impl From<&[u8]> for P2ABytes { diff --git a/crates/brk_types/src/p2msoutputindex.rs b/crates/brk_types/src/p2msoutputindex.rs index 1506aad5e..69287c988 100644 --- a/crates/brk_types/src/p2msoutputindex.rs +++ b/crates/brk_types/src/p2msoutputindex.rs @@ -2,7 +2,7 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; @@ -19,6 +19,7 @@ use crate::TypeIndex; DerefMut, Default, Serialize, + Deserialize, Pco, JsonSchema, )] diff --git a/crates/brk_types/src/p2pk33addressindex.rs b/crates/brk_types/src/p2pk33addressindex.rs index 718c61b78..4a58d6769 100644 --- a/crates/brk_types/src/p2pk33addressindex.rs +++ b/crates/brk_types/src/p2pk33addressindex.rs @@ -2,7 +2,7 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; @@ -19,6 +19,7 @@ use crate::TypeIndex; DerefMut, Default, Serialize, + Deserialize, Pco, JsonSchema, )] diff --git a/crates/brk_types/src/p2pk33bytes.rs b/crates/brk_types/src/p2pk33bytes.rs index 712094de7..1084ac3c9 100644 --- a/crates/brk_types/src/p2pk33bytes.rs +++ b/crates/brk_types/src/p2pk33bytes.rs @@ -2,12 +2,25 @@ use std::fmt; use derive_deref::Deref; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{Bytes, Formattable}; use crate::U8x33; -#[derive(Debug, Clone, Deref, PartialEq, Eq, PartialOrd, Ord, Serialize, Bytes, Hash, JsonSchema)] +#[derive( + Debug, + Clone, + Deref, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Bytes, + Hash, + JsonSchema, +)] pub struct P2PK33Bytes(U8x33); impl From<&[u8]> for P2PK33Bytes { diff --git a/crates/brk_types/src/p2pk65addressindex.rs b/crates/brk_types/src/p2pk65addressindex.rs index 8c37f90d8..bddde6a67 100644 --- a/crates/brk_types/src/p2pk65addressindex.rs +++ b/crates/brk_types/src/p2pk65addressindex.rs @@ -2,7 +2,7 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; @@ -19,6 +19,7 @@ use crate::TypeIndex; DerefMut, Default, Serialize, + Deserialize, Pco, JsonSchema, )] diff --git a/crates/brk_types/src/p2pk65bytes.rs b/crates/brk_types/src/p2pk65bytes.rs index b266065d0..cb2734414 100644 --- a/crates/brk_types/src/p2pk65bytes.rs +++ b/crates/brk_types/src/p2pk65bytes.rs @@ -2,12 +2,25 @@ use std::fmt; use derive_deref::Deref; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{Bytes, Formattable}; use crate::U8x65; -#[derive(Debug, Clone, Deref, PartialEq, Eq, PartialOrd, Ord, Serialize, Bytes, Hash, JsonSchema)] +#[derive( + Debug, + Clone, + Deref, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Bytes, + Hash, + JsonSchema, +)] pub struct P2PK65Bytes(U8x65); impl From<&[u8]> for P2PK65Bytes { diff --git a/crates/brk_types/src/p2pkhaddressindex.rs b/crates/brk_types/src/p2pkhaddressindex.rs index 9bff56bb3..65629dac4 100644 --- a/crates/brk_types/src/p2pkhaddressindex.rs +++ b/crates/brk_types/src/p2pkhaddressindex.rs @@ -2,7 +2,7 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; @@ -19,6 +19,7 @@ use crate::TypeIndex; DerefMut, Default, Serialize, + Deserialize, Pco, JsonSchema, )] diff --git a/crates/brk_types/src/p2pkhbytes.rs b/crates/brk_types/src/p2pkhbytes.rs index b7f1b7e68..2f9412b4b 100644 --- a/crates/brk_types/src/p2pkhbytes.rs +++ b/crates/brk_types/src/p2pkhbytes.rs @@ -2,12 +2,25 @@ use std::fmt; use derive_deref::Deref; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{Bytes, Formattable}; use crate::U8x20; -#[derive(Debug, Clone, Deref, PartialEq, Eq, PartialOrd, Ord, Serialize, Bytes, Hash, JsonSchema)] +#[derive( + Debug, + Clone, + Deref, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Bytes, + Hash, + JsonSchema, +)] pub struct P2PKHBytes(U8x20); impl From<&[u8]> for P2PKHBytes { diff --git a/crates/brk_types/src/p2shaddressindex.rs b/crates/brk_types/src/p2shaddressindex.rs index 7f4f3b6c0..468b52ca0 100644 --- a/crates/brk_types/src/p2shaddressindex.rs +++ b/crates/brk_types/src/p2shaddressindex.rs @@ -2,7 +2,7 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; @@ -19,6 +19,7 @@ use crate::TypeIndex; DerefMut, Default, Serialize, + Deserialize, Pco, JsonSchema, )] diff --git a/crates/brk_types/src/p2shbytes.rs b/crates/brk_types/src/p2shbytes.rs index 225ca2734..3dfa89cfa 100644 --- a/crates/brk_types/src/p2shbytes.rs +++ b/crates/brk_types/src/p2shbytes.rs @@ -2,12 +2,25 @@ use std::fmt; use derive_deref::Deref; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{Bytes, Formattable}; use crate::U8x20; -#[derive(Debug, Clone, Deref, PartialEq, Eq, PartialOrd, Ord, Serialize, Bytes, Hash, JsonSchema)] +#[derive( + Debug, + Clone, + Deref, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Bytes, + Hash, + JsonSchema, +)] pub struct P2SHBytes(U8x20); impl From<&[u8]> for P2SHBytes { diff --git a/crates/brk_types/src/p2traddressindex.rs b/crates/brk_types/src/p2traddressindex.rs index 773d97dad..5849311c6 100644 --- a/crates/brk_types/src/p2traddressindex.rs +++ b/crates/brk_types/src/p2traddressindex.rs @@ -2,7 +2,7 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; @@ -19,6 +19,7 @@ use crate::TypeIndex; DerefMut, Default, Serialize, + Deserialize, Pco, JsonSchema, )] diff --git a/crates/brk_types/src/p2trbytes.rs b/crates/brk_types/src/p2trbytes.rs index a23708594..ed1f8086a 100644 --- a/crates/brk_types/src/p2trbytes.rs +++ b/crates/brk_types/src/p2trbytes.rs @@ -2,12 +2,25 @@ use std::fmt; use derive_deref::Deref; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{Bytes, Formattable}; use crate::U8x32; -#[derive(Debug, Clone, Deref, PartialEq, Eq, PartialOrd, Ord, Serialize, Bytes, Hash, JsonSchema)] +#[derive( + Debug, + Clone, + Deref, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Bytes, + Hash, + JsonSchema, +)] pub struct P2TRBytes(U8x32); impl From<&[u8]> for P2TRBytes { diff --git a/crates/brk_types/src/p2wpkhaddressindex.rs b/crates/brk_types/src/p2wpkhaddressindex.rs index 955574c03..3bcb5ceb6 100644 --- a/crates/brk_types/src/p2wpkhaddressindex.rs +++ b/crates/brk_types/src/p2wpkhaddressindex.rs @@ -2,7 +2,7 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; @@ -19,6 +19,7 @@ use crate::TypeIndex; DerefMut, Default, Serialize, + Deserialize, Pco, JsonSchema, )] diff --git a/crates/brk_types/src/p2wpkhbytes.rs b/crates/brk_types/src/p2wpkhbytes.rs index dadd08f8e..f086595a3 100644 --- a/crates/brk_types/src/p2wpkhbytes.rs +++ b/crates/brk_types/src/p2wpkhbytes.rs @@ -2,12 +2,25 @@ use std::fmt; use derive_deref::Deref; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{Bytes, Formattable}; use crate::U8x20; -#[derive(Debug, Clone, Deref, PartialEq, Eq, PartialOrd, Ord, Serialize, Bytes, Hash, JsonSchema)] +#[derive( + Debug, + Clone, + Deref, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Bytes, + Hash, + JsonSchema, +)] pub struct P2WPKHBytes(U8x20); impl From<&[u8]> for P2WPKHBytes { diff --git a/crates/brk_types/src/p2wshaddressindex.rs b/crates/brk_types/src/p2wshaddressindex.rs index 5a758352d..79a2f0d84 100644 --- a/crates/brk_types/src/p2wshaddressindex.rs +++ b/crates/brk_types/src/p2wshaddressindex.rs @@ -2,7 +2,7 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; @@ -19,6 +19,7 @@ use crate::TypeIndex; DerefMut, Default, Serialize, + Deserialize, Pco, JsonSchema, )] diff --git a/crates/brk_types/src/p2wshbytes.rs b/crates/brk_types/src/p2wshbytes.rs index f55c31a90..5f56ba752 100644 --- a/crates/brk_types/src/p2wshbytes.rs +++ b/crates/brk_types/src/p2wshbytes.rs @@ -2,12 +2,25 @@ use std::fmt; use derive_deref::Deref; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{Bytes, Formattable}; use crate::U8x32; -#[derive(Debug, Clone, Deref, PartialEq, Eq, PartialOrd, Ord, Serialize, Bytes, Hash, JsonSchema)] +#[derive( + Debug, + Clone, + Deref, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Bytes, + Hash, + JsonSchema, +)] pub struct P2WSHBytes(U8x32); impl From<&[u8]> for P2WSHBytes { diff --git a/crates/brk_types/src/pooldetail.rs b/crates/brk_types/src/pooldetail.rs index 2a40f72ef..f4ce632a0 100644 --- a/crates/brk_types/src/pooldetail.rs +++ b/crates/brk_types/src/pooldetail.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + use schemars::JsonSchema; use serde::{Deserialize, Serialize}; diff --git a/crates/brk_types/src/stored_i16.rs b/crates/brk_types/src/stored_i16.rs index 722a107d9..f7cf80816 100644 --- a/crates/brk_types/src/stored_i16.rs +++ b/crates/brk_types/src/stored_i16.rs @@ -2,11 +2,23 @@ use std::ops::{Add, AddAssign, Div}; use derive_deref::Deref; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; #[derive( - Debug, Deref, Clone, Default, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Pco, JsonSchema, + Debug, + Deref, + Clone, + Default, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Pco, + JsonSchema, )] pub struct StoredI16(i16); diff --git a/crates/brk_types/src/stored_u16.rs b/crates/brk_types/src/stored_u16.rs index cb2b4dd7c..852dc5deb 100644 --- a/crates/brk_types/src/stored_u16.rs +++ b/crates/brk_types/src/stored_u16.rs @@ -2,7 +2,7 @@ use std::ops::{Add, AddAssign, Div}; use derive_deref::Deref; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use super::{ @@ -12,7 +12,19 @@ use super::{ }; #[derive( - Debug, Deref, Clone, Default, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Pco, JsonSchema, + Debug, + Deref, + Clone, + Default, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Pco, + JsonSchema, )] pub struct StoredU16(u16); diff --git a/crates/brk_types/src/unknownoutputindex.rs b/crates/brk_types/src/unknownoutputindex.rs index 7617273da..e2ed8eefe 100644 --- a/crates/brk_types/src/unknownoutputindex.rs +++ b/crates/brk_types/src/unknownoutputindex.rs @@ -2,13 +2,26 @@ use std::ops::Add; use derive_deref::{Deref, DerefMut}; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use vecdb::{CheckedSub, Formattable, Pco, PrintableIndex}; use crate::TypeIndex; #[derive( - Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Deref, DerefMut, Default, Serialize, Pco, JsonSchema, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Clone, + Copy, + Deref, + DerefMut, + Default, + Serialize, + Deserialize, + Pco, + JsonSchema, )] pub struct UnknownOutputIndex(TypeIndex);