From 411c5e4c4d24ee263e359528302e353ad0490aa0 Mon Sep 17 00:00:00 2001 From: nym21 Date: Sun, 18 May 2025 17:28:09 +0200 Subject: [PATCH] global: snapshot --- crates/brk_computer/src/vecs/blocks.rs | 15 +- crates/brk_computer/src/vecs/fetched.rs | 102 +++++----- .../brk_computer/src/vecs/grouped/builder.rs | 39 ++-- .../src/vecs/grouped/from_dateindex.rs | 6 +- .../src/vecs/grouped/from_height.rs | 12 +- .../src/vecs/grouped/from_height_strict.rs | 3 +- .../src/vecs/grouped/from_txindex.rs | 12 +- .../src/vecs/grouped/value_from_txindex.rs | 11 +- crates/brk_computer/src/vecs/indexes.rs | 179 ++++++++---------- crates/brk_computer/src/vecs/transactions.rs | 16 +- crates/brk_computer/src/vecs/utxos.rs | 33 +++- crates/brk_fetcher/src/lib.rs | 12 +- crates/brk_indexer/src/vecs/mod.rs | 135 ++++++++----- crates/brk_vec/examples/main.rs | 6 +- crates/brk_vec/src/traits/any.rs | 2 +- crates/brk_vec/src/traits/index.rs | 20 +- crates/brk_vec/src/traits/iterator.rs | 2 +- crates/brk_vec/src/variants/compressed.rs | 2 +- crates/brk_vec/src/variants/computed.rs | 31 ++- crates/brk_vec/src/variants/eager.rs | 11 +- crates/brk_vec/src/variants/indexed.rs | 15 +- crates/brk_vec/src/variants/lazy1.rs | 6 +- crates/brk_vec/src/variants/lazy2.rs | 6 +- crates/brk_vec/src/variants/lazy3.rs | 6 +- crates/brk_vec/src/variants/raw.rs | 2 +- crates/brk_vec/src/variants/stored.rs | 14 +- .../kibo.money/scripts/vecid-to-indexes.js | 9 + 27 files changed, 378 insertions(+), 329 deletions(-) diff --git a/crates/brk_computer/src/vecs/blocks.rs b/crates/brk_computer/src/vecs/blocks.rs index 78243fac9..210c8e131 100644 --- a/crates/brk_computer/src/vecs/blocks.rs +++ b/crates/brk_computer/src/vecs/blocks.rs @@ -39,7 +39,8 @@ impl Vecs { Ok(Self { height_to_interval: EagerVec::forced_import( - &path.join("height_to_interval"), + path, + "interval", Version::ZERO, compressed, )?, @@ -85,11 +86,7 @@ impl Vecs { compressed, StorableVecGeneatorOptions::default().add_sum().add_total(), )?, - height_to_vbytes: EagerVec::forced_import( - &path.join("height_to_vbytes"), - Version::ZERO, - compressed, - )?, + height_to_vbytes: EagerVec::forced_import(path, "vbytes", Version::ZERO, compressed)?, indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import( path, "block_vbytes", @@ -99,12 +96,14 @@ impl Vecs { StorableVecGeneatorOptions::default().add_sum().add_total(), )?, difficultyepoch_to_timestamp: EagerVec::forced_import( - &path.join("difficultyepoch_to_timestamp"), + path, + "timestamp", Version::ZERO, compressed, )?, halvingepoch_to_timestamp: EagerVec::forced_import( - &path.join("halvingepoch_to_timestamp"), + path, + "timestamp", Version::ZERO, compressed, )?, diff --git a/crates/brk_computer/src/vecs/fetched.rs b/crates/brk_computer/src/vecs/fetched.rs index b56308e06..12042cb24 100644 --- a/crates/brk_computer/src/vecs/fetched.rs +++ b/crates/brk_computer/src/vecs/fetched.rs @@ -82,72 +82,76 @@ impl Vecs { Ok(Self { dateindex_to_ohlc_in_cents: EagerVec::forced_import( - &fetched_path.join("dateindex_to_ohlc_in_cents"), - Version::ZERO, - compressed, - )?, - dateindex_to_ohlc: EagerVec::forced_import( - &path.join("dateindex_to_ohlc"), + &fetched_path, + "ohlc_in_cents", Version::ZERO, compressed, )?, + dateindex_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?, dateindex_to_ohlc_in_sats: EagerVec::forced_import( - &path.join("dateindex_to_ohlc_in_sats"), + path, + "ohlc_in_sats", VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, dateindex_to_close_in_cents: EagerVec::forced_import( - &path.join("dateindex_to_close_in_cents"), + path, + "close_in_cents", Version::ZERO, compressed, )?, dateindex_to_high_in_cents: EagerVec::forced_import( - &path.join("dateindex_to_high_in_cents"), + path, + "high_in_cents", Version::ZERO, compressed, )?, dateindex_to_low_in_cents: EagerVec::forced_import( - &path.join("dateindex_to_low_in_cents"), + path, + "low_in_cents", Version::ZERO, compressed, )?, dateindex_to_open_in_cents: EagerVec::forced_import( - &path.join("dateindex_to_open_in_cents"), + path, + "open_in_cents", Version::ZERO, compressed, )?, height_to_ohlc_in_cents: EagerVec::forced_import( - &fetched_path.join("height_to_ohlc_in_cents"), - Version::ZERO, - compressed, - )?, - height_to_ohlc: EagerVec::forced_import( - &path.join("height_to_ohlc"), + &fetched_path, + "ohlc_in_cents", Version::ZERO, compressed, )?, + height_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?, height_to_ohlc_in_sats: EagerVec::forced_import( - &path.join("height_to_ohlc_in_sats"), + path, + "ohlc_in_sats", VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, height_to_close_in_cents: EagerVec::forced_import( - &path.join("height_to_close_in_cents"), + path, + "close_in_cents", Version::ZERO, compressed, )?, height_to_high_in_cents: EagerVec::forced_import( - &path.join("height_to_high_in_cents"), + path, + "high_in_cents", Version::ZERO, compressed, )?, height_to_low_in_cents: EagerVec::forced_import( - &path.join("height_to_low_in_cents"), + path, + "low_in_cents", Version::ZERO, compressed, )?, height_to_open_in_cents: EagerVec::forced_import( - &path.join("height_to_open_in_cents"), + path, + "open_in_cents", Version::ZERO, compressed, )?, @@ -263,64 +267,52 @@ impl Vecs { compressed, StorableVecGeneatorOptions::default().add_last(), )?, - weekindex_to_ohlc: EagerVec::forced_import( - &path.join("weekindex_to_ohlc"), - Version::ZERO, - compressed, - )?, + weekindex_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?, weekindex_to_ohlc_in_sats: EagerVec::forced_import( - &path.join("weekindex_to_ohlc_in_sats"), + path, + "ohlc_in_sats", VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, difficultyepoch_to_ohlc: EagerVec::forced_import( - &path.join("difficultyepoch_to_ohlc"), + path, + "ohlc", Version::ZERO, compressed, )?, difficultyepoch_to_ohlc_in_sats: EagerVec::forced_import( - &path.join("difficultyepoch_to_ohlc_in_sats"), + path, + "ohlc_in_sats", VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, - monthindex_to_ohlc: EagerVec::forced_import( - &path.join("monthindex_to_ohlc"), - Version::ZERO, - compressed, - )?, + monthindex_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?, monthindex_to_ohlc_in_sats: EagerVec::forced_import( - &path.join("monthindex_to_ohlc_in_sats"), + path, + "ohlc_in_sats", VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, - quarterindex_to_ohlc: EagerVec::forced_import( - &path.join("quarterindex_to_ohlc"), - Version::ZERO, - compressed, - )?, + quarterindex_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?, quarterindex_to_ohlc_in_sats: EagerVec::forced_import( - &path.join("quarterindex_to_ohlc_in_sats"), + path, + "ohlc_in_sats", VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, - yearindex_to_ohlc: EagerVec::forced_import( - &path.join("yearindex_to_ohlc"), - Version::ZERO, - compressed, - )?, + yearindex_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?, yearindex_to_ohlc_in_sats: EagerVec::forced_import( - &path.join("yearindex_to_ohlc_in_sats"), + path, + "ohlc_in_sats", VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, - // halvingepoch_to_ohlc: StorableVec::forced_import(&path.join("halvingepoch_to_ohlc"), Version::ZERO, compressed)?, - decadeindex_to_ohlc: EagerVec::forced_import( - &path.join("decadeindex_to_ohlc"), - Version::ZERO, - compressed, - )?, + // halvingepoch_to_ohlc: StorableVec::forced_import(path, + // "halvingepoch_to_ohlc"), Version::ZERO, compressed)?, + decadeindex_to_ohlc: EagerVec::forced_import(path, "ohlc", Version::ZERO, compressed)?, decadeindex_to_ohlc_in_sats: EagerVec::forced_import( - &path.join("decadeindex_to_ohlc_in_sats"), + path, + "ohlc_in_sats", VERSION + VERSION_IN_SATS + Version::ZERO, compressed, )?, diff --git a/crates/brk_computer/src/vecs/grouped/builder.rs b/crates/brk_computer/src/vecs/grouped/builder.rs index 74906ce3e..051ffd93d 100644 --- a/crates/brk_computer/src/vecs/grouped/builder.rs +++ b/crates/brk_computer/src/vecs/grouped/builder.rs @@ -46,27 +46,23 @@ where compressed: Compressed, options: StorableVecGeneatorOptions, ) -> color_eyre::Result { - let key = I::to_string().split("::").last().unwrap().to_lowercase(); - let only_one_active = options.is_only_one_active(); - let default = || path.join(format!("{key}_to_{name}")); - - let prefix = |s: &str| path.join(format!("{key}_to_{s}_{name}")); + let prefix = |s: &str| format!("{s}_{name}"); let maybe_prefix = |s: &str| { if only_one_active { - default() + name.to_string() } else { prefix(s) } }; - let suffix = |s: &str| path.join(format!("{key}_to_{name}_{s}")); + let suffix = |s: &str| format!("{name}_{s}"); let maybe_suffix = |s: &str| { if only_one_active { - default() + name.to_string() } else { suffix(s) } @@ -78,6 +74,7 @@ where first: options.first.then(|| { Box::new( EagerVec::forced_import( + path, &maybe_prefix("first"), version + Version::ZERO, compressed, @@ -87,17 +84,14 @@ where }), last: options.last.then(|| { Box::new( - EagerVec::forced_import( - &path.join(format!("{key}_to_{name}")), - version + Version::ZERO, - compressed, - ) - .unwrap(), + EagerVec::forced_import(path, name, version + Version::ZERO, compressed) + .unwrap(), ) }), min: options.min.then(|| { Box::new( EagerVec::forced_import( + path, &maybe_suffix("min"), version + Version::ZERO, compressed, @@ -108,6 +102,7 @@ where max: options.max.then(|| { Box::new( EagerVec::forced_import( + path, &maybe_suffix("max"), version + Version::ZERO, compressed, @@ -118,6 +113,7 @@ where median: options.median.then(|| { Box::new( EagerVec::forced_import( + path, &maybe_suffix("median"), version + Version::ZERO, compressed, @@ -128,6 +124,7 @@ where average: options.average.then(|| { Box::new( EagerVec::forced_import( + path, &maybe_suffix("average"), version + Version::ZERO, compressed, @@ -138,6 +135,7 @@ where sum: options.sum.then(|| { Box::new( EagerVec::forced_import( + path, &maybe_suffix("sum"), version + Version::ZERO, compressed, @@ -147,13 +145,19 @@ where }), total: options.total.then(|| { Box::new( - EagerVec::forced_import(&prefix("total"), version + Version::ZERO, compressed) - .unwrap(), + EagerVec::forced_import( + path, + &prefix("total"), + version + Version::ZERO, + compressed, + ) + .unwrap(), ) }), _90p: options._90p.then(|| { Box::new( EagerVec::forced_import( + path, &maybe_suffix("90p"), version + Version::ZERO, compressed, @@ -164,6 +168,7 @@ where _75p: options._75p.then(|| { Box::new( EagerVec::forced_import( + path, &maybe_suffix("75p"), version + Version::ZERO, compressed, @@ -174,6 +179,7 @@ where _25p: options._25p.then(|| { Box::new( EagerVec::forced_import( + path, &maybe_suffix("25p"), version + Version::ZERO, compressed, @@ -184,6 +190,7 @@ where _10p: options._10p.then(|| { Box::new( EagerVec::forced_import( + path, &maybe_suffix("10p"), version + Version::ZERO, compressed, diff --git a/crates/brk_computer/src/vecs/grouped/from_dateindex.rs b/crates/brk_computer/src/vecs/grouped/from_dateindex.rs index 0bc57daf1..383a887e3 100644 --- a/crates/brk_computer/src/vecs/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/vecs/grouped/from_dateindex.rs @@ -49,11 +49,7 @@ where let options = options.remove_percentiles(); Ok(Self { - dateindex: EagerVec::forced_import( - &path.join(format!("dateindex_to_{name}")), - version, - compressed, - )?, + dateindex: EagerVec::forced_import(path, name, version, compressed)?, dateindex_extra, weekindex: ComputedVecBuilder::forced_import(path, name, version, compressed, options)?, monthindex: ComputedVecBuilder::forced_import( diff --git a/crates/brk_computer/src/vecs/grouped/from_height.rs b/crates/brk_computer/src/vecs/grouped/from_height.rs index 3d20840b1..eb2591e53 100644 --- a/crates/brk_computer/src/vecs/grouped/from_height.rs +++ b/crates/brk_computer/src/vecs/grouped/from_height.rs @@ -45,16 +45,8 @@ where ) -> color_eyre::Result { let version = VERSION + version; - let height = compute_source.then(|| { - Box::new( - EagerVec::forced_import( - &path.join(format!("height_to_{name}")), - version, - compressed, - ) - .unwrap(), - ) - }); + let height = compute_source + .then(|| Box::new(EagerVec::forced_import(path, name, version, compressed).unwrap())); let height_extra = ComputedVecBuilder::forced_import( path, diff --git a/crates/brk_computer/src/vecs/grouped/from_height_strict.rs b/crates/brk_computer/src/vecs/grouped/from_height_strict.rs index dee3406b9..19d7185f1 100644 --- a/crates/brk_computer/src/vecs/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/vecs/grouped/from_height_strict.rs @@ -36,8 +36,7 @@ where ) -> color_eyre::Result { let version = VERSION + version; - let height = - EagerVec::forced_import(&path.join(format!("height_to_{name}")), version, compressed)?; + let height = EagerVec::forced_import(path, name, version, compressed)?; let height_extra = ComputedVecBuilder::forced_import( path, diff --git a/crates/brk_computer/src/vecs/grouped/from_txindex.rs b/crates/brk_computer/src/vecs/grouped/from_txindex.rs index b98164b1c..4c53b9a2c 100644 --- a/crates/brk_computer/src/vecs/grouped/from_txindex.rs +++ b/crates/brk_computer/src/vecs/grouped/from_txindex.rs @@ -48,16 +48,8 @@ where ) -> color_eyre::Result { let version = VERSION + version; - let txindex = compute_source.then(|| { - Box::new( - EagerVec::forced_import( - &path.join(format!("txindex_to_{name}")), - version, - compressed, - ) - .unwrap(), - ) - }); + let txindex = compute_source + .then(|| Box::new(EagerVec::forced_import(path, name, version, compressed).unwrap())); let height = ComputedVecBuilder::forced_import(path, name, version, compressed, options)?; diff --git a/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs b/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs index 8910d0b51..bd84aee49 100644 --- a/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/vecs/grouped/value_from_txindex.rs @@ -51,6 +51,9 @@ impl ComputedValueVecsFromTxindex { let compute_source = source.is_none(); let compute_dollars = fetched.is_some(); + let name_in_btc = format!("{name}_in_btc"); + let name_in_usd = format!("{name}_in_usd"); + let sats = ComputedVecsFromTxindex::forced_import( path, name, @@ -61,7 +64,7 @@ impl ComputedValueVecsFromTxindex { )?; let bitcoin_txindex = LazyVecFrom1::init( - "txindex_to_{name}_in_btc", + &name_in_btc, VERSION + version, source.map_or_else(|| sats.txindex.as_ref().unwrap().boxed_clone(), |s| s), |txindex: TxIndex, iter| { @@ -74,7 +77,7 @@ impl ComputedValueVecsFromTxindex { let bitcoin = ComputedVecsFromTxindex::forced_import( path, - &format!("{name}_in_btc"), + &name_in_btc, false, VERSION + version, compressed, @@ -85,7 +88,7 @@ impl ComputedValueVecsFromTxindex { ComputedVecFrom3::forced_import_or_init_from_3( computation, path, - "txindex_to_{name}_in_usd", + &name_in_usd, VERSION + version, compressed, bitcoin_txindex.boxed_clone(), @@ -120,7 +123,7 @@ impl ComputedValueVecsFromTxindex { dollars: compute_dollars.then(|| { ComputedVecsFromTxindex::forced_import( path, - &format!("{name}_in_usd"), + &name_in_usd, false, VERSION + version, compressed, diff --git a/crates/brk_computer/src/vecs/indexes.rs b/crates/brk_computer/src/vecs/indexes.rs index 0e4f021f5..6864d8486 100644 --- a/crates/brk_computer/src/vecs/indexes.rs +++ b/crates/brk_computer/src/vecs/indexes.rs @@ -99,7 +99,7 @@ impl Vecs { let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "outputindex_to_outputindex", + "outputindex", Version::ZERO, compressed, indexer.vecs().outputindex_to_value.boxed_clone(), @@ -109,7 +109,7 @@ impl Vecs { let inputindex_to_inputindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "inputindex_to_inputindex", + "inputindex", Version::ZERO, compressed, indexer.vecs().inputindex_to_outputindex.boxed_clone(), @@ -119,7 +119,7 @@ impl Vecs { let txindex_to_txindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "txindex_to_txindex", + "txindex", Version::ZERO, compressed, indexer.vecs().txindex_to_txid.boxed_clone(), @@ -129,7 +129,7 @@ impl Vecs { let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2( computation, path, - "txindex_to_input_count", + "input_count", Version::ZERO, compressed, indexer.vecs().txindex_to_first_inputindex.boxed_clone(), @@ -152,7 +152,7 @@ impl Vecs { let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2( computation, path, - "txindex_to_output_count", + "output_count", Version::ZERO, compressed, indexer.vecs().txindex_to_first_outputindex.boxed_clone(), @@ -175,7 +175,7 @@ impl Vecs { let p2pk33index_to_p2pk33index = ComputedVec::forced_import_or_init_from_1( computation, path, - "p2pk33index_to_p2pk33index", + "p2pk33index", Version::ZERO, compressed, indexer.vecs().p2pk33index_to_p2pk33bytes.boxed_clone(), @@ -184,7 +184,7 @@ impl Vecs { let p2pk65index_to_p2pk65index = ComputedVec::forced_import_or_init_from_1( computation, path, - "p2pk65index_to_p2pk65index", + "p2pk65index", Version::ZERO, compressed, indexer.vecs().p2pk65index_to_p2pk65bytes.boxed_clone(), @@ -193,7 +193,7 @@ impl Vecs { let p2pkhindex_to_p2pkhindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "p2pkhindex_to_p2pkhindex", + "p2pkhindex", Version::ZERO, compressed, indexer.vecs().p2pkhindex_to_p2pkhbytes.boxed_clone(), @@ -202,7 +202,7 @@ impl Vecs { let p2shindex_to_p2shindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "p2shindex_to_p2shindex", + "p2shindex", Version::ZERO, compressed, indexer.vecs().p2shindex_to_p2shbytes.boxed_clone(), @@ -211,7 +211,7 @@ impl Vecs { let p2trindex_to_p2trindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "p2trindex_to_p2trindex", + "p2trindex", Version::ZERO, compressed, indexer.vecs().p2trindex_to_p2trbytes.boxed_clone(), @@ -220,7 +220,7 @@ impl Vecs { let p2wpkhindex_to_p2wpkhindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "p2wpkhindex_to_p2wpkhindex", + "p2wpkhindex", Version::ZERO, compressed, indexer.vecs().p2wpkhindex_to_p2wpkhbytes.boxed_clone(), @@ -229,7 +229,7 @@ impl Vecs { let p2wshindex_to_p2wshindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "p2wshindex_to_p2wshindex", + "p2wshindex", Version::ZERO, compressed, indexer.vecs().p2wshindex_to_p2wshbytes.boxed_clone(), @@ -238,7 +238,7 @@ impl Vecs { let p2aindex_to_p2aindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "p2aindex_to_p2aindex", + "p2aindex", Version::ZERO, compressed, indexer.vecs().p2aindex_to_p2abytes.boxed_clone(), @@ -247,7 +247,7 @@ impl Vecs { let p2msindex_to_p2msindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "p2msindex_to_p2msindex", + "p2msindex", Version::ZERO, compressed, indexer.vecs().p2msindex_to_txindex.boxed_clone(), @@ -256,7 +256,7 @@ impl Vecs { let emptyoutputindex_to_emptyoutputindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "emptyoutputindex_to_emptyoutputindex", + "emptyoutputindex", Version::ZERO, compressed, indexer.vecs().emptyoutputindex_to_txindex.boxed_clone(), @@ -265,7 +265,7 @@ impl Vecs { let unknownoutputindex_to_unknownoutputindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "unknownoutputindex_to_unknownoutputindex", + "unknownoutputindex", Version::ZERO, compressed, indexer.vecs().unknownoutputindex_to_txindex.boxed_clone(), @@ -274,23 +274,20 @@ impl Vecs { let opreturnindex_to_opreturnindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "opreturnindex_to_opreturnindex", + "opreturnindex", Version::ZERO, compressed, indexer.vecs().opreturnindex_to_txindex.boxed_clone(), |index, _| Some(index), )?; - let dateindex_to_first_height = EagerVec::forced_import( - &path.join("dateindex_to_first_height"), - Version::ZERO, - compressed, - )?; + let dateindex_to_first_height = + EagerVec::forced_import(path, "first_height", Version::ZERO, compressed)?; let dateindex_to_dateindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "dateindex_to_dateindex", + "dateindex", Version::ZERO, compressed, dateindex_to_first_height.boxed_clone(), @@ -300,20 +297,19 @@ impl Vecs { let dateindex_to_date = ComputedVec::forced_import_or_init_from_1( computation, path, - "dateindex_to_date", + "date", Version::ZERO, compressed, dateindex_to_dateindex.boxed_clone(), |index, _| Some(Date::from(index)), )?; - let height_to_date = - EagerVec::forced_import(&path.join("height_to_date"), Version::ZERO, compressed)?; + let height_to_date = EagerVec::forced_import(path, "date", Version::ZERO, compressed)?; let height_to_height = ComputedVec::forced_import_or_init_from_1( computation, path, - "height_to_height", + "height", Version::ZERO, compressed, height_to_date.boxed_clone(), @@ -323,23 +319,20 @@ impl Vecs { let height_to_difficultyepoch = ComputedVec::forced_import_or_init_from_1( computation, path, - "height_to_difficultyepoch", + "difficultyepoch", Version::ZERO, compressed, height_to_height.boxed_clone(), |index, _| Some(DifficultyEpoch::from(index)), )?; - let difficultyepoch_to_first_height = EagerVec::forced_import( - &path.join("difficultyepoch_to_first_height"), - Version::ZERO, - compressed, - )?; + let difficultyepoch_to_first_height = + EagerVec::forced_import(path, "first_height", Version::ZERO, compressed)?; let difficultyepoch_to_difficultyepoch = ComputedVec::forced_import_or_init_from_1( computation, path, - "difficultyepoch_to_difficultyepoch", + "difficultyepoch", Version::ZERO, compressed, difficultyepoch_to_first_height.boxed_clone(), @@ -349,67 +342,52 @@ impl Vecs { let height_to_halvingepoch = ComputedVec::forced_import_or_init_from_1( computation, path, - "height_to_halvingepoch", + "halvingepoch", Version::ZERO, compressed, height_to_height.boxed_clone(), |index, _| Some(HalvingEpoch::from(index)), )?; - let halvingepoch_to_first_height = EagerVec::forced_import( - &path.join("halvingepoch_to_first_height"), - Version::ZERO, - compressed, - )?; + let halvingepoch_to_first_height = + EagerVec::forced_import(path, "first_height", Version::ZERO, compressed)?; let halvingepoch_to_halvingepoch = ComputedVec::forced_import_or_init_from_1( computation, path, - "halvingepoch_to_halvingepoch", + "halvingepoch", Version::ZERO, compressed, halvingepoch_to_first_height.boxed_clone(), |index, _| Some(index), )?; - let dateindex_to_weekindex = EagerVec::forced_import( - &path.join("dateindex_to_weekindex"), - Version::ZERO, - compressed, - )?; + let dateindex_to_weekindex = + EagerVec::forced_import(path, "weekindex", Version::ZERO, compressed)?; - let weekindex_to_first_dateindex = EagerVec::forced_import( - &path.join("weekindex_to_first_dateindex"), - Version::ZERO, - compressed, - )?; + let weekindex_to_first_dateindex = + EagerVec::forced_import(path, "first_dateindex", Version::ZERO, compressed)?; let weekindex_to_weekindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "weekindex_to_weekindex", + "weekindex", Version::ZERO, compressed, weekindex_to_first_dateindex.boxed_clone(), |index, _| Some(index), )?; - let dateindex_to_monthindex = EagerVec::forced_import( - &path.join("dateindex_to_monthindex"), - Version::ZERO, - compressed, - )?; + let dateindex_to_monthindex = + EagerVec::forced_import(path, "monthindex", Version::ZERO, compressed)?; - let monthindex_to_first_dateindex = EagerVec::forced_import( - &path.join("monthindex_to_first_dateindex"), - Version::ZERO, - compressed, - )?; + let monthindex_to_first_dateindex = + EagerVec::forced_import(path, "first_dateindex", Version::ZERO, compressed)?; let monthindex_to_monthindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "monthindex_to_monthindex", + "monthindex", Version::ZERO, compressed, monthindex_to_first_dateindex.boxed_clone(), @@ -419,23 +397,20 @@ impl Vecs { let monthindex_to_quarterindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "monthindex_to_quarterindex", + "quarterindex", Version::ZERO, compressed, monthindex_to_monthindex.boxed_clone(), |index, _| Some(QuarterIndex::from(index)), )?; - let quarterindex_to_first_monthindex = EagerVec::forced_import( - &path.join("quarterindex_to_first_monthindex"), - Version::ZERO, - compressed, - )?; + let quarterindex_to_first_monthindex = + EagerVec::forced_import(path, "first_monthindex", Version::ZERO, compressed)?; let quarterindex_to_quarterindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "quarterindex_to_quarterindex", + "quarterindex", Version::ZERO, compressed, quarterindex_to_first_monthindex.boxed_clone(), @@ -445,23 +420,20 @@ impl Vecs { let monthindex_to_yearindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "monthindex_to_yearindex", + "yearindex", Version::ZERO, compressed, monthindex_to_monthindex.boxed_clone(), |index, _| Some(YearIndex::from(index)), )?; - let yearindex_to_first_monthindex = EagerVec::forced_import( - &path.join("yearindex_to_first_monthindex"), - Version::ZERO, - compressed, - )?; + let yearindex_to_first_monthindex = + EagerVec::forced_import(path, "first_monthindex", Version::ZERO, compressed)?; let yearindex_to_yearindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "yearindex_to_yearindex", + "yearindex", Version::ZERO, compressed, yearindex_to_first_monthindex.boxed_clone(), @@ -471,23 +443,20 @@ impl Vecs { let yearindex_to_decadeindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "yearindex_to_decadeindex", + "decadeindex", Version::ZERO, compressed, yearindex_to_yearindex.boxed_clone(), |index, _| Some(DecadeIndex::from(index)), )?; - let decadeindex_to_first_yearindex = EagerVec::forced_import( - &path.join("decadeindex_to_first_yearindex"), - Version::ZERO, - compressed, - )?; + let decadeindex_to_first_yearindex = + EagerVec::forced_import(path, "first_yearindex", Version::ZERO, compressed)?; let decadeindex_to_decadeindex = ComputedVec::forced_import_or_init_from_1( computation, path, - "decadeindex_to_decadeindex", + "decadeindex", Version::ZERO, compressed, decadeindex_to_first_yearindex.boxed_clone(), @@ -539,67 +508,75 @@ impl Vecs { yearindex_to_first_monthindex, yearindex_to_yearindex, height_to_date_fixed: EagerVec::forced_import( - &path.join("height_to_date_fixed"), + path, + "date_fixed", Version::ZERO, compressed, )?, height_to_dateindex: EagerVec::forced_import( - &path.join("height_to_dateindex"), - Version::ZERO, - compressed, - )?, - txindex_to_height: EagerVec::forced_import( - &path.join("txindex_to_height"), + path, + "dateindex", Version::ZERO, compressed, )?, + txindex_to_height: EagerVec::forced_import(path, "height", Version::ZERO, compressed)?, height_to_timestamp_fixed: EagerVec::forced_import( - &path.join("height_to_timestamp_fixed"), + path, + "timestamp_fixed", Version::ZERO, compressed, )?, height_to_txindex_count: EagerVec::forced_import( - &path.join("height_to_txindex_count"), + path, + "txindex_count", Version::ZERO, compressed, )?, dateindex_to_height_count: EagerVec::forced_import( - &path.join("dateindex_to_height_count"), + path, + "height_count", Version::ZERO, compressed, )?, weekindex_to_dateindex_count: EagerVec::forced_import( - &path.join("weekindex_to_dateindex_count"), + path, + "dateindex_count", Version::ZERO, compressed, )?, difficultyepoch_to_height_count: EagerVec::forced_import( - &path.join("difficultyepoch_to_height_count"), + path, + "height_count", Version::ZERO, compressed, )?, monthindex_to_dateindex_count: EagerVec::forced_import( - &path.join("monthindex_to_dateindex_count"), + path, + "dateindex_count", Version::ZERO, compressed, )?, quarterindex_to_monthindex_count: EagerVec::forced_import( - &path.join("quarterindex_to_monthindex_count"), + path, + "monthindex_count", Version::ZERO, compressed, )?, yearindex_to_monthindex_count: EagerVec::forced_import( - &path.join("yearindex_to_monthindex_count"), + path, + "monthindex_count", Version::ZERO, compressed, )?, decadeindex_to_yearindex_count: EagerVec::forced_import( - &path.join("decadeindex_to_yearindex_count"), + path, + "yearindex_count", Version::ZERO, compressed, )?, outputindex_to_txindex: EagerVec::forced_import( - &path.join("outputindex_to_txindex"), + path, + "txindex", Version::ZERO, compressed, )?, diff --git a/crates/brk_computer/src/vecs/transactions.rs b/crates/brk_computer/src/vecs/transactions.rs index fb09256ba..8ded154f8 100644 --- a/crates/brk_computer/src/vecs/transactions.rs +++ b/crates/brk_computer/src/vecs/transactions.rs @@ -102,7 +102,7 @@ impl Vecs { let inputindex_to_value = ComputedVec::forced_import_or_init_from_2( computation, path, - "inputindex_to_value", + "value", Version::ZERO, compressed, indexer.vecs().inputindex_to_outputindex.boxed_clone(), @@ -129,7 +129,7 @@ impl Vecs { let txindex_to_weight = ComputedVec::forced_import_or_init_from_2( computation, path, - "txindex_to_weight", + "weight", Version::ZERO, compressed, indexer.vecs().txindex_to_base_size.boxed_clone(), @@ -157,7 +157,7 @@ impl Vecs { let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1( computation, path, - "txindex_to_vsize", + "vsize", Version::ZERO, compressed, txindex_to_weight.boxed_clone(), @@ -174,7 +174,7 @@ impl Vecs { let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2( computation, path, - "txindex_to_is_coinbase", + "is_coinbase", Version::ZERO, compressed, indexes.txindex_to_height.boxed_clone(), @@ -198,7 +198,7 @@ impl Vecs { let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3( computation, path, - "txindex_to_input_value", + "input_value", Version::ZERO, compressed, indexer.vecs().txindex_to_first_inputindex.boxed_clone(), @@ -247,7 +247,7 @@ impl Vecs { let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3( computation, path, - "txindex_to_output_value", + "output_value", Version::ZERO, compressed, indexer.vecs().txindex_to_first_outputindex.boxed_clone(), @@ -296,7 +296,7 @@ impl Vecs { let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2( computation, path, - "txindex_to_fee", + "fee", Version::ZERO, compressed, txindex_to_input_value.boxed_clone(), @@ -319,7 +319,7 @@ impl Vecs { let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2( computation, path, - "txindex_to_feerate", + "feerate", Version::ZERO, compressed, txindex_to_fee.boxed_clone(), diff --git a/crates/brk_computer/src/vecs/utxos.rs b/crates/brk_computer/src/vecs/utxos.rs index 8dbe69ca3..861271554 100644 --- a/crates/brk_computer/src/vecs/utxos.rs +++ b/crates/brk_computer/src/vecs/utxos.rs @@ -12,7 +12,7 @@ use brk_vec::{ use derive_deref::{Deref, DerefMut}; use rayon::prelude::*; -use crate::states::{CohortState, Outputs, RealizedState, ReceivedState, SentState}; +use crate::states::{CohortState, Outputs, OutputsByTerm, RealizedState, ReceivedState, SentState}; use super::{ Indexes, fetched, @@ -33,7 +33,11 @@ impl Vecs { fetched: Option<&fetched::Vecs>, ) -> color_eyre::Result { Ok(Self(Outputs { - all: Vecs_::forced_import(path, _computation, compressed, fetched)?, + all: Vecs_::forced_import(path, None, _computation, compressed, fetched)?, + // by_term: OutputsByTerm { + // short: Vecs_::forced_import(path, Some("sth"), _computation, compressed, fetched)?, + // long: Vecs_::forced_import(path, Some("lth"), _computation, compressed, fetched)?, + // }, })) } @@ -373,6 +377,7 @@ pub struct Vecs_ { impl Vecs_ { pub fn forced_import( path: &Path, + cohort_name: Option<&str>, _computation: Computation, compressed: Compressed, fetched: Option<&fetched::Vecs>, @@ -381,10 +386,15 @@ impl Vecs_ { fs::create_dir_all(path)?; + // let prefix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{s}_{name}")); + + let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}")); + Ok(Self { height_to_realized_cap: compute_dollars.then(|| { EagerVec::forced_import( - &path.join("height_to_realized_cap"), + path, + &suffix("realized_cap"), VERSION + Version::ZERO, compressed, ) @@ -393,7 +403,7 @@ impl Vecs_ { indexes_to_realized_cap: compute_dollars.then(|| { ComputedVecsFromHeight::forced_import( path, - "realized_cap", + &suffix("realized_cap"), false, VERSION + Version::ZERO, compressed, @@ -402,13 +412,14 @@ impl Vecs_ { .unwrap() }), height_to_supply: EagerVec::forced_import( - &path.join("height_to_supply"), + path, + &suffix("supply"), VERSION + Version::ZERO, compressed, )?, indexes_to_supply: ComputedValueVecsFromHeight::forced_import( path, - "supply", + &suffix("supply"), false, VERSION + Version::ZERO, compressed, @@ -416,13 +427,14 @@ impl Vecs_ { compute_dollars, )?, height_to_unspendable_supply: EagerVec::forced_import( - &path.join("height_to_unspendable_supply"), + path, + &suffix("unspendable_supply"), VERSION + Version::ZERO, compressed, )?, indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( path, - "unspendable_supply", + &suffix("unspendable_supply"), false, VERSION + Version::ZERO, compressed, @@ -430,13 +442,14 @@ impl Vecs_ { compute_dollars, )?, height_to_utxo_count: EagerVec::forced_import( - &path.join("height_to_utxo_count"), + path, + &suffix("utxo_count"), VERSION + Version::new(111), compressed, )?, indexes_to_utxo_count: ComputedVecsFromHeight::forced_import( path, - "utxo_count", + &suffix("utxo_count"), false, VERSION + Version::ZERO, compressed, diff --git a/crates/brk_fetcher/src/lib.rs b/crates/brk_fetcher/src/lib.rs index a9f10372a..bee431ca8 100644 --- a/crates/brk_fetcher/src/lib.rs +++ b/crates/brk_fetcher/src/lib.rs @@ -72,13 +72,13 @@ impl Fetcher { let ohlc = self .kraken .get_from_1mn(timestamp, previous_timestamp) - .unwrap_or_else(|e| { - eprintln!("{e}"); + .unwrap_or_else(|_| { + // eprintln!("{e}"); self.binance .get_from_1mn(timestamp, previous_timestamp) - .unwrap_or_else(|e| { - eprintln!("{e}"); - self.kibo.get_from_height(height).unwrap_or_else(|e| { + .unwrap_or_else(|_| { + // eprintln!("{e}"); + self.kibo.get_from_height(height).unwrap_or_else(|_| { sleep(Duration::from_secs(30)); if tries < 8 * 60 * 2 { @@ -88,7 +88,7 @@ impl Fetcher { } let date = Date::from(timestamp); - eprintln!("{e}"); + // eprintln!("{e}"); panic!( " Can't find the price for: height: {height} - date: {date} diff --git a/crates/brk_indexer/src/vecs/mod.rs b/crates/brk_indexer/src/vecs/mod.rs index 6c46d586c..1d7879bf9 100644 --- a/crates/brk_indexer/src/vecs/mod.rs +++ b/crates/brk_indexer/src/vecs/mod.rs @@ -68,222 +68,261 @@ impl Vecs { Ok(Self { emptyoutputindex_to_txindex: IndexedVec::forced_import( - &path.join("emptyoutputindex_to_txindex"), + path, + "txindex", Version::ZERO, compressed, )?, height_to_blockhash: IndexedVec::forced_import( - &path.join("height_to_blockhash"), + path, + "blockhash", Version::ZERO, Compressed::NO, )?, height_to_difficulty: IndexedVec::forced_import( - &path.join("height_to_difficulty"), + path, + "difficulty", Version::ZERO, compressed, )?, height_to_first_emptyoutputindex: IndexedVec::forced_import( - &path.join("height_to_first_emptyoutputindex"), + path, + "first_emptyoutputindex", Version::ZERO, compressed, )?, height_to_first_inputindex: IndexedVec::forced_import( - &path.join("height_to_first_inputindex"), + path, + "first_inputindex", Version::ZERO, compressed, )?, height_to_first_opreturnindex: IndexedVec::forced_import( - &path.join("height_to_first_opreturnindex"), + path, + "first_opreturnindex", Version::ZERO, compressed, )?, height_to_first_outputindex: IndexedVec::forced_import( - &path.join("height_to_first_outputindex"), + path, + "first_outputindex", Version::ZERO, compressed, )?, height_to_first_p2aindex: IndexedVec::forced_import( - &path.join("height_to_first_p2aindex"), + path, + "first_p2aindex", Version::ZERO, compressed, )?, height_to_first_p2msindex: IndexedVec::forced_import( - &path.join("height_to_first_p2msindex"), + path, + "first_p2msindex", Version::ZERO, compressed, )?, height_to_first_p2pk33index: IndexedVec::forced_import( - &path.join("height_to_first_p2pk33index"), + path, + "first_p2pk33index", Version::ZERO, compressed, )?, height_to_first_p2pk65index: IndexedVec::forced_import( - &path.join("height_to_first_p2pk65index"), + path, + "first_p2pk65index", Version::ZERO, compressed, )?, height_to_first_p2pkhindex: IndexedVec::forced_import( - &path.join("height_to_first_p2pkhindex"), + path, + "first_p2pkhindex", Version::ZERO, compressed, )?, height_to_first_p2shindex: IndexedVec::forced_import( - &path.join("height_to_first_p2shindex"), + path, + "first_p2shindex", Version::ZERO, compressed, )?, height_to_first_p2trindex: IndexedVec::forced_import( - &path.join("height_to_first_p2trindex"), + path, + "first_p2trindex", Version::ZERO, compressed, )?, height_to_first_p2wpkhindex: IndexedVec::forced_import( - &path.join("height_to_first_p2wpkhindex"), + path, + "first_p2wpkhindex", Version::ZERO, compressed, )?, height_to_first_p2wshindex: IndexedVec::forced_import( - &path.join("height_to_first_p2wshindex"), + path, + "first_p2wshindex", Version::ZERO, compressed, )?, height_to_first_txindex: IndexedVec::forced_import( - &path.join("height_to_first_txindex"), + path, + "first_txindex", Version::ZERO, compressed, )?, height_to_first_unknownoutputindex: IndexedVec::forced_import( - &path.join("height_to_first_unknownoutputindex"), + path, + "first_unknownoutputindex", Version::ZERO, compressed, )?, height_to_timestamp: IndexedVec::forced_import( - &path.join("height_to_timestamp"), + path, + "timestamp", Version::ZERO, compressed, )?, height_to_total_size: IndexedVec::forced_import( - &path.join("height_to_total_size"), - Version::ZERO, - compressed, - )?, - height_to_weight: IndexedVec::forced_import( - &path.join("height_to_weight"), + path, + "total_size", Version::ZERO, compressed, )?, + height_to_weight: IndexedVec::forced_import(path, "weight", Version::ZERO, compressed)?, inputindex_to_outputindex: IndexedVec::forced_import( - &path.join("inputindex_to_outputindex"), + path, + "outputindex", Version::ZERO, compressed, )?, opreturnindex_to_txindex: IndexedVec::forced_import( - &path.join("opreturnindex_to_txindex"), + path, + "txindex", Version::ZERO, compressed, )?, outputindex_to_outputtype: IndexedVec::forced_import( - &path.join("outputindex_to_outputtype"), + path, + "outputtype", Version::ZERO, compressed, )?, outputindex_to_outputtypeindex: IndexedVec::forced_import( - &path.join("outputindex_to_outputtypeindex"), + path, + "outputtypeindex", Version::ZERO, compressed, )?, outputindex_to_value: IndexedVec::forced_import( - &path.join("outputindex_to_value"), + path, + "value", Version::ZERO, compressed, )?, p2aindex_to_p2abytes: IndexedVec::forced_import( - &path.join("p2aindex_to_p2abytes"), + path, + "p2abytes", Version::ZERO, Compressed::NO, )?, p2msindex_to_txindex: IndexedVec::forced_import( - &path.join("p2msindex_to_txindex"), + path, + "txindex", Version::ZERO, compressed, )?, p2pk33index_to_p2pk33bytes: IndexedVec::forced_import( - &path.join("p2pk33index_to_p2pk33bytes"), + path, + "p2pk33bytes", Version::ZERO, Compressed::NO, )?, p2pk65index_to_p2pk65bytes: IndexedVec::forced_import( - &path.join("p2pk65index_to_p2pk65bytes"), + path, + "p2pk65bytes", Version::ZERO, Compressed::NO, )?, p2pkhindex_to_p2pkhbytes: IndexedVec::forced_import( - &path.join("p2pkhindex_to_p2pkhbytes"), + path, + "p2pkhbytes", Version::ZERO, Compressed::NO, )?, p2shindex_to_p2shbytes: IndexedVec::forced_import( - &path.join("p2shindex_to_p2shbytes"), + path, + "p2shbytes", Version::ZERO, Compressed::NO, )?, p2trindex_to_p2trbytes: IndexedVec::forced_import( - &path.join("p2trindex_to_p2trbytes"), + path, + "p2trbytes", Version::ZERO, Compressed::NO, )?, p2wpkhindex_to_p2wpkhbytes: IndexedVec::forced_import( - &path.join("p2wpkhindex_to_p2wpkhbytes"), + path, + "p2wpkhbytes", Version::ZERO, Compressed::NO, )?, p2wshindex_to_p2wshbytes: IndexedVec::forced_import( - &path.join("p2wshindex_to_p2wshbytes"), + path, + "p2wshbytes", Version::ZERO, Compressed::NO, )?, txindex_to_base_size: IndexedVec::forced_import( - &path.join("txindex_to_base_size"), + path, + "base_size", Version::ZERO, compressed, )?, txindex_to_first_inputindex: IndexedVec::forced_import( - &path.join("txindex_to_first_inputindex"), + path, + "first_inputindex", Version::ZERO, compressed, )?, txindex_to_first_outputindex: IndexedVec::forced_import( - &path.join("txindex_to_first_outputindex"), + path, + "first_outputindex", Version::ZERO, Compressed::NO, )?, txindex_to_is_explicitly_rbf: IndexedVec::forced_import( - &path.join("txindex_to_is_explicitly_rbf"), + path, + "is_explicitly_rbf", Version::ZERO, compressed, )?, txindex_to_rawlocktime: IndexedVec::forced_import( - &path.join("txindex_to_rawlocktime"), + path, + "rawlocktime", Version::ZERO, compressed, )?, txindex_to_total_size: IndexedVec::forced_import( - &path.join("txindex_to_total_size"), + path, + "total_size", Version::ZERO, compressed, )?, txindex_to_txid: IndexedVec::forced_import( - &path.join("txindex_to_txid"), + path, + "txid", Version::ZERO, Compressed::NO, )?, txindex_to_txversion: IndexedVec::forced_import( - &path.join("txindex_to_txversion"), + path, + "txversion", Version::ZERO, compressed, )?, unknownoutputindex_to_txindex: IndexedVec::forced_import( - &path.join("unknownoutputindex_to_txindex"), + path, + "txindex", Version::ZERO, compressed, )?, diff --git a/crates/brk_vec/examples/main.rs b/crates/brk_vec/examples/main.rs index 7a41a565d..015052a1a 100644 --- a/crates/brk_vec/examples/main.rs +++ b/crates/brk_vec/examples/main.rs @@ -12,7 +12,7 @@ fn main() -> Result<(), Box> { { let mut vec: StoredVec = - StoredVec::forced_import(Path::new("./vec"), version, compressed)?; + StoredVec::forced_import(Path::new("."), "vec", version, compressed)?; (0..21_u32).for_each(|v| { vec.push(v); @@ -28,7 +28,7 @@ fn main() -> Result<(), Box> { { let mut vec: StoredVec = - StoredVec::forced_import(Path::new("./vec"), version, compressed)?; + StoredVec::forced_import(Path::new("."), "vec", version, compressed)?; let mut iter = vec.into_iter(); dbg!(iter.get(0)); @@ -54,7 +54,7 @@ fn main() -> Result<(), Box> { { let mut vec: StoredVec = - StoredVec::forced_import(Path::new("./vec"), version, compressed)?; + StoredVec::forced_import(Path::new("."), "vec", version, compressed)?; let mut iter = vec.into_iter(); dbg!(iter.get(0)); diff --git a/crates/brk_vec/src/traits/any.rs b/crates/brk_vec/src/traits/any.rs index 79662708d..bda7cfa15 100644 --- a/crates/brk_vec/src/traits/any.rs +++ b/crates/brk_vec/src/traits/any.rs @@ -12,7 +12,7 @@ pub trait AnyVec: Send + Sync { self.len() == 0 } fn modified_time(&self) -> Result; - fn index_type_to_string(&self) -> &str; + fn index_type_to_string(&self) -> String; } pub trait AnyIterableVec: AnyVec { diff --git a/crates/brk_vec/src/traits/index.rs b/crates/brk_vec/src/traits/index.rs index c0fb5d470..431c02133 100644 --- a/crates/brk_vec/src/traits/index.rs +++ b/crates/brk_vec/src/traits/index.rs @@ -1,4 +1,8 @@ -use std::{fmt::Debug, ops::Add}; +use std::{ + fmt::Debug, + ops::Add, + path::{Path, PathBuf}, +}; use crate::{Error, Result}; @@ -20,8 +24,14 @@ where { fn unwrap_to_usize(self) -> usize; fn to_usize(self) -> Result; - fn to_string<'a>() -> &'a str; + fn to_string() -> String; fn decremented(self) -> Option; + fn to_folder_name(value_name: &str) -> String { + format!("{}_to_{value_name}", Self::to_string().to_lowercase()) + } + fn path(path: &Path, value_name: &str) -> PathBuf { + path.join(Self::to_folder_name(value_name)) + } } impl StoredIndex for I @@ -51,8 +61,12 @@ where } #[inline] - fn to_string<'a>() -> &'a str { + fn to_string() -> String { std::any::type_name::() + .split("::") + .last() + .unwrap() + .to_lowercase() } #[inline] diff --git a/crates/brk_vec/src/traits/iterator.rs b/crates/brk_vec/src/traits/iterator.rs index 6ed1258da..485d3d85a 100644 --- a/crates/brk_vec/src/traits/iterator.rs +++ b/crates/brk_vec/src/traits/iterator.rs @@ -79,7 +79,7 @@ pub trait VecIterator<'a>: BaseVecIterator)> self.next().map(|(i, v)| (i, Value::Owned(v.into_inner()))) } - fn index_type_to_string(&self) -> &str { + fn index_type_to_string(&self) -> String { Self::I::to_string() } } diff --git a/crates/brk_vec/src/variants/compressed.rs b/crates/brk_vec/src/variants/compressed.rs index 82250a229..3862b383f 100644 --- a/crates/brk_vec/src/variants/compressed.rs +++ b/crates/brk_vec/src/variants/compressed.rs @@ -357,7 +357,7 @@ where } #[inline] - fn index_type_to_string(&self) -> &str { + fn index_type_to_string(&self) -> String { I::to_string() } } diff --git a/crates/brk_vec/src/variants/computed.rs b/crates/brk_vec/src/variants/computed.rs index 5ac7eb20f..65649b943 100644 --- a/crates/brk_vec/src/variants/computed.rs +++ b/crates/brk_vec/src/variants/computed.rs @@ -84,21 +84,20 @@ where pub fn forced_import_or_init_from_1( mode: Computation, path: &Path, - name: &str, + value_name: &str, version: Version, compressed: Compressed, source: BoxedAnyIterableVec, compute: ComputeFrom1, ) -> Result { - let full_path = path.join(name); Ok(match mode { Computation::Eager => Self::Eager { - vec: EagerVec::forced_import(&full_path, version, compressed)?, + vec: EagerVec::forced_import(path, value_name, version, compressed)?, deps: Dependencies::From1(source, compute), }, Computation::Lazy => { - let _ = fs::remove_dir_all(full_path); - Self::LazyFrom1(LazyVecFrom1::init(name, version, source, compute)) + let _ = fs::remove_dir_all(I::path(path, value_name)); + Self::LazyFrom1(LazyVecFrom1::init(value_name, version, source, compute)) } }) } @@ -107,22 +106,23 @@ where pub fn forced_import_or_init_from_2( mode: Computation, path: &Path, - name: &str, + value_name: &str, version: Version, compressed: Compressed, source1: BoxedAnyIterableVec, source2: BoxedAnyIterableVec, compute: ComputeFrom2, ) -> Result { - let full_path = path.join(name); Ok(match mode { Computation::Eager => Self::Eager { - vec: EagerVec::forced_import(&full_path, version, compressed)?, + vec: EagerVec::forced_import(path, value_name, version, compressed)?, deps: Dependencies::From2((source1, source2), compute), }, Computation::Lazy => { - let _ = fs::remove_dir_all(full_path); - Self::LazyFrom2(LazyVecFrom2::init(name, version, source1, source2, compute)) + let _ = fs::remove_dir_all(I::path(path, value_name)); + Self::LazyFrom2(LazyVecFrom2::init( + value_name, version, source1, source2, compute, + )) } }) } @@ -131,7 +131,7 @@ where pub fn forced_import_or_init_from_3( mode: Computation, path: &Path, - name: &str, + value_name: &str, version: Version, compressed: Compressed, source1: BoxedAnyIterableVec, @@ -139,16 +139,15 @@ where source3: BoxedAnyIterableVec, compute: ComputeFrom3, ) -> Result { - let full_path = path.join(name); Ok(match mode { Computation::Eager => Self::Eager { - vec: EagerVec::forced_import(&full_path, version, compressed)?, + vec: EagerVec::forced_import(path, value_name, version, compressed)?, deps: Dependencies::From3((source1, source2, source3), compute), }, Computation::Lazy => { - let _ = fs::remove_dir_all(full_path); + let _ = fs::remove_dir_all(I::path(path, value_name)); Self::LazyFrom3(LazyVecFrom3::init( - name, version, source1, source2, source3, compute, + value_name, version, source1, source2, source3, compute, )) } }) @@ -228,7 +227,7 @@ where } } - fn index_type_to_string(&self) -> &str { + fn index_type_to_string(&self) -> String { I::to_string() } diff --git a/crates/brk_vec/src/variants/eager.rs b/crates/brk_vec/src/variants/eager.rs index e60e01756..a5e6092ac 100644 --- a/crates/brk_vec/src/variants/eager.rs +++ b/crates/brk_vec/src/variants/eager.rs @@ -38,8 +38,13 @@ where { const SIZE_OF: usize = size_of::(); - pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result { - let inner = StoredVec::forced_import(path, version, compressed)?; + pub fn forced_import( + path: &Path, + value_name: &str, + version: Version, + compressed: Compressed, + ) -> Result { + let inner = StoredVec::forced_import(path, value_name, version, compressed)?; Ok(Self { computed_version: None, @@ -937,7 +942,7 @@ where } #[inline] - fn index_type_to_string(&self) -> &str { + fn index_type_to_string(&self) -> String { I::to_string() } } diff --git a/crates/brk_vec/src/variants/indexed.rs b/crates/brk_vec/src/variants/indexed.rs index 4225d9583..0c3f594ee 100644 --- a/crates/brk_vec/src/variants/indexed.rs +++ b/crates/brk_vec/src/variants/indexed.rs @@ -26,10 +26,17 @@ where I: StoredIndex, T: StoredType, { - pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result { + pub fn forced_import( + path: &Path, + value_name: &str, + version: Version, + compressed: Compressed, + ) -> Result { + let inner = StoredVec::forced_import(path, value_name, version, compressed)?; + Ok(Self { - height: Height::try_from(Self::path_height_(path).as_path()).ok(), - inner: StoredVec::forced_import(path, version, compressed)?, + height: Height::try_from(Self::path_height_(inner.path()).as_path()).ok(), + inner, }) } @@ -117,7 +124,7 @@ where } #[inline] - fn index_type_to_string(&self) -> &str { + fn index_type_to_string(&self) -> String { I::to_string() } } diff --git a/crates/brk_vec/src/variants/lazy1.rs b/crates/brk_vec/src/variants/lazy1.rs index 138a7de99..748e6be45 100644 --- a/crates/brk_vec/src/variants/lazy1.rs +++ b/crates/brk_vec/src/variants/lazy1.rs @@ -26,7 +26,7 @@ where S1T: StoredType, { pub fn init( - name: &str, + value_name: &str, version: Version, source: BoxedAnyIterableVec, compute: ComputeFrom1, @@ -36,7 +36,7 @@ where } Self { - name: name.to_owned(), + name: I::to_folder_name(value_name), version, source, compute, @@ -129,7 +129,7 @@ where self.name.clone() } - fn index_type_to_string(&self) -> &str { + fn index_type_to_string(&self) -> String { I::to_string() } diff --git a/crates/brk_vec/src/variants/lazy2.rs b/crates/brk_vec/src/variants/lazy2.rs index 5c606484b..71a6c5537 100644 --- a/crates/brk_vec/src/variants/lazy2.rs +++ b/crates/brk_vec/src/variants/lazy2.rs @@ -31,7 +31,7 @@ where S2T: StoredType, { pub fn init( - name: &str, + value_name: &str, version: Version, source1: BoxedAnyIterableVec, source2: BoxedAnyIterableVec, @@ -50,7 +50,7 @@ where } Self { - name: name.to_string(), + name: I::to_folder_name(value_name), version, source1, source2, @@ -163,7 +163,7 @@ where self.name.clone() } - fn index_type_to_string(&self) -> &str { + fn index_type_to_string(&self) -> String { I::to_string() } diff --git a/crates/brk_vec/src/variants/lazy3.rs b/crates/brk_vec/src/variants/lazy3.rs index ec916c499..6ef31c95e 100644 --- a/crates/brk_vec/src/variants/lazy3.rs +++ b/crates/brk_vec/src/variants/lazy3.rs @@ -35,7 +35,7 @@ where S3T: StoredType, { pub fn init( - name: &str, + value_name: &str, version: Version, source1: BoxedAnyIterableVec, source2: BoxedAnyIterableVec, @@ -56,7 +56,7 @@ where } Self { - name: name.to_string(), + name: I::to_folder_name(value_name), version, source1, source2, @@ -192,7 +192,7 @@ where self.name.clone() } - fn index_type_to_string(&self) -> &str { + fn index_type_to_string(&self) -> String { I::to_string() } diff --git a/crates/brk_vec/src/variants/raw.rs b/crates/brk_vec/src/variants/raw.rs index 4b6063dce..77e203a4b 100644 --- a/crates/brk_vec/src/variants/raw.rs +++ b/crates/brk_vec/src/variants/raw.rs @@ -191,7 +191,7 @@ where } #[inline] - fn index_type_to_string(&self) -> &str { + fn index_type_to_string(&self) -> String { I::to_string() } } diff --git a/crates/brk_vec/src/variants/stored.rs b/crates/brk_vec/src/variants/stored.rs index 6ef5c53a8..0ec41e4e1 100644 --- a/crates/brk_vec/src/variants/stored.rs +++ b/crates/brk_vec/src/variants/stored.rs @@ -21,13 +21,19 @@ where I: StoredIndex, T: StoredType, { - pub fn forced_import(path: &Path, version: Version, compressed: Compressed) -> Result { + pub fn forced_import( + path: &Path, + value_name: &str, + version: Version, + compressed: Compressed, + ) -> Result { + let path = I::path(path, value_name); if *compressed { Ok(Self::Compressed(CompressedVec::forced_import( - path, version, + &path, version, )?)) } else { - Ok(Self::Raw(RawVec::forced_import(path, version)?)) + Ok(Self::Raw(RawVec::forced_import(&path, version)?)) } } } @@ -113,7 +119,7 @@ where } #[inline] - fn index_type_to_string(&self) -> &str { + fn index_type_to_string(&self) -> String { I::to_string() } diff --git a/websites/kibo.money/scripts/vecid-to-indexes.js b/websites/kibo.money/scripts/vecid-to-indexes.js index d8a7591d0..830ed8d62 100644 --- a/websites/kibo.money/scripts/vecid-to-indexes.js +++ b/websites/kibo.money/scripts/vecid-to-indexes.js @@ -960,6 +960,9 @@ export function createVecIdToIndexes() { "total-tx-v1": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "total-tx-v2": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "total-tx-v3": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "total-unclaimed-rewards": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "total-unclaimed-rewards-in-btc": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "total-unclaimed-rewards-in-usd": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "total-unknownoutput-count": [DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "tx-count": [Height], "tx-count-10p": [DateIndex], @@ -997,6 +1000,12 @@ export function createVecIdToIndexes() { "txindex": [EmptyOutputIndex, OpReturnIndex, OutputIndex, P2MSIndex, TxIndex, UnknownOutputIndex], "txindex-count": [Height], "txversion": [TxIndex], + "unclaimed-rewards": [Height], + "unclaimed-rewards-in-btc": [Height], + "unclaimed-rewards-in-btc-sum": [DateIndex, DecadeIndex, DifficultyEpoch, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "unclaimed-rewards-in-usd": [Height], + "unclaimed-rewards-in-usd-sum": [DateIndex, DecadeIndex, DifficultyEpoch, MonthIndex, QuarterIndex, WeekIndex, YearIndex], + "unclaimed-rewards-sum": [DateIndex, DecadeIndex, DifficultyEpoch, MonthIndex, QuarterIndex, WeekIndex, YearIndex], "unknownoutput-count": [Height], "unknownoutput-count-10p": [DateIndex], "unknownoutput-count-25p": [DateIndex],