Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

vecs: part 10

Browse Source
This commit is contained in:
nym21
2025-07-25 20:22:54 +02:00
parent 49a66f72fc
commit dfc286b393
69 changed files with 1506 additions and 1323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
Cargo.lock generated
View File

@@ -519,7 +519,7 @@ dependencies = [
"brk_logger",
"brk_parser",
"brk_server",
"brk_vec",
"brk_vecs",
"clap",
"clap_derive",
"color-eyre",
@@ -543,7 +543,7 @@ dependencies = [
"brk_logger",
"brk_parser",
"brk_store",
"brk_vec",
"brk_vecs",
"color-eyre",
"derive_deref",
"fjall",
@@ -624,7 +624,7 @@ dependencies = [
"brk_computer",
"brk_core",
"brk_indexer",
"brk_vec",
"brk_vecs",
"color-eyre",
"derive_deref",
"rmcp",
@@ -997,7 +997,7 @@ dependencies = [
"brk_logger",
"brk_mcp",
"brk_parser",
"brk_vec",
"brk_vecs",
"clap",
"clap_derive",
"color-eyre",
@@ -1057,9 +1057,10 @@ dependencies = [
name = "brk_vecs"
version = "0.0.81"
dependencies = [
"bincode",
"brk_core",
"brk_exit",
"clap",
"clap_derive",
"libc",
"log",
"memmap2",

2
crates/brk_cli/Cargo.toml
View File

@@ -17,7 +17,7 @@ brk_indexer = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_server = { workspace = true }
brk_vec = { workspace = true }
brk_vecs = { workspace = true }
clap = { workspace = true }
clap_derive = { workspace = true }
color-eyre = { workspace = true }

6
crates/brk_cli/src/config.rs
View File

@@ -7,13 +7,12 @@ use bitcoincore_rpc::{self, Auth, Client};
use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
use brk_fetcher::Fetcher;
use brk_server::Website;
use brk_vec::{Computation, Format};
use brk_vecs::{Computation, Format};
use clap::Parser;
use clap_derive::Parser;
use color_eyre::eyre::eyre;
use serde::{Deserialize, Serialize};
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
#[command(version, about)]
pub struct Config {
@@ -32,7 +31,6 @@ pub struct Config {
#[arg(long, value_name = "PATH")]
brkdir: Option<String>,
/// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
@@ -124,7 +122,6 @@ impl Config {
config_saved.brkdir = Some(brkdir);
}
if let Some(computation) = config_args.computation.take() {
config_saved.computation = Some(computation);
}
@@ -298,7 +295,6 @@ impl Config {
self.outputsdir().join("hars")
}
fn path_cookiefile(&self) -> PathBuf {
self.rpccookiefile.as_ref().map_or_else(
|| self.bitcoindir().join(".cookie"),

2
crates/brk_computer/Cargo.toml
View File

@@ -18,7 +18,7 @@ brk_indexer = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_store = { workspace = true }
brk_vec = { workspace = true }
brk_vecs = { workspace = true }
color-eyre = { workspace = true }
derive_deref = { workspace = true }
fjall = { workspace = true }

2
crates/brk_computer/examples/main.rs
View File

@@ -5,7 +5,7 @@ use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_vec::{Computation, Format};
use brk_vecs::{Computation, Format};
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;

29
crates/brk_computer/src/all.rs
View File

@@ -1,10 +1,10 @@
use std::path::Path;
use std::{path::Path, sync::Arc};
use brk_core::Version;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Computation, Format};
use brk_vecs::{AnyCollectableVec, Computation, File, Format};
use log::info;
use crate::{blocks, cointime, constants, fetched, indexes, market, mining, transactions};
@@ -27,16 +27,19 @@ pub struct Vecs {
}
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn import(
path: &Path,
file: &Arc<File>,
version: Version,
indexer: &Indexer,
fetch: bool,
computation: Computation,
format: Format,
fetched_file: &Arc<File>,
states_path: &Path,
) -> color_eyre::Result<Self> {
let indexes = indexes::Vecs::forced_import(
path,
file,
version + VERSION + Version::ZERO,
indexer,
computation,
@@ -45,7 +48,8 @@ impl Vecs {
let fetched = fetch.then(|| {
fetched::Vecs::forced_import(
path,
file,
fetched_file,
version + VERSION + Version::ZERO,
computation,
format,
@@ -56,43 +60,44 @@ impl Vecs {
Ok(Self {
blocks: blocks::Vecs::forced_import(
path,
file,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
mining: mining::Vecs::forced_import(
path,
file,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
constants: constants::Vecs::forced_import(
path,
file,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
market: market::Vecs::forced_import(
path,
file,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
)?,
stateful: stateful::Vecs::forced_import(
path,
file,
version + VERSION + Version::ZERO,
computation,
format,
&indexes,
fetched.as_ref(),
states_path,
)?,
transactions: transactions::Vecs::forced_import(
path,
file,
version + VERSION + Version::ZERO,
indexer,
&indexes,
@@ -101,7 +106,7 @@ impl Vecs {
fetched.as_ref(),
)?,
cointime: cointime::Vecs::forced_import(
path,
file,
version + VERSION + Version::ZERO,
computation,
format,

26
crates/brk_computer/src/blocks.rs
View File

@@ -1,4 +1,4 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize,
@@ -6,7 +6,7 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, File, Format};
use crate::grouped::Source;
@@ -34,7 +34,7 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
file: &Arc<File>,
version: Version,
computation: Computation,
format: Format,
@@ -42,13 +42,13 @@ impl Vecs {
) -> color_eyre::Result<Self> {
Ok(Self {
height_to_interval: EagerVec::forced_import(
path,
file,
"interval",
version + VERSION + Version::ZERO,
format,
)?,
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
path,
file,
"timestamp",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -58,7 +58,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(),
)?,
indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
path,
file,
"block_interval",
Source::None,
version + VERSION + Version::ZERO,
@@ -71,7 +71,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_block_count: ComputedVecsFromHeight::forced_import(
path,
file,
"block_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -81,7 +81,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
path,
file,
"block_weight",
Source::None,
version + VERSION + Version::ZERO,
@@ -91,7 +91,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_block_size: ComputedVecsFromHeight::forced_import(
path,
file,
"block_size",
Source::None,
version + VERSION + Version::ZERO,
@@ -101,13 +101,13 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
height_to_vbytes: EagerVec::forced_import(
path,
file,
"vbytes",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
path,
file,
"block_vbytes",
Source::None,
version + VERSION + Version::ZERO,
@@ -117,13 +117,13 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
difficultyepoch_to_timestamp: EagerVec::forced_import(
path,
file,
"timestamp",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_timestamp: EagerVec::forced_import(
path,
file,
"timestamp",
version + VERSION + Version::ZERO,
format,

52
crates/brk_computer/src/cointime.rs
View File

@@ -1,9 +1,9 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{Bitcoin, CheckedSub, Dollars, StoredF64, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
use brk_vecs::{AnyCollectableVec, Computation, File, Format, VecIterator};
use super::{
Indexes, fetched,
@@ -46,7 +46,7 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
file: &Arc<File>,
version: Version,
computation: Computation,
format: Format,
@@ -57,7 +57,7 @@ impl Vecs {
Ok(Self {
indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import(
path,
file,
"coinblocks_created",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -67,7 +67,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import(
path,
file,
"coinblocks_stored",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -77,7 +77,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_liveliness: ComputedVecsFromHeight::forced_import(
path,
file,
"liveliness",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -87,7 +87,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaultedness: ComputedVecsFromHeight::forced_import(
path,
file,
"vaultedness",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -97,7 +97,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import(
path,
file,
"activity_to_vaultedness_ratio",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -107,7 +107,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import(
path,
file,
"vaulted_supply",
Source::Compute,
version + VERSION + Version::ONE,
@@ -118,7 +118,7 @@ impl Vecs {
indexes,
)?,
indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import(
path,
file,
"active_supply",
Source::Compute,
version + VERSION + Version::ONE,
@@ -129,7 +129,7 @@ impl Vecs {
indexes,
)?,
indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import(
path,
file,
"thermo_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -139,7 +139,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_investor_cap: ComputedVecsFromHeight::forced_import(
path,
file,
"investor_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -149,7 +149,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import(
path,
file,
"vaulted_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -159,7 +159,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_active_cap: ComputedVecsFromHeight::forced_import(
path,
file,
"active_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -169,7 +169,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import(
path,
file,
"vaulted_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -179,7 +179,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
path,
file,
"vaulted_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -189,7 +189,7 @@ impl Vecs {
true,
)?,
indexes_to_active_price: ComputedVecsFromHeight::forced_import(
path,
file,
"active_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -199,7 +199,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
path,
file,
"active_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -209,7 +209,7 @@ impl Vecs {
true,
)?,
indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import(
path,
file,
"true_market_mean",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -219,7 +219,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import(
path,
file,
"true_market_mean",
Source::None,
version + VERSION + Version::ZERO,
@@ -229,7 +229,7 @@ impl Vecs {
true,
)?,
indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import(
path,
file,
"cointime_value_destroyed",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -239,7 +239,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import(
path,
file,
"cointime_value_created",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -249,7 +249,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import(
path,
file,
"cointime_value_stored",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -259,7 +259,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_price: ComputedVecsFromHeight::forced_import(
path,
file,
"cointime_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -269,7 +269,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import(
path,
file,
"cointime_cap",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -279,7 +279,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
path,
file,
"cointime_price",
Source::None,
version + VERSION + Version::ZERO,

14
crates/brk_computer/src/constants.rs
View File

@@ -1,9 +1,9 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{StoredU8, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format};
use brk_vecs::{AnyCollectableVec, AnyVec, Computation, File, Format};
use crate::grouped::Source;
@@ -25,7 +25,7 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
file: &Arc<File>,
version: Version,
computation: Computation,
format: Format,
@@ -33,7 +33,7 @@ impl Vecs {
) -> color_eyre::Result<Self> {
Ok(Self {
constant_0: ComputedVecsFromHeight::forced_import(
path,
file,
"constant_0",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -43,7 +43,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_1: ComputedVecsFromHeight::forced_import(
path,
file,
"constant_1",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -53,7 +53,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_50: ComputedVecsFromHeight::forced_import(
path,
file,
"constant_50",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -63,7 +63,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_100: ComputedVecsFromHeight::forced_import(
path,
file,
"constant_100",
Source::Compute,
version + VERSION + Version::ZERO,

103
crates/brk_computer/src/fetched.rs
View File

@@ -1,4 +1,4 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{
Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex,
@@ -8,8 +8,8 @@ use brk_core::{
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, StoredIndex,
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, File, Format, StoredIndex,
VecIterator,
};
@@ -76,103 +76,100 @@ const VERSION_IN_SATS: Version = Version::ZERO;
impl Vecs {
pub fn forced_import(
path: &Path,
file: &Arc<File>,
fetched_file: &Arc<File>,
version: Version,
computation: Computation,
format: Format,
indexes: &indexes::Vecs,
) -> color_eyre::Result<Self> {
let mut fetched_path = path.to_owned();
fetched_path.pop();
fetched_path = fetched_path.join("fetched");
Ok(Self {
dateindex_to_ohlc_in_cents: EagerVec::forced_import(
&fetched_path,
fetched_file,
"ohlc_in_cents",
version + Version::ZERO,
format,
)?,
dateindex_to_ohlc: EagerVec::forced_import(
path,
file,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
file,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
dateindex_to_close_in_cents: EagerVec::forced_import(
path,
file,
"close_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_high_in_cents: EagerVec::forced_import(
path,
file,
"high_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_low_in_cents: EagerVec::forced_import(
path,
file,
"low_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_open_in_cents: EagerVec::forced_import(
path,
file,
"open_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_ohlc_in_cents: EagerVec::forced_import(
&fetched_path,
fetched_file,
"ohlc_in_cents",
version + Version::ZERO,
format,
)?,
height_to_ohlc: EagerVec::forced_import(
path,
file,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
height_to_ohlc_in_sats: EagerVec::forced_import(
path,
file,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
height_to_close_in_cents: EagerVec::forced_import(
path,
file,
"close_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_high_in_cents: EagerVec::forced_import(
path,
file,
"high_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_low_in_cents: EagerVec::forced_import(
path,
file,
"low_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_open_in_cents: EagerVec::forced_import(
path,
file,
"open_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
timeindexes_to_open: ComputedVecsFromDateIndex::forced_import(
path,
file,
"open",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -182,7 +179,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(),
)?,
timeindexes_to_high: ComputedVecsFromDateIndex::forced_import(
path,
file,
"high",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -192,7 +189,7 @@ impl Vecs {
VecBuilderOptions::default().add_max(),
)?,
timeindexes_to_low: ComputedVecsFromDateIndex::forced_import(
path,
file,
"low",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -202,7 +199,7 @@ impl Vecs {
VecBuilderOptions::default().add_min(),
)?,
timeindexes_to_close: ComputedVecsFromDateIndex::forced_import(
path,
file,
"close",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -212,7 +209,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
file,
"open_in_sats",
Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -222,7 +219,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(),
)?,
timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
file,
"high_in_sats",
Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -232,7 +229,7 @@ impl Vecs {
VecBuilderOptions::default().add_max(),
)?,
timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
file,
"low_in_sats",
Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -242,7 +239,7 @@ impl Vecs {
VecBuilderOptions::default().add_min(),
)?,
timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
file,
"close_in_sats",
Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -252,143 +249,143 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import(
path,
file,
"open",
version + VERSION + Version::ZERO,
format,
VecBuilderOptions::default().add_first(),
)?,
chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import(
path,
file,
"high",
version + VERSION + Version::ZERO,
format,
VecBuilderOptions::default().add_max(),
)?,
chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import(
path,
file,
"low",
version + VERSION + Version::ZERO,
format,
VecBuilderOptions::default().add_min(),
)?,
chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import(
path,
file,
"close",
version + VERSION + Version::ZERO,
format,
VecBuilderOptions::default().add_last(),
)?,
chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import(
path,
file,
"open_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
VecBuilderOptions::default().add_first(),
)?,
chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import(
path,
file,
"high_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
VecBuilderOptions::default().add_max(),
)?,
chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import(
path,
file,
"low_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
VecBuilderOptions::default().add_min(),
)?,
chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import(
path,
file,
"close_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
VecBuilderOptions::default().add_last(),
)?,
weekindex_to_ohlc: EagerVec::forced_import(
path,
file,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
file,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
difficultyepoch_to_ohlc: EagerVec::forced_import(
path,
file,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_ohlc_in_sats: EagerVec::forced_import(
path,
file,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
monthindex_to_ohlc: EagerVec::forced_import(
path,
file,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
file,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
quarterindex_to_ohlc: EagerVec::forced_import(
path,
file,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
file,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
semesterindex_to_ohlc: EagerVec::forced_import(
path,
file,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
file,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
yearindex_to_ohlc: EagerVec::forced_import(
path,
file,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
file,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
)?,
// halvingepoch_to_ohlc: StorableVec::forced_import(path,
// halvingepoch_to_ohlc: StorableVec::forced_import(file,
// "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?,
decadeindex_to_ohlc: EagerVec::forced_import(
path,
file,
"ohlc",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_ohlc_in_sats: EagerVec::forced_import(
path,
file,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,

24
crates/brk_computer/src/grouped/builder_computed.rs
View File

@@ -1,10 +1,10 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{FromCoarserIndex, Result, Version};
use brk_exit::Exit;
use brk_vec::{
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, Computation,
ComputedVec, ComputedVecFrom2, Format, StoredIndex,
ComputedVec, ComputedVecFrom2, File, Format, StoredIndex,
};
use crate::grouped::{EagerVecBuilder, VecBuilderOptions};
@@ -39,7 +39,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
file: &Arc<File>,
name: &str,
version: Version,
format: Format,
@@ -66,7 +66,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
file,
&maybe_suffix("first"),
version + VERSION + Version::ZERO,
format,
@@ -91,7 +91,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -100,7 +100,7 @@ where
source
.as_ref()
.unwrap_or_else(|| {
dbg!(path, name, I::to_string());
dbg!(file, name, I::to_string());
panic!()
})
.clone()
@@ -124,7 +124,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
file,
&maybe_suffix("min"),
version + VERSION + Version::ZERO,
format,
@@ -149,7 +149,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
file,
&maybe_suffix("max"),
version + VERSION + Version::ZERO,
format,
@@ -174,7 +174,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
file,
&maybe_suffix("average"),
version + VERSION + Version::ZERO,
format,
@@ -206,7 +206,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
file,
&(if !options.last && !options.average && !options.min && !options.max {
name.to_string()
} else {
@@ -241,7 +241,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
path,
file,
&suffix("cumulative"),
version + VERSION + Version::ZERO,
format,

34
crates/brk_computer/src/grouped/builder_eager.rs
View File

@@ -1,8 +1,10 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{CheckedSub, Result, StoredUsize, Version};
use brk_exit::Exit;
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format, StoredIndex, StoredType};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, EagerVec, File, Format, StoredIndex, StoredType,
};
use color_eyre::eyre::ContextCompat;
use crate::utils::get_percentile;
@@ -37,7 +39,7 @@ where
T: ComputedType,
{
pub fn forced_import(
path: &Path,
file: &Arc<File>,
name: &str,
version: Version,
format: Format,
@@ -59,7 +61,7 @@ where
first: options.first.then(|| {
Box::new(
EagerVec::forced_import(
path,
file,
&maybe_suffix("first"),
version + VERSION + Version::ZERO,
format,
@@ -69,13 +71,13 @@ where
}),
last: options.last.then(|| {
Box::new(
EagerVec::forced_import(path, name, version + Version::ZERO, format).unwrap(),
EagerVec::forced_import(file, name, version + Version::ZERO, format).unwrap(),
)
}),
min: options.min.then(|| {
Box::new(
EagerVec::forced_import(
path,
file,
&maybe_suffix("min"),
version + VERSION + Version::ZERO,
format,
@@ -86,7 +88,7 @@ where
max: options.max.then(|| {
Box::new(
EagerVec::forced_import(
path,
file,
&maybe_suffix("max"),
version + VERSION + Version::ZERO,
format,
@@ -97,7 +99,7 @@ where
median: options.median.then(|| {
Box::new(
EagerVec::forced_import(
path,
file,
&maybe_suffix("median"),
version + VERSION + Version::ZERO,
format,
@@ -108,7 +110,7 @@ where
average: options.average.then(|| {
Box::new(
EagerVec::forced_import(
path,
file,
&maybe_suffix("average"),
version + VERSION + Version::ZERO,
format,
@@ -119,7 +121,7 @@ where
sum: options.sum.then(|| {
Box::new(
EagerVec::forced_import(
path,
file,
&(if !options.last && !options.average && !options.min && !options.max {
name.to_string()
} else {
@@ -134,7 +136,7 @@ where
cumulative: options.cumulative.then(|| {
Box::new(
EagerVec::forced_import(
path,
file,
&suffix("cumulative"),
version + VERSION + Version::ZERO,
format,
@@ -145,7 +147,7 @@ where
_90p: options._90p.then(|| {
Box::new(
EagerVec::forced_import(
path,
file,
&maybe_suffix("90p"),
version + VERSION + Version::ZERO,
format,
@@ -156,7 +158,7 @@ where
_75p: options._75p.then(|| {
Box::new(
EagerVec::forced_import(
path,
file,
&maybe_suffix("75p"),
version + VERSION + Version::ZERO,
format,
@@ -167,7 +169,7 @@ where
_25p: options._25p.then(|| {
Box::new(
EagerVec::forced_import(
path,
file,
&maybe_suffix("25p"),
version + VERSION + Version::ZERO,
format,
@@ -178,7 +180,7 @@ where
_10p: options._10p.then(|| {
Box::new(
EagerVec::forced_import(
path,
file,
&maybe_suffix("10p"),
version + VERSION + Version::ZERO,
format,
@@ -309,7 +311,7 @@ where
.inspect_err(|_| {
dbg!(
&values,
max.path(),
max.name(),
first_indexes.name(),
first_index,
count_indexes.name(),

24
crates/brk_computer/src/grouped/from_dateindex.rs
View File

@@ -1,4 +1,4 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, SemesterIndex, Version, WeekIndex,
@@ -6,8 +6,8 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format,
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, File, Format,
};
use crate::{Indexes, grouped::ComputedVecBuilder, indexes};
@@ -37,7 +37,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
file: &Arc<File>,
name: &str,
source: Source<DateIndex, T>,
version: Version,
@@ -47,11 +47,11 @@ where
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
let dateindex = source.is_compute().then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap()
});
let dateindex_extra = EagerVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -64,7 +64,7 @@ where
Ok(Self {
weekindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -75,7 +75,7 @@ where
options.into(),
)?,
monthindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -86,7 +86,7 @@ where
options.into(),
)?,
quarterindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -97,7 +97,7 @@ where
options.into(),
)?,
semesterindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -108,7 +108,7 @@ where
options.into(),
)?,
yearindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -119,7 +119,7 @@ where
options.into(),
)?,
decadeindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,

30
crates/brk_computer/src/grouped/from_height.rs
View File

@@ -1,4 +1,4 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result,
@@ -6,8 +6,8 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format,
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, File, Format,
};
use crate::{
@@ -45,7 +45,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
file: &Arc<File>,
name: &str,
source: Source<Height, T>,
version: Version,
@@ -55,11 +55,11 @@ where
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
let height = source.is_compute().then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap()
});
let height_extra = EagerVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -67,7 +67,7 @@ where
)?;
let dateindex = EagerVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -78,7 +78,7 @@ where
Ok(Self {
weekindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -89,7 +89,7 @@ where
options.into(),
)?,
monthindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -100,7 +100,7 @@ where
options.into(),
)?,
quarterindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -111,7 +111,7 @@ where
options.into(),
)?,
semesterindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -122,7 +122,7 @@ where
options.into(),
)?,
yearindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -133,7 +133,7 @@ where
options.into(),
)?,
decadeindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -143,12 +143,12 @@ where
indexes.decadeindex_to_decadeindex.boxed_clone(),
options.into(),
)?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
// halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
height,
height_extra,
dateindex,
difficultyepoch: EagerVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,

14
crates/brk_computer/src/grouped/from_height_strict.rs
View File

@@ -1,9 +1,9 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{DifficultyEpoch, Height, Result, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, EagerVec, Format};
use brk_vecs::{AnyCollectableVec, EagerVec, File, Format};
use crate::{Indexes, indexes};
@@ -28,17 +28,17 @@ where
f64: From<T>,
{
pub fn forced_import(
path: &Path,
file: &Arc<File>,
name: &str,
version: Version,
format: Format,
options: VecBuilderOptions,
) -> color_eyre::Result<Self> {
let height =
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)?;
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)?;
let height_extra = EagerVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -51,13 +51,13 @@ where
height,
height_extra,
difficultyepoch: EagerVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
// halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
})
}

30
crates/brk_computer/src/grouped/from_txindex.rs
View File

@@ -1,4 +1,4 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
@@ -6,9 +6,9 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
use brk_vecs::{
AnyCollectableVec, AnyVec, CloneableAnyIterableVec, CollectableVec, Computation, EagerVec,
Format, StoredIndex, VecIterator,
File, Format, StoredIndex, VecIterator,
};
use crate::{
@@ -46,7 +46,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
file: &Arc<File>,
name: &str,
source: Source<TxIndex, T>,
version: Version,
@@ -57,13 +57,13 @@ where
) -> color_eyre::Result<Self> {
let txindex = source.is_compute().then(|| {
Box::new(
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)
.unwrap(),
)
});
let height = EagerVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -73,7 +73,7 @@ where
let options = options.remove_percentiles();
let dateindex = EagerVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -82,7 +82,7 @@ where
Ok(Self {
weekindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -93,7 +93,7 @@ where
options.into(),
)?,
monthindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -104,7 +104,7 @@ where
options.into(),
)?,
quarterindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -115,7 +115,7 @@ where
options.into(),
)?,
semesterindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -126,7 +126,7 @@ where
options.into(),
)?,
yearindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -137,7 +137,7 @@ where
options.into(),
)?,
decadeindex: ComputedVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
@@ -152,13 +152,13 @@ where
height,
dateindex,
difficultyepoch: EagerVecBuilder::forced_import(
path,
file,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
// halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
})
}

88
crates/brk_computer/src/grouped/ratio_from_dateindex.rs
View File

@@ -1,10 +1,10 @@
use std::{f32, path::Path};
use std::{f32, sync::Arc};
use brk_core::{Date, DateIndex, Dollars, Result, StoredF32, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Computation, EagerVec, Format,
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Computation, EagerVec, File, Format,
StoredIndex, VecIterator,
};
@@ -60,7 +60,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedRatioVecsFromDateIndex {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
file: &Arc<File>,
name: &str,
source: Source<DateIndex, Dollars>,
version: Version,
@@ -74,7 +74,7 @@ impl ComputedRatioVecsFromDateIndex {
Ok(Self {
price: source.is_compute().then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
name,
Source::Compute,
version + VERSION,
@@ -86,7 +86,7 @@ impl ComputedRatioVecsFromDateIndex {
.unwrap()
}),
ratio: ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -97,7 +97,7 @@ impl ComputedRatioVecsFromDateIndex {
)?,
ratio_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -110,7 +110,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1w_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_1w_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -123,7 +123,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1m_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_1m_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -136,7 +136,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_1y_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -149,7 +149,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_4y_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_4y_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -162,7 +162,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_sma_momentum_oscillator: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_1y_sma_momentum_oscillator"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -175,7 +175,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -188,7 +188,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_4y_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_4y_sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -201,7 +201,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_1y_sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -214,7 +214,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_9: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p99_9"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -227,7 +227,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_5: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p99_5"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -240,7 +240,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p99"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -253,7 +253,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p1"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -266,7 +266,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_5: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p0_5"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -279,7 +279,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_1: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p0_1"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -292,7 +292,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p1sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -305,7 +305,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p2sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p2sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -318,7 +318,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p3sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p3sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -331,7 +331,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m1sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_m1sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -344,7 +344,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m2sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_m2sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -357,7 +357,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m3sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_m3sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -370,7 +370,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_9_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p99_9_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -383,7 +383,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_5_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p99_5_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -396,7 +396,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p99_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -409,7 +409,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p1_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -422,7 +422,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_5_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p0_5_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -435,7 +435,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_1_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p0_1_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -448,7 +448,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p1sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -461,7 +461,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p2sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p2sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -474,7 +474,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p3sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_p3sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -487,7 +487,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m1sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_m1sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -500,7 +500,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m2sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_m2sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -513,7 +513,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m3sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_m3sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -526,7 +526,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_zscore"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -539,7 +539,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_4y_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_4y_zscore"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -552,7 +552,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_ratio_1y_zscore"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -1069,6 +1069,10 @@ impl ComputedRatioVecsFromDateIndex {
Ok(())
})?;
drop(sma_iter);
drop(_4y_sma_iter);
drop(_1y_sma_iter);
self.mut_ratio_vecs()
.into_iter()
.try_for_each(|v| v.safe_flush(exit))?;

2
crates/brk_computer/src/grouped/source.rs
View File

@@ -1,4 +1,4 @@
use brk_vec::BoxedAnyIterableVec;
use brk_vecs::BoxedAnyIterableVec;
#[derive(Clone)]
pub enum Source<I, T> {

2
crates/brk_computer/src/grouped/type.rs
View File

@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign, Div};
use brk_vec::StoredType;
use brk_vecs::StoredType;
pub trait ComputedType
where

12
crates/brk_computer/src/grouped/value_from_dateindex.rs
View File

@@ -1,9 +1,9 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec};
use brk_vecs::{AnyCollectableVec, CollectableVec, Computation, EagerVec, File, Format, StoredVec};
use crate::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
@@ -21,7 +21,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromDateIndex {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
file: &Arc<File>,
name: &str,
source: Source<DateIndex, Sats>,
version: Version,
@@ -33,7 +33,7 @@ impl ComputedValueVecsFromDateIndex {
) -> color_eyre::Result<Self> {
Ok(Self {
sats: ComputedVecsFromDateIndex::forced_import(
path,
file,
name,
source,
version + VERSION,
@@ -43,7 +43,7 @@ impl ComputedValueVecsFromDateIndex {
options,
)?,
bitcoin: ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_in_btc"),
Source::Compute,
version + VERSION,
@@ -54,7 +54,7 @@ impl ComputedValueVecsFromDateIndex {
)?,
dollars: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&format!("{name}_in_usd"),
Source::Compute,
version + VERSION,

12
crates/brk_computer/src/grouped/value_from_height.rs
View File

@@ -1,9 +1,9 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec};
use brk_vecs::{AnyCollectableVec, CollectableVec, Computation, EagerVec, File, Format, StoredVec};
use crate::{Indexes, fetched, grouped::Source, indexes};
@@ -21,7 +21,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromHeight {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
file: &Arc<File>,
name: &str,
source: Source<Height, Sats>,
version: Version,
@@ -33,7 +33,7 @@ impl ComputedValueVecsFromHeight {
) -> color_eyre::Result<Self> {
Ok(Self {
sats: ComputedVecsFromHeight::forced_import(
path,
file,
name,
source,
version + VERSION,
@@ -43,7 +43,7 @@ impl ComputedValueVecsFromHeight {
options,
)?,
bitcoin: ComputedVecsFromHeight::forced_import(
path,
file,
&format!("{name}_in_btc"),
Source::Compute,
version + VERSION,
@@ -54,7 +54,7 @@ impl ComputedValueVecsFromHeight {
)?,
dollars: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&format!("{name}_in_usd"),
Source::Compute,
version + VERSION,

16
crates/brk_computer/src/grouped/value_from_txindex.rs
View File

@@ -1,11 +1,11 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
use brk_vecs::{
AnyCollectableVec, CloneableAnyIterableVec, CollectableVec, Computation, ComputedVecFrom3,
Format, LazyVecFrom1, StoredIndex, StoredVec,
File, Format, LazyVecFrom1, StoredIndex, StoredVec,
};
use crate::{Indexes, fetched, grouped::Source, indexes};
@@ -38,7 +38,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromTxindex {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
file: &Arc<File>,
name: &str,
indexes: &indexes::Vecs,
source: Source<TxIndex, Sats>,
@@ -54,7 +54,7 @@ impl ComputedValueVecsFromTxindex {
let name_in_usd = format!("{name}_in_usd");
let sats = ComputedVecsFromTxindex::forced_import(
path,
file,
name,
source.clone(),
version + VERSION,
@@ -79,7 +79,7 @@ impl ComputedValueVecsFromTxindex {
);
let bitcoin = ComputedVecsFromTxindex::forced_import(
path,
file,
&name_in_btc,
Source::None,
version + VERSION,
@@ -92,7 +92,7 @@ impl ComputedValueVecsFromTxindex {
let dollars_txindex = fetched.map(|fetched| {
ComputedVecFrom3::forced_import_or_init_from_3(
computation,
path,
file,
&name_in_usd,
version + VERSION,
format,
@@ -127,7 +127,7 @@ impl ComputedValueVecsFromTxindex {
dollars_txindex,
dollars: compute_dollars.then(|| {
ComputedVecsFromTxindex::forced_import(
path,
file,
&name_in_usd,
Source::None,
version + VERSION,

12
crates/brk_computer/src/grouped/value_height.rs
View File

@@ -1,9 +1,9 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
use brk_vecs::{AnyCollectableVec, CollectableVec, EagerVec, File, Format, StoredVec};
use crate::{Indexes, fetched, grouped::Source, indexes};
@@ -18,7 +18,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedHeightValueVecs {
pub fn forced_import(
path: &Path,
file: &Arc<File>,
name: &str,
source: Source<Height, Sats>,
version: Version,
@@ -27,18 +27,18 @@ impl ComputedHeightValueVecs {
) -> color_eyre::Result<Self> {
Ok(Self {
sats: source.is_compute().then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)
.unwrap()
}),
bitcoin: EagerVec::forced_import(
path,
file,
&format!("{name}_in_btc"),
version + VERSION + Version::ZERO,
format,
)?,
dollars: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&format!("{name}_in_usd"),
version + VERSION + Version::ZERO,
format,

128
crates/brk_computer/src/indexes.rs
View File

@@ -1,4 +1,4 @@
use std::{ops::Deref, path::Path};
use std::{ops::Deref, sync::Arc};
use brk_core::{
Date, DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height,
@@ -10,9 +10,9 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
use brk_vecs::{
AnyCollectableVec, CloneableAnyIterableVec, Computation, ComputedVec, ComputedVecFrom1,
ComputedVecFrom2, EagerVec, Format, StoredIndex, VecIterator,
ComputedVecFrom2, EagerVec, File, Format, StoredIndex, VecIterator,
};
const VERSION: Version = Version::ZERO;
@@ -97,7 +97,7 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
file: &Arc<File>,
version: Version,
indexer: &Indexer,
computation: Computation,
@@ -105,7 +105,7 @@ impl Vecs {
) -> color_eyre::Result<Self> {
let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"outputindex",
version + VERSION + Version::ZERO,
format,
@@ -115,7 +115,7 @@ impl Vecs {
let inputindex_to_inputindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"inputindex",
version + VERSION + Version::ZERO,
format,
@@ -125,7 +125,7 @@ impl Vecs {
let txindex_to_txindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"txindex",
version + VERSION + Version::ZERO,
format,
@@ -135,7 +135,7 @@ impl Vecs {
let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2(
computation,
path,
file,
"input_count",
version + VERSION + Version::ZERO,
format,
@@ -158,7 +158,7 @@ impl Vecs {
let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2(
computation,
path,
file,
"output_count",
version + VERSION + Version::ZERO,
format,
@@ -181,7 +181,7 @@ impl Vecs {
let p2pk33addressindex_to_p2pk33addressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"p2pk33addressindex",
version + VERSION + Version::ZERO,
format,
@@ -190,7 +190,7 @@ impl Vecs {
)?;
let p2pk65addressindex_to_p2pk65addressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"p2pk65addressindex",
version + VERSION + Version::ZERO,
format,
@@ -199,7 +199,7 @@ impl Vecs {
)?;
let p2pkhaddressindex_to_p2pkhaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"p2pkhaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -208,7 +208,7 @@ impl Vecs {
)?;
let p2shaddressindex_to_p2shaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"p2shaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -217,7 +217,7 @@ impl Vecs {
)?;
let p2traddressindex_to_p2traddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"p2traddressindex",
version + VERSION + Version::ZERO,
format,
@@ -226,7 +226,7 @@ impl Vecs {
)?;
let p2wpkhaddressindex_to_p2wpkhaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"p2wpkhaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -235,7 +235,7 @@ impl Vecs {
)?;
let p2wshaddressindex_to_p2wshaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"p2wshaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -244,7 +244,7 @@ impl Vecs {
)?;
let p2aaddressindex_to_p2aaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"p2aaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -253,7 +253,7 @@ impl Vecs {
)?;
let p2msoutputindex_to_p2msoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"p2msoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -262,7 +262,7 @@ impl Vecs {
)?;
let emptyoutputindex_to_emptyoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"emptyoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -271,7 +271,7 @@ impl Vecs {
)?;
let unknownoutputindex_to_unknownoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"unknownoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -280,7 +280,7 @@ impl Vecs {
)?;
let opreturnindex_to_opreturnindex = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"opreturnindex",
version + VERSION + Version::ZERO,
format,
@@ -308,259 +308,259 @@ impl Vecs {
unknownoutputindex_to_unknownoutputindex,
dateindex_to_date: EagerVec::forced_import(
path,
file,
"date",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_dateindex: EagerVec::forced_import(
path,
file,
"dateindex",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_first_height: EagerVec::forced_import(
path,
file,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_monthindex: EagerVec::forced_import(
path,
file,
"monthindex",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_weekindex: EagerVec::forced_import(
path,
file,
"weekindex",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_decadeindex: EagerVec::forced_import(
path,
file,
"decadeindex",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_first_yearindex: EagerVec::forced_import(
path,
file,
"first_yearindex",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_difficultyepoch: EagerVec::forced_import(
path,
file,
"difficultyepoch",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_first_height: EagerVec::forced_import(
path,
file,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_first_height: EagerVec::forced_import(
path,
file,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_halvingepoch: EagerVec::forced_import(
path,
file,
"halvingepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_date: EagerVec::forced_import(
path,
file,
"date",
version + VERSION + Version::ZERO,
format,
)?,
height_to_difficultyepoch: EagerVec::forced_import(
path,
file,
"difficultyepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_halvingepoch: EagerVec::forced_import(
path,
file,
"halvingepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_height: EagerVec::forced_import(
path,
file,
"height",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_first_dateindex: EagerVec::forced_import(
path,
file,
"first_dateindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_monthindex: EagerVec::forced_import(
path,
file,
"monthindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_quarterindex: EagerVec::forced_import(
path,
file,
"quarterindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_semesterindex: EagerVec::forced_import(
path,
file,
"semesterindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_yearindex: EagerVec::forced_import(
path,
file,
"yearindex",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_first_monthindex: EagerVec::forced_import(
path,
file,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_first_monthindex: EagerVec::forced_import(
path,
file,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_first_dateindex: EagerVec::forced_import(
path,
file,
"first_dateindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_first_monthindex: EagerVec::forced_import(
path,
file,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_quarterindex: EagerVec::forced_import(
path,
file,
"quarterindex",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_semesterindex: EagerVec::forced_import(
path,
file,
"semesterindex",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_weekindex: EagerVec::forced_import(
path,
file,
"weekindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_decadeindex: EagerVec::forced_import(
path,
file,
"decadeindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_yearindex: EagerVec::forced_import(
path,
file,
"yearindex",
version + VERSION + Version::ZERO,
format,
)?,
height_to_date_fixed: EagerVec::forced_import(
path,
file,
"date_fixed",
version + VERSION + Version::ZERO,
format,
)?,
height_to_dateindex: EagerVec::forced_import(
path,
file,
"dateindex",
version + VERSION + Version::ZERO,
format,
)?,
txindex_to_height: EagerVec::forced_import(
path,
file,
"height",
version + VERSION + Version::ZERO,
format,
)?,
height_to_timestamp_fixed: EagerVec::forced_import(
path,
file,
"timestamp_fixed",
version + VERSION + Version::ZERO,
format,
)?,
height_to_txindex_count: EagerVec::forced_import(
path,
file,
"txindex_count",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_height_count: EagerVec::forced_import(
path,
file,
"height_count",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_dateindex_count: EagerVec::forced_import(
path,
file,
"dateindex_count",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_height_count: EagerVec::forced_import(
path,
file,
"height_count",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_dateindex_count: EagerVec::forced_import(
path,
file,
"dateindex_count",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_monthindex_count: EagerVec::forced_import(
path,
file,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_monthindex_count: EagerVec::forced_import(
path,
file,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_monthindex_count: EagerVec::forced_import(
path,
file,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_yearindex_count: EagerVec::forced_import(
path,
file,
"yearindex_count",
version + VERSION + Version::ZERO,
format,
)?,
outputindex_to_txindex: EagerVec::forced_import(
path,
file,
"txindex",
version + VERSION + Version::ZERO,
format,

15
crates/brk_computer/src/lib.rs
View File

@@ -3,13 +3,13 @@
#![doc = include_str!("../examples/main.rs")]
#![doc = "```"]
use std::path::Path;
use std::{path::Path, sync::Arc};
use brk_core::Version;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{Computation, Format};
use brk_vecs::{Computation, File, Format};
use log::info;
mod all;
@@ -47,15 +47,22 @@ impl Computer {
fetcher: Option<Fetcher>,
format: Format,
) -> color_eyre::Result<Self> {
let computed_path = outputs_dir.join("computed");
let states_path = computed_path.join("states");
let file = Arc::new(File::open(&computed_path.join("vecs"))?);
let file_fetched = Arc::new(File::open(&outputs_dir.join("fetched/vecs"))?);
Ok(Self {
vecs: all::Vecs::import(
// TODO: Give self.path, join inside import
&outputs_dir.join("vecs/computed"),
&file,
VERSION + Version::ZERO,
indexer,
fetcher.is_some(),
computation,
format,
&file_fetched,
&states_path,
)?,
fetcher,
})

274
crates/brk_computer/src/market.rs
View File

File diff suppressed because it is too large Load Diff

12
crates/brk_computer/src/mining.rs
View File

@@ -1,9 +1,9 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
use brk_vecs::{AnyCollectableVec, Computation, File, Format, VecIterator};
use crate::grouped::Source;
@@ -24,7 +24,7 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
file: &Arc<File>,
version: Version,
computation: Computation,
format: Format,
@@ -32,7 +32,7 @@ impl Vecs {
) -> color_eyre::Result<Self> {
Ok(Self {
indexes_to_difficulty: ComputedVecsFromHeight::forced_import(
path,
file,
"difficulty",
Source::None,
version + VERSION + Version::ZERO,
@@ -42,7 +42,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
path,
file,
"difficultyepoch",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -52,7 +52,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
path,
file,
"halvingepoch",
Source::Compute,
version + VERSION + Version::ZERO,

14
crates/brk_computer/src/stateful/address_cohort.rs
View File

@@ -1,10 +1,10 @@
use std::{ops::Deref, path::Path};
use std::{ops::Deref, path::Path, sync::Arc};
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, StoredUsize, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, VecIterator,
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, File, Format, VecIterator,
};
use crate::{
@@ -35,7 +35,7 @@ pub struct Vecs {
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
file: &Arc<File>,
cohort_name: Option<&str>,
computation: Computation,
format: Format,
@@ -57,13 +57,13 @@ impl Vecs {
compute_dollars,
)?,
height_to_address_count: EagerVec::forced_import(
path,
file,
&suffix("address_count"),
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_address_count: ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("address_count"),
Source::None,
version + VERSION + Version::ZERO,
@@ -73,7 +73,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
inner: common::Vecs::forced_import(
path,
file,
cohort_name,
computation,
format,

88
crates/brk_computer/src/stateful/address_cohorts.rs
View File

@@ -1,4 +1,4 @@
use std::path::Path;
use std::{path::Path, sync::Arc};
use brk_core::{
AddressGroups, Bitcoin, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, DateIndex,
@@ -6,7 +6,7 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyIterableVec, Computation, Format};
use brk_vecs::{AnyIterableVec, Computation, File, Format};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
@@ -25,7 +25,7 @@ pub struct Vecs(AddressGroups<(GroupFilter, address_cohort::Vecs)>);
impl Vecs {
pub fn forced_import(
path: &Path,
file: &Arc<File>,
version: Version,
_computation: Computation,
format: Format,
@@ -37,7 +37,7 @@ impl Vecs {
AddressGroups {
amount_range: ByAmountRange {
_0sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_with_0sats"),
_computation,
format,
@@ -48,7 +48,7 @@ impl Vecs {
true,
)?,
_1sat_to_10sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_1sat_under_10sats"),
_computation,
format,
@@ -59,7 +59,7 @@ impl Vecs {
true,
)?,
_10sats_to_100sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_10sats_under_100sats"),
_computation,
format,
@@ -70,7 +70,7 @@ impl Vecs {
true,
)?,
_100sats_to_1k_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_100sats_under_1k_sats"),
_computation,
format,
@@ -81,7 +81,7 @@ impl Vecs {
true,
)?,
_1k_sats_to_10k_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_1k_sats_under_10k_sats"),
_computation,
format,
@@ -92,7 +92,7 @@ impl Vecs {
true,
)?,
_10k_sats_to_100k_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_10k_sats_under_100k_sats"),
_computation,
format,
@@ -103,7 +103,7 @@ impl Vecs {
true,
)?,
_100k_sats_to_1m_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_100k_sats_under_1m_sats"),
_computation,
format,
@@ -114,7 +114,7 @@ impl Vecs {
true,
)?,
_1m_sats_to_10m_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_1m_sats_under_10m_sats"),
_computation,
format,
@@ -125,7 +125,7 @@ impl Vecs {
true,
)?,
_10m_sats_to_1btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_10m_sats_under_1btc"),
_computation,
format,
@@ -136,7 +136,7 @@ impl Vecs {
true,
)?,
_1btc_to_10btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_1btc_under_10btc"),
_computation,
format,
@@ -147,7 +147,7 @@ impl Vecs {
true,
)?,
_10btc_to_100btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_10btc_under_100btc"),
_computation,
format,
@@ -158,7 +158,7 @@ impl Vecs {
true,
)?,
_100btc_to_1k_btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_100btc_under_1k_btc"),
_computation,
format,
@@ -169,7 +169,7 @@ impl Vecs {
true,
)?,
_1k_btc_to_10k_btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_1k_btc_under_10k_btc"),
_computation,
format,
@@ -180,7 +180,7 @@ impl Vecs {
true,
)?,
_10k_btc_to_100k_btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_10k_btc_under_100k_btc"),
_computation,
format,
@@ -191,7 +191,7 @@ impl Vecs {
true,
)?,
_100k_btc_or_more: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_100k_btc"),
_computation,
format,
@@ -204,7 +204,7 @@ impl Vecs {
},
lt_amount: ByLowerThanAmount {
_10sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_10sats"),
_computation,
format,
@@ -215,7 +215,7 @@ impl Vecs {
true,
)?,
_100sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_100sats"),
_computation,
format,
@@ -226,7 +226,7 @@ impl Vecs {
true,
)?,
_1k_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_1k_sats"),
_computation,
format,
@@ -237,7 +237,7 @@ impl Vecs {
true,
)?,
_10k_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_10k_sats"),
_computation,
format,
@@ -248,7 +248,7 @@ impl Vecs {
true,
)?,
_100k_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_100k_sats"),
_computation,
format,
@@ -259,7 +259,7 @@ impl Vecs {
true,
)?,
_1m_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_1m_sats"),
_computation,
format,
@@ -270,7 +270,7 @@ impl Vecs {
true,
)?,
_10m_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_10m_sats"),
_computation,
format,
@@ -281,7 +281,7 @@ impl Vecs {
true,
)?,
_1btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_1btc"),
_computation,
format,
@@ -292,7 +292,7 @@ impl Vecs {
true,
)?,
_10btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_10btc"),
_computation,
format,
@@ -303,7 +303,7 @@ impl Vecs {
true,
)?,
_100btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_100btc"),
_computation,
format,
@@ -314,7 +314,7 @@ impl Vecs {
true,
)?,
_1k_btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_1k_btc"),
_computation,
format,
@@ -325,7 +325,7 @@ impl Vecs {
true,
)?,
_10k_btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_10k_btc"),
_computation,
format,
@@ -336,7 +336,7 @@ impl Vecs {
true,
)?,
_100k_btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_under_100k_btc"),
_computation,
format,
@@ -349,7 +349,7 @@ impl Vecs {
},
ge_amount: ByGreatEqualAmount {
_1sat: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_1sat"),
_computation,
format,
@@ -360,7 +360,7 @@ impl Vecs {
true,
)?,
_10sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_10sats"),
_computation,
format,
@@ -371,7 +371,7 @@ impl Vecs {
true,
)?,
_100sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_100sats"),
_computation,
format,
@@ -382,7 +382,7 @@ impl Vecs {
true,
)?,
_1k_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_1k_sats"),
_computation,
format,
@@ -393,7 +393,7 @@ impl Vecs {
true,
)?,
_10k_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_10k_sats"),
_computation,
format,
@@ -404,7 +404,7 @@ impl Vecs {
true,
)?,
_100k_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_100k_sats"),
_computation,
format,
@@ -415,7 +415,7 @@ impl Vecs {
true,
)?,
_1m_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_1m_sats"),
_computation,
format,
@@ -426,7 +426,7 @@ impl Vecs {
true,
)?,
_10m_sats: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_10m_sats"),
_computation,
format,
@@ -437,7 +437,7 @@ impl Vecs {
true,
)?,
_1btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_1btc"),
_computation,
format,
@@ -448,7 +448,7 @@ impl Vecs {
true,
)?,
_10btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_10btc"),
_computation,
format,
@@ -459,7 +459,7 @@ impl Vecs {
true,
)?,
_100btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_100btc"),
_computation,
format,
@@ -470,7 +470,7 @@ impl Vecs {
true,
)?,
_1k_btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_1k_btc"),
_computation,
format,
@@ -481,7 +481,7 @@ impl Vecs {
true,
)?,
_10k_btc: address_cohort::Vecs::forced_import(
path,
file,
Some("addrs_above_10k_btc"),
_computation,
format,

2
crates/brk_computer/src/stateful/addresstype_to_addresscount.rs
View File

@@ -1,5 +1,5 @@
use brk_core::{ByAddressType, Height};
use brk_vec::VecIterator;
use brk_vecs::VecIterator;
use derive_deref::{Deref, DerefMut};
use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;

2
crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs
View File

@@ -1,6 +1,6 @@
use brk_core::{ByAddressType, Height, Result, StoredUsize};
use brk_exit::Exit;
use brk_vec::EagerVec;
use brk_vecs::EagerVec;
use derive_deref::{Deref, DerefMut};
use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;

2
crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs
View File

@@ -1,6 +1,6 @@
use brk_core::{ByAddressType, StoredUsize};
use brk_exit::Exit;
use brk_vec::AnyCollectableVec;
use brk_vecs::AnyCollectableVec;
use derive_deref::{Deref, DerefMut};
use crate::{

170
crates/brk_computer/src/stateful/common.rs
View File

@@ -1,13 +1,13 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{
Bitcoin, DateIndex, Dollars, Height, Result, Sats, StoredF32, StoredF64, StoredUsize, Version,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyVec, CloneableAnyIterableVec, Computation, EagerVec,
Format, VecIterator,
File, Format, VecIterator,
};
use crate::{
@@ -128,7 +128,7 @@ pub struct Vecs {
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
file: &Arc<File>,
cohort_name: Option<&str>,
computation: Computation,
format: Format,
@@ -146,7 +146,7 @@ impl Vecs {
let dateindex_to_supply_in_profit = compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("supply_in_profit"),
version + VERSION + Version::ZERO,
format,
@@ -156,7 +156,7 @@ impl Vecs {
let dateindex_to_supply_even = compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("supply_even"),
version + VERSION + Version::ZERO,
format,
@@ -166,7 +166,7 @@ impl Vecs {
let dateindex_to_supply_in_loss = compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("supply_in_loss"),
version + VERSION + Version::ZERO,
format,
@@ -176,7 +176,7 @@ impl Vecs {
let dateindex_to_unrealized_profit = compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("unrealized_profit"),
version + VERSION + Version::ZERO,
format,
@@ -186,7 +186,7 @@ impl Vecs {
let dateindex_to_unrealized_loss = compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("unrealized_loss"),
version + VERSION + Version::ZERO,
format,
@@ -197,7 +197,7 @@ impl Vecs {
Ok(Self {
height_to_supply_in_profit: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("supply_in_profit"),
version + VERSION + Version::ZERO,
format,
@@ -206,7 +206,7 @@ impl Vecs {
}),
indexes_to_supply_in_profit: compute_dollars.then(|| {
ComputedValueVecsFromDateIndex::forced_import(
path,
file,
&suffix("supply_in_profit"),
dateindex_to_supply_in_profit.as_ref().map(|v | v.boxed_clone()).into(),
version + VERSION + Version::ZERO,
@@ -221,7 +221,7 @@ impl Vecs {
dateindex_to_supply_in_profit,
height_to_supply_even: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("supply_even"),
version + VERSION + Version::ZERO,
format,
@@ -230,7 +230,7 @@ impl Vecs {
}),
indexes_to_supply_even: compute_dollars.then(|| {
ComputedValueVecsFromDateIndex::forced_import(
path,
file,
&suffix("supply_even"),
dateindex_to_supply_even.as_ref().map(|v | v.boxed_clone()).into(),
version + VERSION + Version::ZERO,
@@ -245,7 +245,7 @@ impl Vecs {
dateindex_to_supply_even,
height_to_supply_in_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("supply_in_loss"),
version + VERSION + Version::ZERO,
format,
@@ -254,7 +254,7 @@ impl Vecs {
}),
indexes_to_supply_in_loss: compute_dollars.then(|| {
ComputedValueVecsFromDateIndex::forced_import(
path,
file,
&suffix("supply_in_loss"),
dateindex_to_supply_in_loss.as_ref().map(|v | v.boxed_clone()).into(),
version + VERSION + Version::ZERO,
@@ -269,7 +269,7 @@ impl Vecs {
dateindex_to_supply_in_loss,
height_to_unrealized_profit: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("unrealized_profit"),
version + VERSION + Version::ZERO,
format,
@@ -278,7 +278,7 @@ impl Vecs {
}),
indexes_to_unrealized_profit: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("unrealized_profit"),
dateindex_to_unrealized_profit.as_ref().map(|v | v.boxed_clone()).into(),
version + VERSION + Version::ZERO,
@@ -292,7 +292,7 @@ impl Vecs {
dateindex_to_unrealized_profit,
height_to_unrealized_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("unrealized_loss"),
version + VERSION + Version::ZERO,
format,
@@ -301,7 +301,7 @@ impl Vecs {
}),
height_to_min_price_paid: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("min_price_paid"),
version + VERSION + Version::ZERO,
format,
@@ -310,7 +310,7 @@ impl Vecs {
}),
height_to_max_price_paid: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("max_price_paid"),
version + VERSION + Version::ZERO,
format,
@@ -319,7 +319,7 @@ impl Vecs {
}),
indexes_to_unrealized_loss: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("unrealized_loss"),
dateindex_to_unrealized_loss.as_ref().map(|v | v.boxed_clone()).into(),
version + VERSION + Version::ZERO,
@@ -333,7 +333,7 @@ impl Vecs {
dateindex_to_unrealized_loss,
height_to_realized_cap: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("realized_cap"),
version + VERSION + Version::ZERO,
format,
@@ -342,7 +342,7 @@ impl Vecs {
}),
indexes_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("realized_cap"),
Source::None,
version + VERSION + Version::ZERO,
@@ -355,7 +355,7 @@ impl Vecs {
}),
indexes_to_min_price_paid: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("min_price_paid"),
Source::None,
version + VERSION + Version::ZERO,
@@ -368,7 +368,7 @@ impl Vecs {
}),
indexes_to_max_price_paid: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("max_price_paid"),
Source::None,
version + VERSION + Version::ZERO,
@@ -380,13 +380,13 @@ impl Vecs {
.unwrap()
}),
height_to_supply: EagerVec::forced_import(
path,
file,
&suffix("supply"),
version + VERSION + Version::ZERO,
format,
)?,
height_to_supply_value: ComputedHeightValueVecs::forced_import(
path,
file,
&suffix("supply"),
Source::None,
version + VERSION + Version::ZERO,
@@ -394,7 +394,7 @@ impl Vecs {
compute_dollars,
)?,
indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import(
path,
file,
&suffix("supply"),
Source::Compute,
version + VERSION + Version::ONE,
@@ -405,13 +405,13 @@ impl Vecs {
indexes,
)?,
height_to_utxo_count: EagerVec::forced_import(
path,
file,
&suffix("utxo_count"),
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("utxo_count"),
Source::None,
version + VERSION + Version::ZERO,
@@ -422,7 +422,7 @@ impl Vecs {
)?,
indexes_to_realized_price: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("realized_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -435,7 +435,7 @@ impl Vecs {
}),
indexes_to_realized_price_extra: compute_dollars.then(|| {
ComputedRatioVecsFromDateIndex::forced_import(
path,
file,
&suffix("realized_price"),
Source::None,
version + VERSION + Version::ZERO,
@@ -448,7 +448,7 @@ impl Vecs {
}),
height_to_realized_profit: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("realized_profit"),
version + VERSION + Version::ZERO,
format,
@@ -457,7 +457,7 @@ impl Vecs {
}),
indexes_to_realized_profit: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("realized_profit"),
Source::None,
version + VERSION + Version::ZERO,
@@ -472,7 +472,7 @@ impl Vecs {
}),
height_to_realized_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("realized_loss"),
version + VERSION + Version::ZERO,
format,
@@ -481,7 +481,7 @@ impl Vecs {
}),
indexes_to_realized_loss: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("realized_loss"),
Source::None,
version + VERSION + Version::ZERO,
@@ -496,7 +496,7 @@ impl Vecs {
}),
indexes_to_negative_realized_loss: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("negative_realized_loss"),
Source::Compute,
version + VERSION + Version::ONE,
@@ -509,7 +509,7 @@ impl Vecs {
}),
height_to_value_created: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("value_created"),
version + VERSION + Version::ZERO,
format,
@@ -518,7 +518,7 @@ impl Vecs {
}),
indexes_to_value_created: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("value_created"),
Source::None,
version + VERSION + Version::ZERO,
@@ -531,7 +531,7 @@ impl Vecs {
}),
indexes_to_realized_value: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("realized_value"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -544,7 +544,7 @@ impl Vecs {
}),
height_to_adjusted_value_created: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("adjusted_value_created"),
version + VERSION + Version::ZERO,
format,
@@ -553,7 +553,7 @@ impl Vecs {
}),
indexes_to_adjusted_value_created: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("adjusted_value_created"),
Source::None,
version + VERSION + Version::ZERO,
@@ -566,7 +566,7 @@ impl Vecs {
}),
height_to_value_destroyed: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("value_destroyed"),
version + VERSION + Version::ZERO,
format,
@@ -575,7 +575,7 @@ impl Vecs {
}),
indexes_to_value_destroyed: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("value_destroyed"),
Source::None,
version + VERSION + Version::ZERO,
@@ -588,7 +588,7 @@ impl Vecs {
}),
height_to_adjusted_value_destroyed: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("adjusted_value_destroyed"),
version + VERSION + Version::ZERO,
format,
@@ -597,7 +597,7 @@ impl Vecs {
}),
indexes_to_adjusted_value_destroyed: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("adjusted_value_destroyed"),
Source::None,
version + VERSION + Version::ZERO,
@@ -610,7 +610,7 @@ impl Vecs {
}),
indexes_to_realized_cap_30d_change: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("realized_cap_30d_change"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -623,7 +623,7 @@ impl Vecs {
}),
indexes_to_net_realized_profit_and_loss: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("net_realized_profit_and_loss"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -638,7 +638,7 @@ impl Vecs {
}),
dateindex_to_sell_side_risk_ratio: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("sell_side_risk_ratio"),
version + VERSION + Version::ONE,
format,
@@ -647,7 +647,7 @@ impl Vecs {
}),
dateindex_to_spent_output_profit_ratio: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("spent_output_profit_ratio"),
version + VERSION + Version::ZERO,
format,
@@ -656,7 +656,7 @@ impl Vecs {
}),
dateindex_to_adjusted_spent_output_profit_ratio: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("adjusted_spent_output_profit_ratio"),
version + VERSION + Version::ZERO,
format,
@@ -664,7 +664,7 @@ impl Vecs {
.unwrap()
}),
height_to_halved_supply_value: ComputedHeightValueVecs::forced_import(
path,
file,
&suffix("halved_supply"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -672,7 +672,7 @@ impl Vecs {
compute_dollars,
)?,
indexes_to_halved_supply: ComputedValueVecsFromDateIndex::forced_import(
path,
file,
&suffix("halved_supply"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -684,7 +684,7 @@ impl Vecs {
)?,
height_to_negative_unrealized_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("negative_unrealized_loss"),
version + VERSION + Version::ZERO,
format,
@@ -693,7 +693,7 @@ impl Vecs {
}),
indexes_to_negative_unrealized_loss: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("negative_unrealized_loss"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -706,7 +706,7 @@ impl Vecs {
}),
height_to_net_unrealized_profit_and_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("net_unrealized_profit_and_loss"),
version + VERSION + Version::ZERO,
format,
@@ -715,7 +715,7 @@ impl Vecs {
}),
indexes_to_net_unrealized_profit_and_loss: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("net_unrealized_profit_and_loss"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -729,7 +729,7 @@ impl Vecs {
height_to_net_unrealized_profit_and_loss_relative_to_market_cap: compute_dollars.then(
|| {
EagerVec::forced_import(
path,
file,
&suffix("net_unrealized_profit_and_loss_relative_to_market_cap"),
version + VERSION + Version::ONE,
format,
@@ -740,7 +740,7 @@ impl Vecs {
indexes_to_net_unrealized_profit_and_loss_relative_to_market_cap: compute_dollars.then(
|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("net_unrealized_profit_and_loss_relative_to_market_cap"),
Source::Compute,
version + VERSION + Version::ONE,
@@ -754,7 +754,7 @@ impl Vecs {
),
indexes_to_realized_profit_relative_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("realized_profit_relative_to_realized_cap"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -767,7 +767,7 @@ impl Vecs {
}),
indexes_to_realized_loss_relative_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("realized_loss_relative_to_realized_cap"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -781,7 +781,7 @@ impl Vecs {
indexes_to_net_realized_profit_and_loss_relative_to_realized_cap: compute_dollars.then(
|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("net_realized_profit_and_loss_relative_to_realized_cap"),
Source::Compute,
version + VERSION + Version::ONE,
@@ -795,7 +795,7 @@ impl Vecs {
),
height_to_supply_even_value: compute_dollars.then(|| {
ComputedHeightValueVecs::forced_import(
path,
file,
&suffix("supply_even"),
Source::None,
version + VERSION + Version::ZERO,
@@ -806,7 +806,7 @@ impl Vecs {
}),
height_to_supply_in_loss_value: compute_dollars.then(|| {
ComputedHeightValueVecs::forced_import(
path,
file,
&suffix("supply_in_loss"),
Source::None,
version + VERSION + Version::ZERO,
@@ -817,7 +817,7 @@ impl Vecs {
}),
height_to_supply_in_profit_value: compute_dollars.then(|| {
ComputedHeightValueVecs::forced_import(
path,
file,
&suffix("supply_in_profit"),
Source::None,
version + VERSION + Version::ZERO,
@@ -828,7 +828,7 @@ impl Vecs {
}),
height_to_supply_even_relative_to_own_supply: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("supply_even_relative_to_own_supply"),
version + VERSION + Version::ONE,
format,
@@ -837,7 +837,7 @@ impl Vecs {
}),
height_to_supply_in_loss_relative_to_own_supply: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("supply_in_loss_relative_to_own_supply"),
version + VERSION + Version::ONE,
format,
@@ -846,7 +846,7 @@ impl Vecs {
}),
height_to_supply_in_profit_relative_to_own_supply: compute_dollars.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("supply_in_profit_relative_to_own_supply"),
version + VERSION + Version::ONE,
format,
@@ -855,7 +855,7 @@ impl Vecs {
}),
indexes_to_supply_even_relative_to_own_supply: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("supply_even_relative_to_own_supply"),
Source::Compute,
version + VERSION + Version::ONE,
@@ -868,7 +868,7 @@ impl Vecs {
}),
indexes_to_supply_in_loss_relative_to_own_supply: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("supply_in_loss_relative_to_own_supply"),
Source::Compute,
version + VERSION + Version::ONE,
@@ -881,7 +881,7 @@ impl Vecs {
}),
indexes_to_supply_in_profit_relative_to_own_supply: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("supply_in_profit_relative_to_own_supply"),
Source::Compute,
version + VERSION + Version::ONE,
@@ -894,7 +894,7 @@ impl Vecs {
}),
indexes_to_supply_relative_to_circulating_supply: compute_relative_to_all.then(|| {
ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("supply_relative_to_circulating_supply"),
Source::Compute,
version + VERSION + Version::ONE,
@@ -909,7 +909,7 @@ impl Vecs {
&& compute_dollars)
.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("supply_even_relative_to_circulating_supply"),
version + VERSION + Version::ONE,
format,
@@ -920,7 +920,7 @@ impl Vecs {
&& compute_dollars)
.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("supply_in_loss_relative_to_circulating_supply"),
version + VERSION + Version::ONE,
format,
@@ -931,7 +931,7 @@ impl Vecs {
&& compute_dollars)
.then(|| {
EagerVec::forced_import(
path,
file,
&suffix("supply_in_profit_relative_to_circulating_supply"),
version + VERSION + Version::ONE,
format,
@@ -942,7 +942,7 @@ impl Vecs {
&& compute_dollars)
.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("supply_even_relative_to_circulating_supply"),
Source::Compute,
version + VERSION + Version::ONE,
@@ -957,7 +957,7 @@ impl Vecs {
&& compute_dollars)
.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("supply_in_loss_relative_to_circulating_supply"),
Source::Compute,
version + VERSION + Version::ONE,
@@ -972,7 +972,7 @@ impl Vecs {
&& compute_dollars)
.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("supply_in_profit_relative_to_circulating_supply"),
Source::Compute,
version + VERSION + Version::ONE,
@@ -984,19 +984,19 @@ impl Vecs {
.unwrap()
}),
height_to_satblocks_destroyed: EagerVec::forced_import(
path,
file,
&suffix("satblocks_destroyed"),
version + VERSION + Version::ZERO,
format,
)?,
height_to_satdays_destroyed: EagerVec::forced_import(
path,
file,
&suffix("satdays_destroyed"),
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_coinblocks_destroyed: ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("coinblocks_destroyed"),
Source::Compute,
version + VERSION + Version::TWO,
@@ -1006,7 +1006,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_coindays_destroyed: ComputedVecsFromHeight::forced_import(
path,
file,
&suffix("coindays_destroyed"),
Source::Compute,
version + VERSION + Version::TWO,
@@ -1017,7 +1017,7 @@ impl Vecs {
)?,
indexes_to_net_realized_profit_and_loss_cumulative_30d_change: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("net_realized_profit_and_loss_cumulative_30d_change"),
Source::Compute,
version + VERSION + Version::new(3),
@@ -1030,7 +1030,7 @@ impl Vecs {
}),
indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap"),
Source::Compute,
version + VERSION + Version::new(3),
@@ -1043,7 +1043,7 @@ impl Vecs {
}),
indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
file,
&suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap"),
Source::Compute,
version + VERSION + Version::new(3),

425
crates/brk_computer/src/stateful/mod.rs
View File

@@ -1,4 +1,4 @@
use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread};
use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, sync::Arc, thread};
use brk_core::{
AnyAddressDataIndexEnum, AnyAddressIndex, ByAddressType, ByAnyAddress, CheckedSub, DateIndex,
@@ -9,9 +9,9 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyIndexedVec, AnyVec, CollectableVec, Computation, EagerVec, Format,
GenericStoredVec, IndexedVec, Mmap, StoredIndex, StoredVec, VecIterator,
use brk_vecs::{
AnyCollectableVec, AnyStampedVec, AnyVec, CollectableVec, Computation, EagerVec, File, Format,
GenericStoredVec, Reader, Stamp, StampedVec, StoredIndex, StoredVec, VecIterator,
};
use log::info;
use rayon::prelude::*;
@@ -71,16 +71,16 @@ pub struct Vecs {
pub utxo_cohorts: utxo_cohorts::Vecs,
pub address_cohorts: address_cohorts::Vecs,
pub p2pk33addressindex_to_anyaddressindex: IndexedVec<P2PK33AddressIndex, AnyAddressIndex>,
pub p2pk65addressindex_to_anyaddressindex: IndexedVec<P2PK65AddressIndex, AnyAddressIndex>,
pub p2pkhaddressindex_to_anyaddressindex: IndexedVec<P2PKHAddressIndex, AnyAddressIndex>,
pub p2shaddressindex_to_anyaddressindex: IndexedVec<P2SHAddressIndex, AnyAddressIndex>,
pub p2traddressindex_to_anyaddressindex: IndexedVec<P2TRAddressIndex, AnyAddressIndex>,
pub p2wpkhaddressindex_to_anyaddressindex: IndexedVec<P2WPKHAddressIndex, AnyAddressIndex>,
pub p2wshaddressindex_to_anyaddressindex: IndexedVec<P2WSHAddressIndex, AnyAddressIndex>,
pub p2aaddressindex_to_anyaddressindex: IndexedVec<P2AAddressIndex, AnyAddressIndex>,
pub loadedaddressindex_to_loadedaddressdata: IndexedVec<LoadedAddressIndex, LoadedAddressData>,
pub emptyaddressindex_to_emptyaddressdata: IndexedVec<EmptyAddressIndex, EmptyAddressData>,
pub p2pk33addressindex_to_anyaddressindex: StampedVec<P2PK33AddressIndex, AnyAddressIndex>,
pub p2pk65addressindex_to_anyaddressindex: StampedVec<P2PK65AddressIndex, AnyAddressIndex>,
pub p2pkhaddressindex_to_anyaddressindex: StampedVec<P2PKHAddressIndex, AnyAddressIndex>,
pub p2shaddressindex_to_anyaddressindex: StampedVec<P2SHAddressIndex, AnyAddressIndex>,
pub p2traddressindex_to_anyaddressindex: StampedVec<P2TRAddressIndex, AnyAddressIndex>,
pub p2wpkhaddressindex_to_anyaddressindex: StampedVec<P2WPKHAddressIndex, AnyAddressIndex>,
pub p2wshaddressindex_to_anyaddressindex: StampedVec<P2WSHAddressIndex, AnyAddressIndex>,
pub p2aaddressindex_to_anyaddressindex: StampedVec<P2AAddressIndex, AnyAddressIndex>,
pub loadedaddressindex_to_loadedaddressdata: StampedVec<LoadedAddressIndex, LoadedAddressData>,
pub emptyaddressindex_to_emptyaddressdata: StampedVec<EmptyAddressIndex, EmptyAddressData>,
pub indexes_to_address_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_empty_address_count: ComputedVecsFromHeight<StoredUsize>,
@@ -88,36 +88,34 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
file: &Arc<File>,
version: Version,
computation: Computation,
format: Format,
indexes: &indexes::Vecs,
fetched: Option<&fetched::Vecs>,
states_path: &Path,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
let mut root_path = path.to_owned();
root_path.pop();
root_path.pop();
let states_path = root_path.join("states");
let chain_file = Arc::new(File::open(&states_path.join("chain"))?);
Ok(Self {
chain_state: StoredVec::forced_import(
&states_path,
&chain_file,
"chain",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
height_to_unspendable_supply: EagerVec::forced_import(
path,
file,
"unspendable_supply",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
path,
file,
"unspendable_supply",
Source::None,
version + VERSION + Version::ZERO,
@@ -128,13 +126,13 @@ impl Vecs {
indexes,
)?,
height_to_opreturn_supply: EagerVec::forced_import(
path,
file,
"opreturn_supply",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import(
path,
file,
"opreturn_supply",
Source::None,
version + VERSION + Version::ZERO,
@@ -145,7 +143,7 @@ impl Vecs {
indexes,
)?,
indexes_to_address_count: ComputedVecsFromHeight::forced_import(
path,
file,
"address_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -155,7 +153,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_empty_address_count: ComputedVecsFromHeight::forced_import(
path,
file,
"empty_address_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -167,49 +165,49 @@ impl Vecs {
addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount::from(
ByAddressType {
p2pk65: EagerVec::forced_import(
path,
file,
"p2pk65_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
path,
file,
"p2pk33_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
path,
file,
"p2pkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
path,
file,
"p2sh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
path,
file,
"p2wpkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
path,
file,
"p2wsh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
path,
file,
"p2tr_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
path,
file,
"p2a_address_count",
version + VERSION + Version::ZERO,
format,
@@ -219,49 +217,49 @@ impl Vecs {
addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount::from(
ByAddressType {
p2pk65: EagerVec::forced_import(
path,
file,
"p2pk65_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
path,
file,
"p2pk33_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
path,
file,
"p2pkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
path,
file,
"p2sh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
path,
file,
"p2wpkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
path,
file,
"p2wsh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
path,
file,
"p2tr_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
path,
file,
"p2a_empty_address_count",
version + VERSION + Version::ZERO,
format,
@@ -271,7 +269,7 @@ impl Vecs {
addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount::from(
ByAddressType {
p2pk65: ComputedVecsFromHeight::forced_import(
path,
file,
"p2pk65_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -281,7 +279,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pk33: ComputedVecsFromHeight::forced_import(
path,
file,
"p2pk33_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -291,7 +289,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pkh: ComputedVecsFromHeight::forced_import(
path,
file,
"p2pkh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -301,7 +299,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2sh: ComputedVecsFromHeight::forced_import(
path,
file,
"p2sh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -311,7 +309,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wpkh: ComputedVecsFromHeight::forced_import(
path,
file,
"p2wpkh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -321,7 +319,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wsh: ComputedVecsFromHeight::forced_import(
path,
file,
"p2wsh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -331,7 +329,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2tr: ComputedVecsFromHeight::forced_import(
path,
file,
"p2tr_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -341,7 +339,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2a: ComputedVecsFromHeight::forced_import(
path,
file,
"p2a_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -355,7 +353,7 @@ impl Vecs {
addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount::from(
ByAddressType {
p2pk65: ComputedVecsFromHeight::forced_import(
path,
file,
"p2pk65_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -365,7 +363,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pk33: ComputedVecsFromHeight::forced_import(
path,
file,
"p2pk33_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -375,7 +373,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pkh: ComputedVecsFromHeight::forced_import(
path,
file,
"p2pkh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -385,7 +383,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2sh: ComputedVecsFromHeight::forced_import(
path,
file,
"p2sh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -395,7 +393,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wpkh: ComputedVecsFromHeight::forced_import(
path,
file,
"p2wpkh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -405,7 +403,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wsh: ComputedVecsFromHeight::forced_import(
path,
file,
"p2wsh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -415,7 +413,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2tr: ComputedVecsFromHeight::forced_import(
path,
file,
"p2tr_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -425,7 +423,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2a: ComputedVecsFromHeight::forced_import(
path,
file,
"p2a_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -437,81 +435,81 @@ impl Vecs {
},
),
utxo_cohorts: utxo_cohorts::Vecs::forced_import(
path,
file,
version,
computation,
format,
indexes,
fetched,
&states_path,
states_path,
)?,
address_cohorts: address_cohorts::Vecs::forced_import(
path,
file,
version,
computation,
format,
indexes,
fetched,
&states_path,
states_path,
)?,
p2aaddressindex_to_anyaddressindex: IndexedVec::forced_import(
path,
p2aaddressindex_to_anyaddressindex: StampedVec::forced_import(
file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pk33addressindex_to_anyaddressindex: IndexedVec::forced_import(
path,
p2pk33addressindex_to_anyaddressindex: StampedVec::forced_import(
file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pk65addressindex_to_anyaddressindex: IndexedVec::forced_import(
path,
p2pk65addressindex_to_anyaddressindex: StampedVec::forced_import(
file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2pkhaddressindex_to_anyaddressindex: IndexedVec::forced_import(
path,
p2pkhaddressindex_to_anyaddressindex: StampedVec::forced_import(
file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2shaddressindex_to_anyaddressindex: IndexedVec::forced_import(
path,
p2shaddressindex_to_anyaddressindex: StampedVec::forced_import(
file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2traddressindex_to_anyaddressindex: IndexedVec::forced_import(
path,
p2traddressindex_to_anyaddressindex: StampedVec::forced_import(
file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2wpkhaddressindex_to_anyaddressindex: IndexedVec::forced_import(
path,
p2wpkhaddressindex_to_anyaddressindex: StampedVec::forced_import(
file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
p2wshaddressindex_to_anyaddressindex: IndexedVec::forced_import(
path,
p2wshaddressindex_to_anyaddressindex: StampedVec::forced_import(
file,
"anyaddressindex",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
loadedaddressindex_to_loadedaddressdata: IndexedVec::forced_import(
path,
loadedaddressindex_to_loadedaddressdata: StampedVec::forced_import(
file,
"loadedaddressdata",
version + VERSION + Version::ZERO,
Format::Raw,
)?,
emptyaddressindex_to_emptyaddressdata: IndexedVec::forced_import(
path,
emptyaddressindex_to_emptyaddressdata: StampedVec::forced_import(
file,
"emptyaddressdata",
version + VERSION + Version::ZERO,
Format::Raw,
@@ -637,54 +635,54 @@ impl Vecs {
)
.min(chain_state_starting_height)
.min(
self.p2pk33addressindex_to_anyaddressindex
.height()
Height::from(u64::from(
self.p2pk33addressindex_to_anyaddressindex.stamp(),
))
.incremented(),
)
.min(
Height::from(u64::from(
self.p2pk65addressindex_to_anyaddressindex.stamp(),
))
.incremented(),
)
.min(
Height::from(u64::from(self.p2pkhaddressindex_to_anyaddressindex.stamp()))
.incremented(),
)
.min(
self.p2pk65addressindex_to_anyaddressindex
.height()
Height::from(u64::from(self.p2shaddressindex_to_anyaddressindex.stamp()))
.incremented(),
)
.min(
self.p2pkhaddressindex_to_anyaddressindex
.height()
Height::from(u64::from(self.p2traddressindex_to_anyaddressindex.stamp()))
.incremented(),
)
.min(
self.p2shaddressindex_to_anyaddressindex
.height()
Height::from(u64::from(
self.p2wpkhaddressindex_to_anyaddressindex.stamp(),
))
.incremented(),
)
.min(
Height::from(u64::from(self.p2wshaddressindex_to_anyaddressindex.stamp()))
.incremented(),
)
.min(
self.p2traddressindex_to_anyaddressindex
.height()
Height::from(u64::from(self.p2aaddressindex_to_anyaddressindex.stamp()))
.incremented(),
)
.min(
self.p2wpkhaddressindex_to_anyaddressindex
.height()
.incremented(),
Height::from(u64::from(
self.loadedaddressindex_to_loadedaddressdata.stamp(),
))
.incremented(),
)
.min(
self.p2wshaddressindex_to_anyaddressindex
.height()
.incremented(),
)
.min(
self.p2aaddressindex_to_anyaddressindex
.height()
.incremented(),
)
.min(
self.loadedaddressindex_to_loadedaddressdata
.height()
.incremented(),
)
.min(
self.emptyaddressindex_to_emptyaddressdata
.height()
.incremented(),
Height::from(u64::from(
self.emptyaddressindex_to_emptyaddressdata.stamp(),
))
.incremented(),
)
.min(Height::from(self.height_to_unspendable_supply.len()))
.min(Height::from(self.height_to_opreturn_supply.len()))
@@ -749,13 +747,13 @@ impl Vecs {
.try_for_each(|(_, v)| v.state.reset_price_to_amount())?;
};
let last_height = indexer.vecs.height_to_blockhash.height();
let last_height = Height::from(u64::from(indexer.vecs.height_to_blockhash.stamp()));
if starting_height <= last_height {
let inputindex_to_outputindex_mmap = inputindex_to_outputindex.create_mmap()?;
let outputindex_to_value_mmap = outputindex_to_value.create_mmap()?;
let outputindex_to_outputtype_mmap = outputindex_to_outputtype.create_mmap()?;
let outputindex_to_typeindex_mmap = outputindex_to_typeindex.create_mmap()?;
let inputindex_to_outputindex_reader = inputindex_to_outputindex.create_reader();
let outputindex_to_value_reader = outputindex_to_value.create_reader();
let outputindex_to_outputtype_reader = outputindex_to_outputtype.create_reader();
let outputindex_to_typeindex_reader = outputindex_to_typeindex.create_reader();
let mut height_to_first_outputindex_iter = height_to_first_outputindex.into_iter();
let mut height_to_first_inputindex_iter = height_to_first_inputindex.into_iter();
@@ -828,14 +826,14 @@ impl Vecs {
AddressTypeToTypeIndexTree::<WithAddressDataSource<LoadedAddressData>>::default();
let mut addresstype_to_typeindex_to_emptyaddressdata =
AddressTypeToTypeIndexTree::<WithAddressDataSource<EmptyAddressData>>::default();
let mut addresstypeindex_to_anyaddressindex_mmap_opt =
ByAddressType::<Option<Mmap>>::default();
let mut anyaddressindex_to_anyaddressdata_mmap_opt =
ByAnyAddress::<Option<Mmap>>::default();
let mut addresstypeindex_to_anyaddressindex_reader_opt =
ByAddressType::<Option<Reader>>::default();
let mut anyaddressindex_to_anyaddressdata_reader_opt =
ByAnyAddress::<Option<Reader>>::default();
self.reset_mmaps_options(
&mut addresstypeindex_to_anyaddressindex_mmap_opt,
&mut anyaddressindex_to_anyaddressdata_mmap_opt,
self.reset_readers_options(
&mut addresstypeindex_to_anyaddressindex_reader_opt,
&mut anyaddressindex_to_anyaddressdata_reader_opt,
);
(height.unwrap_to_usize()..height_to_date_fixed.len())
@@ -912,17 +910,17 @@ impl Vecs {
.map(OutputIndex::from)
.map(|outputindex| {
let value = outputindex_to_value
.unwrap_read(outputindex, &outputindex_to_value_mmap);
.unwrap_read(outputindex, &outputindex_to_value_reader);
let output_type = outputindex_to_outputtype
.unwrap_read(outputindex, &outputindex_to_outputtype_mmap);
.unwrap_read(outputindex, &outputindex_to_outputtype_reader);
if output_type.is_not_address() {
return (value, output_type, None);
}
let typeindex = outputindex_to_typeindex
.unwrap_read(outputindex, &outputindex_to_typeindex_mmap);
.unwrap_read(outputindex, &outputindex_to_typeindex_reader);
let addressdata_opt = Self::get_addressdatawithsource(
output_type,
@@ -930,8 +928,8 @@ impl Vecs {
&first_addressindexes,
&addresstype_to_typeindex_to_loadedaddressdata,
&addresstype_to_typeindex_to_emptyaddressdata,
&addresstypeindex_to_anyaddressindex_mmap_opt,
&anyaddressindex_to_anyaddressdata_mmap_opt,
&addresstypeindex_to_anyaddressindex_reader_opt,
&anyaddressindex_to_anyaddressdata_reader_opt,
&self.p2pk33addressindex_to_anyaddressindex,
&self.p2pk65addressindex_to_anyaddressindex,
&self.p2pkhaddressindex_to_anyaddressindex,
@@ -996,13 +994,13 @@ impl Vecs {
.map(InputIndex::from)
.map(|inputindex| {
let outputindex =
inputindex_to_outputindex.unwrap_read(inputindex, &inputindex_to_outputindex_mmap);
inputindex_to_outputindex.unwrap_read(inputindex, &inputindex_to_outputindex_reader);
let value = outputindex_to_value
.unwrap_read(outputindex, &outputindex_to_value_mmap);
.unwrap_read(outputindex, &outputindex_to_value_reader);
let input_type = outputindex_to_outputtype
.unwrap_read(outputindex, &outputindex_to_outputtype_mmap);
.unwrap_read(outputindex, &outputindex_to_outputtype_reader);
let prev_height =
*outputindex_range_to_height.get(outputindex).unwrap();
@@ -1012,7 +1010,7 @@ impl Vecs {
}
let typeindex = outputindex_to_typeindex
.unwrap_read(outputindex, &outputindex_to_typeindex_mmap);
.unwrap_read(outputindex, &outputindex_to_typeindex_reader);
let addressdata_opt = Self::get_addressdatawithsource(
input_type,
@@ -1020,8 +1018,8 @@ impl Vecs {
&first_addressindexes,
&addresstype_to_typeindex_to_loadedaddressdata,
&addresstype_to_typeindex_to_emptyaddressdata,
&addresstypeindex_to_anyaddressindex_mmap_opt,
&anyaddressindex_to_anyaddressdata_mmap_opt,
&addresstypeindex_to_anyaddressindex_reader_opt,
&anyaddressindex_to_anyaddressdata_reader_opt,
&self.p2pk33addressindex_to_anyaddressindex,
&self.p2pk65addressindex_to_anyaddressindex,
&self.p2pkhaddressindex_to_anyaddressindex,
@@ -1240,11 +1238,14 @@ impl Vecs {
let _lock = exit.lock();
addresstypeindex_to_anyaddressindex_reader_opt.take();
anyaddressindex_to_anyaddressdata_reader_opt.take();
self.flush_states(height, &chain_state, mem::take(&mut addresstype_to_typeindex_to_loadedaddressdata), mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata), exit)?;
self.reset_mmaps_options(
&mut addresstypeindex_to_anyaddressindex_mmap_opt,
&mut anyaddressindex_to_anyaddressdata_mmap_opt,
self.reset_readers_options(
&mut addresstypeindex_to_anyaddressindex_reader_opt,
&mut anyaddressindex_to_anyaddressdata_reader_opt,
);
}
@@ -1431,18 +1432,18 @@ impl Vecs {
addresstype_to_typeindex_to_emptyaddressdata: &AddressTypeToTypeIndexTree<
WithAddressDataSource<EmptyAddressData>,
>,
addresstypeindex_to_anyaddressindex_mmap_opt: &ByAddressType<Option<Mmap>>,
anyaddressindex_to_anyaddressdata_mmap_opt: &ByAnyAddress<Option<Mmap>>,
p2pk33addressindex_to_anyaddressindex: &IndexedVec<P2PK33AddressIndex, AnyAddressIndex>,
p2pk65addressindex_to_anyaddressindex: &IndexedVec<P2PK65AddressIndex, AnyAddressIndex>,
p2pkhaddressindex_to_anyaddressindex: &IndexedVec<P2PKHAddressIndex, AnyAddressIndex>,
p2shaddressindex_to_anyaddressindex: &IndexedVec<P2SHAddressIndex, AnyAddressIndex>,
p2traddressindex_to_anyaddressindex: &IndexedVec<P2TRAddressIndex, AnyAddressIndex>,
p2wpkhaddressindex_to_anyaddressindex: &IndexedVec<P2WPKHAddressIndex, AnyAddressIndex>,
p2wshaddressindex_to_anyaddressindex: &IndexedVec<P2WSHAddressIndex, AnyAddressIndex>,
p2aaddressindex_to_anyaddressindex: &IndexedVec<P2AAddressIndex, AnyAddressIndex>,
loadedaddressindex_to_loadedaddressdata: &IndexedVec<LoadedAddressIndex, LoadedAddressData>,
emptyaddressindex_to_emptyaddressdata: &IndexedVec<EmptyAddressIndex, EmptyAddressData>,
addresstypeindex_to_anyaddressindex_reader_opt: &ByAddressType<Option<Reader>>,
anyaddressindex_to_anyaddressdata_reader_opt: &ByAnyAddress<Option<Reader>>,
p2pk33addressindex_to_anyaddressindex: &StampedVec<P2PK33AddressIndex, AnyAddressIndex>,
p2pk65addressindex_to_anyaddressindex: &StampedVec<P2PK65AddressIndex, AnyAddressIndex>,
p2pkhaddressindex_to_anyaddressindex: &StampedVec<P2PKHAddressIndex, AnyAddressIndex>,
p2shaddressindex_to_anyaddressindex: &StampedVec<P2SHAddressIndex, AnyAddressIndex>,
p2traddressindex_to_anyaddressindex: &StampedVec<P2TRAddressIndex, AnyAddressIndex>,
p2wpkhaddressindex_to_anyaddressindex: &StampedVec<P2WPKHAddressIndex, AnyAddressIndex>,
p2wshaddressindex_to_anyaddressindex: &StampedVec<P2WSHAddressIndex, AnyAddressIndex>,
p2aaddressindex_to_anyaddressindex: &StampedVec<P2AAddressIndex, AnyAddressIndex>,
loadedaddressindex_to_loadedaddressdata: &StampedVec<LoadedAddressIndex, LoadedAddressData>,
emptyaddressindex_to_emptyaddressdata: &StampedVec<EmptyAddressIndex, EmptyAddressData>,
) -> Option<WithAddressDataSource<LoadedAddressData>> {
if *first_addressindexes.get(address_type).unwrap() <= typeindex {
return Some(WithAddressDataSource::New(LoadedAddressData::default()));
@@ -1460,7 +1461,7 @@ impl Vecs {
return None;
}
let mmap = addresstypeindex_to_anyaddressindex_mmap_opt
let mmap = addresstypeindex_to_anyaddressindex_reader_opt
.get_unwrap(address_type)
.as_ref()
.unwrap();
@@ -1498,7 +1499,7 @@ impl Vecs {
Some(match anyaddressindex.to_enum() {
AnyAddressDataIndexEnum::Loaded(loadedaddressindex) => {
let mmap = anyaddressindex_to_anyaddressdata_mmap_opt
let mmap = anyaddressindex_to_anyaddressdata_reader_opt
.loaded
.as_ref()
.unwrap();
@@ -1515,7 +1516,7 @@ impl Vecs {
))
}
AnyAddressDataIndexEnum::Empty(emtpyaddressindex) => {
let mmap = anyaddressindex_to_anyaddressdata_mmap_opt
let mmap = anyaddressindex_to_anyaddressdata_reader_opt
.empty
.as_ref()
.unwrap();
@@ -1534,60 +1535,60 @@ impl Vecs {
})
}
fn reset_mmaps_options(
fn reset_readers_options(
&self,
addresstypeindex_to_anyaddressindex_mmap_opt: &mut ByAddressType<Option<Mmap>>,
anyaddressindex_to_anyaddressdata_mmap_opt: &mut ByAnyAddress<Option<Mmap>>,
addresstypeindex_to_anyaddressindex_reader_opt: &mut ByAddressType<Option<Reader>>,
anyaddressindex_to_anyaddressdata_reader_opt: &mut ByAnyAddress<Option<Reader>>,
) {
addresstypeindex_to_anyaddressindex_mmap_opt.p2pk65.replace(
self.p2pk65addressindex_to_anyaddressindex
.create_mmap()
.unwrap(),
);
addresstypeindex_to_anyaddressindex_mmap_opt.p2pk33.replace(
self.p2pk33addressindex_to_anyaddressindex
.create_mmap()
.unwrap(),
);
addresstypeindex_to_anyaddressindex_mmap_opt.p2pkh.replace(
self.p2pkhaddressindex_to_anyaddressindex
.create_mmap()
.unwrap(),
);
addresstypeindex_to_anyaddressindex_mmap_opt.p2sh.replace(
addresstypeindex_to_anyaddressindex_reader_opt
.p2pk65
.replace(
self.p2pk65addressindex_to_anyaddressindex
.create_static_reader(),
);
addresstypeindex_to_anyaddressindex_reader_opt
.p2pk33
.replace(
self.p2pk33addressindex_to_anyaddressindex
.create_static_reader(),
);
addresstypeindex_to_anyaddressindex_reader_opt
.p2pkh
.replace(
self.p2pkhaddressindex_to_anyaddressindex
.create_static_reader(),
);
addresstypeindex_to_anyaddressindex_reader_opt.p2sh.replace(
self.p2shaddressindex_to_anyaddressindex
.create_mmap()
.unwrap(),
.create_static_reader(),
);
addresstypeindex_to_anyaddressindex_mmap_opt.p2wpkh.replace(
self.p2wpkhaddressindex_to_anyaddressindex
.create_mmap()
.unwrap(),
);
addresstypeindex_to_anyaddressindex_mmap_opt.p2wsh.replace(
self.p2wshaddressindex_to_anyaddressindex
.create_mmap()
.unwrap(),
);
addresstypeindex_to_anyaddressindex_mmap_opt.p2tr.replace(
addresstypeindex_to_anyaddressindex_reader_opt
.p2wpkh
.replace(
self.p2wpkhaddressindex_to_anyaddressindex
.create_static_reader(),
);
addresstypeindex_to_anyaddressindex_reader_opt
.p2wsh
.replace(
self.p2wshaddressindex_to_anyaddressindex
.create_static_reader(),
);
addresstypeindex_to_anyaddressindex_reader_opt.p2tr.replace(
self.p2traddressindex_to_anyaddressindex
.create_mmap()
.unwrap(),
.create_static_reader(),
);
addresstypeindex_to_anyaddressindex_mmap_opt.p2a.replace(
addresstypeindex_to_anyaddressindex_reader_opt.p2a.replace(
self.p2aaddressindex_to_anyaddressindex
.create_mmap()
.unwrap(),
.create_static_reader(),
);
anyaddressindex_to_anyaddressdata_mmap_opt.loaded.replace(
anyaddressindex_to_anyaddressdata_reader_opt.loaded.replace(
self.loadedaddressindex_to_loadedaddressdata
.create_mmap()
.unwrap(),
.create_static_reader(),
);
anyaddressindex_to_anyaddressdata_mmap_opt.empty.replace(
anyaddressindex_to_anyaddressdata_reader_opt.empty.replace(
self.emptyaddressindex_to_emptyaddressdata
.create_mmap()
.unwrap(),
.create_static_reader(),
);
}
@@ -1765,16 +1766,26 @@ impl Vecs {
})
})?;
self.p2pk33addressindex_to_anyaddressindex.flush(height)?;
self.p2pk65addressindex_to_anyaddressindex.flush(height)?;
self.p2pkhaddressindex_to_anyaddressindex.flush(height)?;
self.p2shaddressindex_to_anyaddressindex.flush(height)?;
self.p2traddressindex_to_anyaddressindex.flush(height)?;
self.p2wpkhaddressindex_to_anyaddressindex.flush(height)?;
self.p2wshaddressindex_to_anyaddressindex.flush(height)?;
self.p2aaddressindex_to_anyaddressindex.flush(height)?;
self.loadedaddressindex_to_loadedaddressdata.flush(height)?;
self.emptyaddressindex_to_emptyaddressdata.flush(height)?;
self.p2pk33addressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
self.p2pk65addressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
self.p2pkhaddressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
self.p2shaddressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
self.p2traddressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
self.p2wpkhaddressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
self.p2wshaddressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
self.p2aaddressindex_to_anyaddressindex
.flush(Stamp::from(u64::from(height)))?;
self.loadedaddressindex_to_loadedaddressdata
.flush(Stamp::from(u64::from(height)))?;
self.emptyaddressindex_to_emptyaddressdata
.flush(Stamp::from(u64::from(height)))?;
self.chain_state.truncate_if_needed(Height::ZERO)?;
chain_state.iter().for_each(|block_state| {

6
crates/brk_computer/src/stateful/range_map.rs
View File

@@ -1,6 +1,6 @@
use std::collections::BTreeMap;
use brk_vec::{IndexedVec, StoredIndex, StoredType};
use brk_vecs::{StampedVec, StoredIndex, StoredType};
#[derive(Debug)]
pub struct RangeMap<I, T>(BTreeMap<I, T>);
@@ -20,12 +20,12 @@ where
}
}
impl<I, T> From<&IndexedVec<I, T>> for RangeMap<T, I>
impl<I, T> From<&StampedVec<I, T>> for RangeMap<T, I>
where
I: StoredIndex,
T: StoredIndex + StoredType,
{
fn from(vec: &IndexedVec<I, T>) -> Self {
fn from(vec: &StampedVec<I, T>) -> Self {
Self(
vec.into_iter()
.map(|(i, v)| (v.into_owned(), i))

2
crates/brk_computer/src/stateful/trait.rs
View File

@@ -1,7 +1,7 @@
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec};
use brk_vecs::{AnyCollectableVec, AnyIterableVec};
use crate::{Indexes, fetched, indexes, market};

8
crates/brk_computer/src/stateful/utxo_cohort.rs
View File

@@ -1,9 +1,9 @@
use std::{ops::Deref, path::Path};
use std::{ops::Deref, path::Path, sync::Arc};
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, Format};
use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, File, Format};
use crate::{
Indexes, UTXOCohortState, fetched, indexes, market,
@@ -25,7 +25,7 @@ pub struct Vecs {
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
file: &Arc<File>,
cohort_name: Option<&str>,
computation: Computation,
format: Format,
@@ -48,7 +48,7 @@ impl Vecs {
)?,
inner: common::Vecs::forced_import(
path,
file,
cohort_name,
computation,
format,

238
crates/brk_computer/src/stateful/utxo_cohorts.rs
View File

@@ -1,4 +1,4 @@
use std::{collections::BTreeMap, ops::ControlFlow, path::Path};
use std::{collections::BTreeMap, ops::ControlFlow, path::Path, sync::Arc};
use brk_core::{
Bitcoin, ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount, ByMaxAge,
@@ -7,7 +7,7 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyIterableVec, Computation, Format, StoredIndex};
use brk_vecs::{AnyIterableVec, Computation, File, Format, StoredIndex};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
@@ -26,7 +26,7 @@ pub struct Vecs(UTXOGroups<(GroupFilter, utxo_cohort::Vecs)>);
impl Vecs {
pub fn forced_import(
path: &Path,
file: &Arc<File>,
version: Version,
_computation: Computation,
format: Format,
@@ -37,7 +37,7 @@ impl Vecs {
Ok(Self(
UTXOGroups {
all: utxo_cohort::Vecs::forced_import(
path,
file,
None,
_computation,
format,
@@ -50,7 +50,7 @@ impl Vecs {
)?,
term: ByTerm {
short: utxo_cohort::Vecs::forced_import(
path,
file,
Some("short_term_holders"),
_computation,
format,
@@ -62,7 +62,7 @@ impl Vecs {
true,
)?,
long: utxo_cohort::Vecs::forced_import(
path,
file,
Some("long_term_holders"),
_computation,
format,
@@ -76,7 +76,7 @@ impl Vecs {
},
epoch: ByEpoch {
_0: utxo_cohort::Vecs::forced_import(
path,
file,
Some("epoch_0"),
_computation,
format,
@@ -88,7 +88,7 @@ impl Vecs {
true,
)?,
_1: utxo_cohort::Vecs::forced_import(
path,
file,
Some("epoch_1"),
_computation,
format,
@@ -100,7 +100,7 @@ impl Vecs {
true,
)?,
_2: utxo_cohort::Vecs::forced_import(
path,
file,
Some("epoch_2"),
_computation,
format,
@@ -112,7 +112,7 @@ impl Vecs {
true,
)?,
_3: utxo_cohort::Vecs::forced_import(
path,
file,
Some("epoch_3"),
_computation,
format,
@@ -124,7 +124,7 @@ impl Vecs {
true,
)?,
_4: utxo_cohort::Vecs::forced_import(
path,
file,
Some("epoch_4"),
_computation,
format,
@@ -138,7 +138,7 @@ impl Vecs {
},
_type: BySpendableType {
p2pk65: utxo_cohort::Vecs::forced_import(
path,
file,
Some("p2pk65"),
_computation,
format,
@@ -150,7 +150,7 @@ impl Vecs {
false,
)?,
p2pk33: utxo_cohort::Vecs::forced_import(
path,
file,
Some("p2pk33"),
_computation,
format,
@@ -162,7 +162,7 @@ impl Vecs {
false,
)?,
p2pkh: utxo_cohort::Vecs::forced_import(
path,
file,
Some("p2pkh"),
_computation,
format,
@@ -174,7 +174,7 @@ impl Vecs {
false,
)?,
p2sh: utxo_cohort::Vecs::forced_import(
path,
file,
Some("p2sh"),
_computation,
format,
@@ -186,7 +186,7 @@ impl Vecs {
false,
)?,
p2wpkh: utxo_cohort::Vecs::forced_import(
path,
file,
Some("p2wpkh"),
_computation,
format,
@@ -198,7 +198,7 @@ impl Vecs {
false,
)?,
p2wsh: utxo_cohort::Vecs::forced_import(
path,
file,
Some("p2wsh"),
_computation,
format,
@@ -210,7 +210,7 @@ impl Vecs {
false,
)?,
p2tr: utxo_cohort::Vecs::forced_import(
path,
file,
Some("p2tr"),
_computation,
format,
@@ -222,7 +222,7 @@ impl Vecs {
false,
)?,
p2a: utxo_cohort::Vecs::forced_import(
path,
file,
Some("p2a"),
_computation,
format,
@@ -234,7 +234,7 @@ impl Vecs {
false,
)?,
p2ms: utxo_cohort::Vecs::forced_import(
path,
file,
Some("p2ms_outputs"),
_computation,
format,
@@ -246,7 +246,7 @@ impl Vecs {
false,
)?,
empty: utxo_cohort::Vecs::forced_import(
path,
file,
Some("empty_outputs"),
_computation,
format,
@@ -258,7 +258,7 @@ impl Vecs {
false,
)?,
unknown: utxo_cohort::Vecs::forced_import(
path,
file,
Some("unknown_outputs"),
_computation,
format,
@@ -272,7 +272,7 @@ impl Vecs {
},
max_age: ByMaxAge {
_1w: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_1w_old"),
_computation,
format,
@@ -284,7 +284,7 @@ impl Vecs {
true,
)?,
_1m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_1m_old"),
_computation,
format,
@@ -296,7 +296,7 @@ impl Vecs {
true,
)?,
_2m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_2m_old"),
_computation,
format,
@@ -308,7 +308,7 @@ impl Vecs {
true,
)?,
_3m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_3m_old"),
_computation,
format,
@@ -320,7 +320,7 @@ impl Vecs {
true,
)?,
_4m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_4m_old"),
_computation,
format,
@@ -332,7 +332,7 @@ impl Vecs {
true,
)?,
_5m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_5m_old"),
_computation,
format,
@@ -344,7 +344,7 @@ impl Vecs {
true,
)?,
_6m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_6m_old"),
_computation,
format,
@@ -356,7 +356,7 @@ impl Vecs {
true,
)?,
_1y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_1y_old"),
_computation,
format,
@@ -368,7 +368,7 @@ impl Vecs {
true,
)?,
_2y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_2y_old"),
_computation,
format,
@@ -380,7 +380,7 @@ impl Vecs {
true,
)?,
_3y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_3y_old"),
_computation,
format,
@@ -392,7 +392,7 @@ impl Vecs {
true,
)?,
_4y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_4y_old"),
_computation,
format,
@@ -404,7 +404,7 @@ impl Vecs {
true,
)?,
_5y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_5y_old"),
_computation,
format,
@@ -416,7 +416,7 @@ impl Vecs {
true,
)?,
_6y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_6y_old"),
_computation,
format,
@@ -428,7 +428,7 @@ impl Vecs {
true,
)?,
_7y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_7y_old"),
_computation,
format,
@@ -440,7 +440,7 @@ impl Vecs {
true,
)?,
_8y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_8y_old"),
_computation,
format,
@@ -452,7 +452,7 @@ impl Vecs {
true,
)?,
_10y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_10y_old"),
_computation,
format,
@@ -464,7 +464,7 @@ impl Vecs {
true,
)?,
_12y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_12y_old"),
_computation,
format,
@@ -476,7 +476,7 @@ impl Vecs {
true,
)?,
_15y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_15y_old"),
_computation,
format,
@@ -490,7 +490,7 @@ impl Vecs {
},
min_age: ByMinAge {
_1d: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_1d_old"),
_computation,
format,
@@ -502,7 +502,7 @@ impl Vecs {
true,
)?,
_1w: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_1w_old"),
_computation,
format,
@@ -514,7 +514,7 @@ impl Vecs {
true,
)?,
_1m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_1m_old"),
_computation,
format,
@@ -526,7 +526,7 @@ impl Vecs {
true,
)?,
_2m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_2m_old"),
_computation,
format,
@@ -538,7 +538,7 @@ impl Vecs {
true,
)?,
_3m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_3m_old"),
_computation,
format,
@@ -550,7 +550,7 @@ impl Vecs {
true,
)?,
_4m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_4m_old"),
_computation,
format,
@@ -562,7 +562,7 @@ impl Vecs {
true,
)?,
_5m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_5m_old"),
_computation,
format,
@@ -574,7 +574,7 @@ impl Vecs {
true,
)?,
_6m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_6m_old"),
_computation,
format,
@@ -586,7 +586,7 @@ impl Vecs {
true,
)?,
_1y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_1y_old"),
_computation,
format,
@@ -598,7 +598,7 @@ impl Vecs {
true,
)?,
_2y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_2y_old"),
_computation,
format,
@@ -610,7 +610,7 @@ impl Vecs {
true,
)?,
_3y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_3y_old"),
_computation,
format,
@@ -622,7 +622,7 @@ impl Vecs {
true,
)?,
_4y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_4y_old"),
_computation,
format,
@@ -634,7 +634,7 @@ impl Vecs {
true,
)?,
_5y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_5y_old"),
_computation,
format,
@@ -646,7 +646,7 @@ impl Vecs {
true,
)?,
_6y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_6y_old"),
_computation,
format,
@@ -658,7 +658,7 @@ impl Vecs {
true,
)?,
_7y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_7y_old"),
_computation,
format,
@@ -670,7 +670,7 @@ impl Vecs {
true,
)?,
_8y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_8y_old"),
_computation,
format,
@@ -682,7 +682,7 @@ impl Vecs {
true,
)?,
_10y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_10y_old"),
_computation,
format,
@@ -694,7 +694,7 @@ impl Vecs {
true,
)?,
_12y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_12y_old"),
_computation,
format,
@@ -708,7 +708,7 @@ impl Vecs {
},
age_range: ByAgeRange {
up_to_1d: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_up_to_1d_old"),
_computation,
format,
@@ -720,7 +720,7 @@ impl Vecs {
true,
)?,
_1d_to_1w: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_1d_up_to_1w_old"),
_computation,
format,
@@ -732,7 +732,7 @@ impl Vecs {
true,
)?,
_1w_to_1m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_1w_up_to_1m_old"),
_computation,
format,
@@ -744,7 +744,7 @@ impl Vecs {
true,
)?,
_1m_to_2m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_1m_up_to_2m_old"),
_computation,
format,
@@ -756,7 +756,7 @@ impl Vecs {
true,
)?,
_2m_to_3m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_2m_up_to_3m_old"),
_computation,
format,
@@ -768,7 +768,7 @@ impl Vecs {
true,
)?,
_3m_to_4m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_3m_up_to_4m_old"),
_computation,
format,
@@ -780,7 +780,7 @@ impl Vecs {
true,
)?,
_4m_to_5m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_4m_up_to_5m_old"),
_computation,
format,
@@ -792,7 +792,7 @@ impl Vecs {
true,
)?,
_5m_to_6m: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_5m_up_to_6m_old"),
_computation,
format,
@@ -804,7 +804,7 @@ impl Vecs {
true,
)?,
_6m_to_1y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_6m_up_to_1y_old"),
_computation,
format,
@@ -816,7 +816,7 @@ impl Vecs {
true,
)?,
_1y_to_2y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_1y_up_to_2y_old"),
_computation,
format,
@@ -828,7 +828,7 @@ impl Vecs {
true,
)?,
_2y_to_3y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_2y_up_to_3y_old"),
_computation,
format,
@@ -840,7 +840,7 @@ impl Vecs {
true,
)?,
_3y_to_4y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_3y_up_to_4y_old"),
_computation,
format,
@@ -852,7 +852,7 @@ impl Vecs {
true,
)?,
_4y_to_5y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_4y_up_to_5y_old"),
_computation,
format,
@@ -864,7 +864,7 @@ impl Vecs {
true,
)?,
_5y_to_6y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_5y_up_to_6y_old"),
_computation,
format,
@@ -876,7 +876,7 @@ impl Vecs {
true,
)?,
_6y_to_7y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_6y_up_to_7y_old"),
_computation,
format,
@@ -888,7 +888,7 @@ impl Vecs {
true,
)?,
_7y_to_8y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_7y_up_to_8y_old"),
_computation,
format,
@@ -900,7 +900,7 @@ impl Vecs {
true,
)?,
_8y_to_10y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_8y_up_to_10y_old"),
_computation,
format,
@@ -912,7 +912,7 @@ impl Vecs {
true,
)?,
_10y_to_12y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_10y_up_to_12y_old"),
_computation,
format,
@@ -924,7 +924,7 @@ impl Vecs {
true,
)?,
_12y_to_15y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_12y_up_to_15y_old"),
_computation,
format,
@@ -936,7 +936,7 @@ impl Vecs {
true,
)?,
from_15y: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_at_least_15y_old"),
_computation,
format,
@@ -950,7 +950,7 @@ impl Vecs {
},
amount_range: ByAmountRange {
_0sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_with_0sats"),
_computation,
format,
@@ -962,7 +962,7 @@ impl Vecs {
false,
)?,
_1sat_to_10sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_1sat_under_10sats"),
_computation,
format,
@@ -974,7 +974,7 @@ impl Vecs {
false,
)?,
_10sats_to_100sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_10sats_under_100sats"),
_computation,
format,
@@ -986,7 +986,7 @@ impl Vecs {
false,
)?,
_100sats_to_1k_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_100sats_under_1k_sats"),
_computation,
format,
@@ -998,7 +998,7 @@ impl Vecs {
false,
)?,
_1k_sats_to_10k_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_1k_sats_under_10k_sats"),
_computation,
format,
@@ -1010,7 +1010,7 @@ impl Vecs {
false,
)?,
_10k_sats_to_100k_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_10k_sats_under_100k_sats"),
_computation,
format,
@@ -1022,7 +1022,7 @@ impl Vecs {
false,
)?,
_100k_sats_to_1m_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_100k_sats_under_1m_sats"),
_computation,
format,
@@ -1034,7 +1034,7 @@ impl Vecs {
false,
)?,
_1m_sats_to_10m_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_1m_sats_under_10m_sats"),
_computation,
format,
@@ -1046,7 +1046,7 @@ impl Vecs {
false,
)?,
_10m_sats_to_1btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_10m_sats_under_1btc"),
_computation,
format,
@@ -1058,7 +1058,7 @@ impl Vecs {
false,
)?,
_1btc_to_10btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_1btc_under_10btc"),
_computation,
format,
@@ -1070,7 +1070,7 @@ impl Vecs {
false,
)?,
_10btc_to_100btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_10btc_under_100btc"),
_computation,
format,
@@ -1082,7 +1082,7 @@ impl Vecs {
false,
)?,
_100btc_to_1k_btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_100btc_under_1k_btc"),
_computation,
format,
@@ -1094,7 +1094,7 @@ impl Vecs {
false,
)?,
_1k_btc_to_10k_btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_1k_btc_under_10k_btc"),
_computation,
format,
@@ -1106,7 +1106,7 @@ impl Vecs {
false,
)?,
_10k_btc_to_100k_btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_10k_btc_under_100k_btc"),
_computation,
format,
@@ -1118,7 +1118,7 @@ impl Vecs {
false,
)?,
_100k_btc_or_more: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_100k_btc"),
_computation,
format,
@@ -1132,7 +1132,7 @@ impl Vecs {
},
lt_amount: ByLowerThanAmount {
_10sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_10sats"),
_computation,
format,
@@ -1144,7 +1144,7 @@ impl Vecs {
false,
)?,
_100sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_100sats"),
_computation,
format,
@@ -1156,7 +1156,7 @@ impl Vecs {
false,
)?,
_1k_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_1k_sats"),
_computation,
format,
@@ -1168,7 +1168,7 @@ impl Vecs {
false,
)?,
_10k_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_10k_sats"),
_computation,
format,
@@ -1180,7 +1180,7 @@ impl Vecs {
false,
)?,
_100k_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_100k_sats"),
_computation,
format,
@@ -1192,7 +1192,7 @@ impl Vecs {
false,
)?,
_1m_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_1m_sats"),
_computation,
format,
@@ -1204,7 +1204,7 @@ impl Vecs {
false,
)?,
_10m_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_10m_sats"),
_computation,
format,
@@ -1216,7 +1216,7 @@ impl Vecs {
false,
)?,
_1btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_1btc"),
_computation,
format,
@@ -1228,7 +1228,7 @@ impl Vecs {
false,
)?,
_10btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_10btc"),
_computation,
format,
@@ -1240,7 +1240,7 @@ impl Vecs {
false,
)?,
_100btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_100btc"),
_computation,
format,
@@ -1252,7 +1252,7 @@ impl Vecs {
false,
)?,
_1k_btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_1k_btc"),
_computation,
format,
@@ -1264,7 +1264,7 @@ impl Vecs {
false,
)?,
_10k_btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_10k_btc"),
_computation,
format,
@@ -1276,7 +1276,7 @@ impl Vecs {
false,
)?,
_100k_btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_under_100k_btc"),
_computation,
format,
@@ -1290,7 +1290,7 @@ impl Vecs {
},
ge_amount: ByGreatEqualAmount {
_1sat: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_1sat"),
_computation,
format,
@@ -1302,7 +1302,7 @@ impl Vecs {
false,
)?,
_10sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_10sats"),
_computation,
format,
@@ -1314,7 +1314,7 @@ impl Vecs {
false,
)?,
_100sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_100sats"),
_computation,
format,
@@ -1326,7 +1326,7 @@ impl Vecs {
false,
)?,
_1k_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_1k_sats"),
_computation,
format,
@@ -1338,7 +1338,7 @@ impl Vecs {
false,
)?,
_10k_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_10k_sats"),
_computation,
format,
@@ -1350,7 +1350,7 @@ impl Vecs {
false,
)?,
_100k_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_100k_sats"),
_computation,
format,
@@ -1362,7 +1362,7 @@ impl Vecs {
false,
)?,
_1m_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_1m_sats"),
_computation,
format,
@@ -1374,7 +1374,7 @@ impl Vecs {
false,
)?,
_10m_sats: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_10m_sats"),
_computation,
format,
@@ -1386,7 +1386,7 @@ impl Vecs {
false,
)?,
_1btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_1btc"),
_computation,
format,
@@ -1398,7 +1398,7 @@ impl Vecs {
false,
)?,
_10btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_10btc"),
_computation,
format,
@@ -1410,7 +1410,7 @@ impl Vecs {
false,
)?,
_100btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_100btc"),
_computation,
format,
@@ -1422,7 +1422,7 @@ impl Vecs {
false,
)?,
_1k_btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_1k_btc"),
_computation,
format,
@@ -1434,7 +1434,7 @@ impl Vecs {
false,
)?,
_10k_btc: utxo_cohort::Vecs::forced_import(
path,
file,
Some("utxos_above_10k_btc"),
_computation,
format,

80
crates/brk_computer/src/transactions.rs
View File

@@ -1,4 +1,4 @@
use std::path::Path;
use std::sync::Arc;
use brk_core::{
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32,
@@ -6,9 +6,9 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, ComputedVec,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, File, Format, StoredIndex, VecIterator,
};
use crate::grouped::{
@@ -86,7 +86,7 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(
path: &Path,
file: &Arc<File>,
version: Version,
indexer: &Indexer,
indexes: &indexes::Vecs,
@@ -98,7 +98,7 @@ impl Vecs {
let inputindex_to_value = ComputedVec::forced_import_or_init_from_2(
computation,
path,
file,
"value",
version + VERSION + Version::ZERO,
format,
@@ -125,7 +125,7 @@ impl Vecs {
let txindex_to_weight = ComputedVec::forced_import_or_init_from_2(
computation,
path,
file,
"weight",
version + VERSION + Version::ZERO,
format,
@@ -153,7 +153,7 @@ impl Vecs {
let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1(
computation,
path,
file,
"vsize",
version + VERSION + Version::ZERO,
format,
@@ -170,7 +170,7 @@ impl Vecs {
let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2(
computation,
path,
file,
"is_coinbase",
version + VERSION + Version::ZERO,
format,
@@ -194,7 +194,7 @@ impl Vecs {
let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3(
computation,
path,
file,
"input_value",
version + VERSION + Version::ZERO,
format,
@@ -230,7 +230,7 @@ impl Vecs {
// let indexes_to_input_value: ComputedVecsFromTxindex<Sats> =
// ComputedVecsFromTxindex::forced_import(
// path,
// file,
// "input_value",
// true,
// version + VERSION + Version::ZERO,
@@ -244,7 +244,7 @@ impl Vecs {
let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3(
computation,
path,
file,
"output_value",
version + VERSION + Version::ZERO,
format,
@@ -280,7 +280,7 @@ impl Vecs {
// let indexes_to_output_value: ComputedVecsFromTxindex<Sats> =
// ComputedVecsFromTxindex::forced_import(
// path,
// file,
// "output_value",
// true,
// version + VERSION + Version::ZERO,
@@ -294,7 +294,7 @@ impl Vecs {
let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2(
computation,
path,
file,
"fee",
version + VERSION + Version::ZERO,
format,
@@ -317,7 +317,7 @@ impl Vecs {
let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2(
computation,
path,
file,
"feerate",
version + VERSION + Version::ZERO,
format,
@@ -337,7 +337,7 @@ impl Vecs {
Ok(Self {
indexes_to_tx_count: ComputedVecsFromHeight::forced_import(
path,
file,
"tx_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -352,7 +352,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_input_count: ComputedVecsFromTxindex::forced_import(
path,
file,
"input_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -367,7 +367,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_output_count: ComputedVecsFromTxindex::forced_import(
path,
file,
"output_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -382,7 +382,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
path,
file,
"tx_v1",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -392,7 +392,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_tx_v2: ComputedVecsFromHeight::forced_import(
path,
file,
"tx_v2",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -402,7 +402,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_tx_v3: ComputedVecsFromHeight::forced_import(
path,
file,
"tx_v3",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -412,7 +412,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_fee: ComputedValueVecsFromTxindex::forced_import(
path,
file,
"fee",
indexes,
Source::Vec(txindex_to_fee.boxed_clone()),
@@ -428,7 +428,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_feerate: ComputedVecsFromTxindex::forced_import(
path,
file,
"feerate",
Source::None,
version + VERSION + Version::ZERO,
@@ -441,7 +441,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import(
path,
file,
"tx_vsize",
Source::None,
version + VERSION + Version::ZERO,
@@ -454,7 +454,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import(
path,
file,
"tx_weight",
Source::None,
version + VERSION + Version::ZERO,
@@ -467,7 +467,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import(
path,
file,
"subsidy",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -483,7 +483,7 @@ impl Vecs {
indexes,
)?,
indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import(
path,
file,
"coinbase",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -499,7 +499,7 @@ impl Vecs {
indexes,
)?,
indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import(
path,
file,
"unclaimed_rewards",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -510,7 +510,7 @@ impl Vecs {
indexes,
)?,
indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
path,
file,
"p2a_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -525,7 +525,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import(
path,
file,
"p2ms_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -540,7 +540,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import(
path,
file,
"p2pk33_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -555,7 +555,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import(
path,
file,
"p2pk65_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -570,7 +570,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import(
path,
file,
"p2pkh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -585,7 +585,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import(
path,
file,
"p2sh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -600,7 +600,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import(
path,
file,
"p2tr_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -615,7 +615,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import(
path,
file,
"p2wpkh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -630,7 +630,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import(
path,
file,
"p2wsh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -645,7 +645,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
path,
file,
"opreturn_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -660,7 +660,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
path,
file,
"unknownoutput_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -675,7 +675,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
path,
file,
"emptyoutput_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -690,7 +690,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import(
path,
file,
"exact_utxo_count",
Source::Compute,
version + VERSION + Version::ZERO,

8
crates/brk_core/src/groups/by_address_type.rs
View File

@@ -171,3 +171,11 @@ where
self.p2a += rhs.p2a;
}
}
impl<T> ByAddressType<Option<T>> {
pub fn take(&mut self) {
self.as_mut_vec().into_iter().for_each(|opt| {
opt.take();
});
}
}

7
crates/brk_core/src/groups/by_any_address.rs
View File

@@ -3,3 +3,10 @@ pub struct ByAnyAddress<T> {
pub loaded: T,
pub empty: T,
}
impl<T> ByAnyAddress<Option<T>> {
pub fn take(&mut self) {
self.loaded.take();
self.empty.take();
}
}

2
crates/brk_core/src/structs/addressbyteshash.rs
View File

@@ -1,5 +1,3 @@
use std::hash::Hasher;
use byteview::ByteView;
use derive_deref::Deref;
use zerocopy::{FromBytes, IntoBytes};

2
crates/brk_core/src/structs/height.rs
View File

@@ -35,7 +35,7 @@ impl Height {
pub const ZERO: Self = Self(0);
pub const MAX: Self = Self(u32::MAX);
pub fn new(height: u32) -> Self {
pub const fn new(height: u32) -> Self {
Self(height)
}

6
crates/brk_indexer/examples/indexer.rs
View File

@@ -15,9 +15,9 @@ fn main() -> color_eyre::Result<()> {
brk_logger::init(Some(Path::new(".log")));
// let bitcoin_dir = brk_core::default_bitcoin_path();
let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
// let outputs_dir = brk_core::default_brk_path().join("outputs");
let outputs_dir = Path::new("/Volumes/WD_BLACK/brk/outputs");
let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk");
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
@@ -33,7 +33,7 @@ fn main() -> color_eyre::Result<()> {
loop {
let i = Instant::now();
indexer.index(&parser, rpc, &exit, false)?;
indexer.index(&parser, rpc, &exit, true)?;
dbg!(i.elapsed());
sleep(Duration::from_secs(5 * 60));

6
crates/brk_indexer/src/indexes.rs
View File

@@ -89,8 +89,10 @@ impl Indexes {
impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
type Error = color_eyre::Report;
fn try_from((vecs, stores, rpc): (&mut Vecs, &Stores, &Client)) -> color_eyre::Result<Self> {
// Height at which we wanna start: min last saved + 1 or 0
let starting_height = vecs.starting_height().min(stores.starting_height());
// Height at which we want to start: min last saved + 1 or 0
let vecs_starting_height = vecs.starting_height();
let stores_starting_height = stores.starting_height();
let starting_height = vecs_starting_height.min(stores_starting_height);
let range = u32::from(
starting_height

95
crates/brk_indexer/src/lib.rs
View File

@@ -3,18 +3,18 @@
#![doc = include_str!("../examples/indexer.rs")]
#![doc = "```"]
use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc, thread};
use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc, thread, time::Instant};
use bitcoin::{Transaction, TxIn, TxOut};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
OutputType, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, TypeIndexWithOutputindex,
Unit, Version, Vin, Vout, setrlimit,
OutputType, Result, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex,
TypeIndexWithOutputindex, Unit, Version, Vin, Vout, setrlimit,
};
use brk_exit::Exit;
use brk_parser::Parser;
use brk_store::AnyStore;
use brk_vecs::{AnyVec, File, Reader, VecIterator};
use brk_vecs::{AnyVec, File, PAGE_SIZE, Reader, VecIterator};
use color_eyre::eyre::{ContextCompat, eyre};
use log::{error, info};
use rayon::prelude::*;
@@ -27,7 +27,7 @@ pub use stores::*;
pub use vecs::*;
const SNAPSHOT_BLOCK_RANGE: usize = 1000;
const COLLISIONS_CHECKED_UP_TO: u32 = 893_000;
const COLLISIONS_CHECKED_UP_TO: Height = Height::new(415_000);
const VERSION: Version = Version::ONE;
#[derive(Clone)]
@@ -41,11 +41,18 @@ impl Indexer {
pub fn forced_import(outputs_dir: &Path) -> color_eyre::Result<Self> {
setrlimit()?;
let file = Arc::new(File::open(&outputs_dir.join("vecs"))?);
let file = Arc::new(File::open(&outputs_dir.join("indexed/vecs"))?);
let vecs = Vecs::forced_import(&file, VERSION + Version::ZERO)?;
file.set_min_len(PAGE_SIZE * 50_000_000)?;
Ok(Self {
vecs: Vecs::forced_import(&file, VERSION + Version::ZERO)?,
stores: Stores::forced_import(&outputs_dir.join("stores"), VERSION + Version::ZERO)?,
vecs,
stores: Stores::forced_import(
&outputs_dir.join("indexed/stores"),
VERSION + Version::ZERO,
)?,
file,
})
}
@@ -57,6 +64,8 @@ impl Indexer {
exit: &Exit,
check_collisions: bool,
) -> color_eyre::Result<Indexes> {
let file = self.file.clone();
let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc))
.unwrap_or_else(|_report| Indexes::default());
@@ -87,23 +96,30 @@ impl Indexer {
info!("Started indexing...");
let export_if_needed = |stores: &mut Stores,
vecs: &mut Vecs,
height: Height,
rem: bool,
exit: &Exit|
-> color_eyre::Result<bool> {
if height == 0 || (height % SNAPSHOT_BLOCK_RANGE != 0) != rem {
return Ok(false);
}
info!("Exporting...");
let _lock = exit.lock();
stores.commit(height)?;
vecs.flush(height)?;
Ok(true)
let should_export = |height: Height, rem: bool| -> bool {
height != 0 && (height % SNAPSHOT_BLOCK_RANGE == 0) != rem
};
let export =
|stores: &mut Stores, vecs: &mut Vecs, height: Height, exit: &Exit| -> Result<()> {
info!("Exporting...");
let _lock = exit.lock();
thread::scope(|scope| -> Result<()> {
scope.spawn(|| {
let i = Instant::now();
stores.commit(height).unwrap();
info!("Commited stores in {}s", i.elapsed().as_secs());
});
let i = Instant::now();
vecs.flush(height)?;
info!("Flushed vecs in {}s", i.elapsed().as_secs());
let i = Instant::now();
file.flush()?;
info!("Flushed file in {}s", i.elapsed().as_secs());
Ok(())
})
};
let mut txindex_to_first_outputindex_reader_opt = None;
let mut p2pk65addressindex_to_p2pk65bytes_reader_opt = None;
let mut p2pk33addressindex_to_p2pk33bytes_reader_opt = None;
@@ -181,7 +197,7 @@ impl Indexer {
let p2aaddressindex_to_p2abytes_mmap = p2aaddressindex_to_p2abytes_reader_opt.as_ref().unwrap();
// Used to check rapidhash collisions
let check_collisions = check_collisions && height > Height::new(COLLISIONS_CHECKED_UP_TO);
let check_collisions = check_collisions && height > COLLISIONS_CHECKED_UP_TO ;
let blockhash = BlockHash::from(blockhash);
let blockhash_prefix = BlockHashPrefix::from(&blockhash);
@@ -739,19 +755,18 @@ impl Indexer {
idxs.inputindex += InputIndex::from(inputs_len);
idxs.outputindex += OutputIndex::from(outputs_len);
txindex_to_first_outputindex_reader_opt.take();
p2pk65addressindex_to_p2pk65bytes_reader_opt.take();
p2pk33addressindex_to_p2pk33bytes_reader_opt.take();
p2pkhaddressindex_to_p2pkhbytes_reader_opt.take();
p2shaddressindex_to_p2shbytes_reader_opt.take();
p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take();
p2wshaddressindex_to_p2wshbytes_reader_opt.take();
p2traddressindex_to_p2trbytes_reader_opt.take();
p2aaddressindex_to_p2abytes_reader_opt.take();
let exported = export_if_needed(stores, vecs, height, false, exit)?;
if exported {
if should_export(height, false) {
txindex_to_first_outputindex_reader_opt.take();
p2pk65addressindex_to_p2pk65bytes_reader_opt.take();
p2pk33addressindex_to_p2pk33bytes_reader_opt.take();
p2pkhaddressindex_to_p2pkhbytes_reader_opt.take();
p2shaddressindex_to_p2shbytes_reader_opt.take();
p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take();
p2wshaddressindex_to_p2wshbytes_reader_opt.take();
p2traddressindex_to_p2trbytes_reader_opt.take();
p2aaddressindex_to_p2abytes_reader_opt.take();
export(stores, vecs, height, exit)?;
reset_mmaps_options(
vecs,
&mut txindex_to_first_outputindex_reader_opt,
@@ -770,9 +785,13 @@ impl Indexer {
},
)?;
export_if_needed(stores, vecs, idxs.height, true, exit)?;
if should_export(idxs.height, true) {
export(stores, vecs, idxs.height, exit)?;
}
unsafe { libc::sync() }
let i = Instant::now();
file.punch_holes()?;
info!("Punched holes in file in {}s", i.elapsed().as_secs());
Ok(starting_indexes)
}

2
crates/brk_interface/Cargo.toml
View File

@@ -11,7 +11,7 @@ repository.workspace = true
brk_core = { workspace = true }
brk_computer = { workspace = true }
brk_indexer = { workspace = true }
brk_vec = { workspace = true }
brk_vecs = { workspace = true }
color-eyre = { workspace = true }
derive_deref = { workspace = true }
rmcp = { workspace = true }

2
crates/brk_interface/examples/main.rs
View File

@@ -3,7 +3,7 @@ use std::path::Path;
use brk_computer::Computer;
use brk_indexer::Indexer;
use brk_interface::{Index, Interface, Params, ParamsOpt};
use brk_vec::{Computation, Format};
use brk_vecs::{Computation, Format};
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;

4
crates/brk_interface/src/lib.rs
View File

@@ -8,7 +8,7 @@ use std::collections::BTreeMap;
use brk_computer::Computer;
use brk_core::{Height, Result};
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIndexedVec};
use brk_vecs::{AnyCollectableVec, AnyStampedVec};
use tabled::settings::Style;
mod deser;
@@ -47,7 +47,7 @@ impl<'a> Interface<'a> {
}
pub fn get_height(&self) -> Height {
self._indexer.vecs.height_to_blockhash.height()
Height::from(u64::from(self._indexer.vecs.height_to_blockhash.stamp()))
}
pub fn search(&self, params: &Params) -> Vec<(String, &&dyn AnyCollectableVec)> {

2
crates/brk_interface/src/vecs.rs
View File

@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
use brk_computer::Computer;
use brk_indexer::Indexer;
use brk_vec::AnyCollectableVec;
use brk_vecs::AnyCollectableVec;
use derive_deref::{Deref, DerefMut};
use crate::pagination::{PaginatedIndexParam, PaginationParam};

2
crates/brk_server/Cargo.toml
View File

@@ -20,7 +20,7 @@ brk_interface = { workspace = true }
brk_logger = { workspace = true }
brk_mcp = { workspace = true }
brk_parser = { workspace = true }
brk_vec = { workspace = true }
brk_vecs = { workspace = true }
clap = { workspace = true }
clap_derive = { workspace = true }
color-eyre = { workspace = true }

2
crates/brk_server/examples/main.rs
View File

@@ -8,7 +8,7 @@ use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_server::{Server, Website};
use brk_vec::{Computation, Format};
use brk_vecs::{Computation, Format};
pub fn main() -> color_eyre::Result<()> {
color_eyre::install()?;

7
crates/brk_vecs/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "brk_vecs"
description = "A storeable vec"
description = "A simple index/value store"
keywords = ["vec", "disk", "data"]
categories = ["database"]
version.workspace = true
@@ -10,13 +10,14 @@ homepage.workspace = true
repository.workspace = true
[dependencies]
bincode = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
clap = { workspace = true }
clap_derive = { workspace = true }
libc = "0.2.174"
log = { workspace = true }
memmap2 = "0.9.7"
parking_lot = {workspace = true}
parking_lot = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }

34
crates/brk_vecs/examples/file.rs
View File

@@ -8,13 +8,13 @@ fn main() -> Result<()> {
let file = File::open(Path::new("vecs"))?;
let file_min_len = PAGE_SIZE * 1_000_000;
let min_regions = 20_000;
// let file_min_len = PAGE_SIZE * 1_000_000;
// let min_regions = 20_000;
file.set_min_len(file_min_len)?;
file.set_min_regions(min_regions)?;
// file.set_min_len(file_min_len)?;
// file.set_min_regions(min_regions)?;
let (region1_i, region1) = file.create_region_if_needed("region1")?;
let (region1_i, _) = file.create_region_if_needed("region1")?;
{
let layout = file.layout();
@@ -150,9 +150,11 @@ fn main() -> Result<()> {
assert!(layout.start_to_hole().is_empty());
}
let (region1_i, region1) = file.create_region_if_needed("region1")?;
let (region2_i, region2) = file.create_region_if_needed("region2")?;
let (region3_i, region3) = file.create_region_if_needed("region3")?;
let (region1_i, _) = file.create_region_if_needed("region1")?;
let (region2_i, _) = file.create_region_if_needed("region2")?;
let (region3_i, _) = file.create_region_if_needed("region3")?;
// dbg!(file.layout());
{
let regions = file.regions();
@@ -228,7 +230,7 @@ fn main() -> Result<()> {
);
}
let (region2_i, region2) = file.create_region_if_needed("region2")?;
let (region2_i, _) = file.create_region_if_needed("region2")?;
{
assert!(region2_i == 1)
@@ -312,7 +314,7 @@ fn main() -> Result<()> {
assert!(start_to_hole.is_empty());
}
let (region2_i, region2) = file.create_region_if_needed("region2")?;
let (region2_i, _) = file.create_region_if_needed("region2")?;
{
let regions = file.regions();
@@ -408,12 +410,14 @@ fn main() -> Result<()> {
file.write_all_to_region(region2_i.into(), &[1; 6000])?;
let (region4_i, region4) = file.create_region_if_needed("region4")?;
file.remove_region(region2_i.into());
file.remove_region(region4_i.into());
let (region4_i, _) = file.create_region_if_needed("region4")?;
file.remove_region(region2_i.into())?;
file.remove_region(region4_i.into())?;
dbg!(file.regions());
dbg!(file.layout());
let regions = file.regions();
dbg!(&regions);
let layout = file.layout();
dbg!(&layout);
Ok(())
}

24
crates/brk_vecs/src/file/layout.rs
View File

@@ -14,9 +14,8 @@ pub struct Layout {
impl From<&Regions> for Layout {
fn from(value: &Regions) -> Self {
let mut start_to_index = BTreeMap::new();
let mut start_to_hole = BTreeMap::new();
let mut prev_end = 0;
let index_to_region = value.index_to_region();
value
.index_to_region()
@@ -27,13 +26,20 @@ impl From<&Regions> for Layout {
let region = region.read();
let start = region.start();
start_to_index.insert(start, index);
if prev_end != start {
start_to_hole.insert(prev_end, start - prev_end);
}
let reserved = region.reserved();
prev_end = start + reserved;
});
let mut start_to_hole = BTreeMap::new();
let mut prev_end = 0;
start_to_index.iter().for_each(|(&start, &index)| {
if prev_end != start {
start_to_hole.insert(prev_end, start - prev_end);
}
let reserved = index_to_region[index].as_ref().unwrap().read().reserved();
prev_end = start + reserved;
});
Self {
start_to_index,
start_to_hole,
@@ -64,13 +70,13 @@ impl Layout {
let last = self.get_last_region();
let is_last = last.is_some_and(|(_, other_index)| index == other_index);
if is_last {
debug_assert!(self.start_to_hole.range(last.unwrap().0..).next().is_none());
assert!(self.start_to_hole.range(last.unwrap().0..).next().is_none());
}
is_last
}
pub fn insert_region(&mut self, start: u64, index: usize) {
debug_assert!(self.start_to_index.insert(start, index).is_none())
assert!(self.start_to_index.insert(start, index).is_none())
// TODO: Other checks related to holes ?
}

288
crates/brk_vecs/src/file/mod.rs
View File

@@ -18,6 +18,7 @@ mod regions;
pub use identifier::*;
use layout::*;
use rayon::prelude::*;
pub use reader::*;
pub use region::*;
use regions::*;
@@ -39,6 +40,7 @@ impl File {
fs::create_dir_all(path)?;
let regions = Regions::open(path)?;
let layout = Layout::from(&regions);
let file = OpenOptions::new()
@@ -159,91 +161,113 @@ impl File {
at: Option<u64>,
) -> Result<()> {
let regions = self.regions.read();
let Some(region) = regions.get_region(identifier.clone()) else {
let Some(region_lock) = regions.get_region(identifier.clone()) else {
return Err(Error::Str("Unknown region"));
};
let region_index = regions.identifier_to_index(identifier).unwrap();
drop(regions);
let region_lock = region.read();
let start = region_lock.start();
let reserved = region_lock.reserved();
let len = region_lock.len();
let region = region_lock.read();
let start = region.start();
let reserved = region.reserved();
let len = region.len();
drop(region);
let data_len = data.len() as u64;
drop(region_lock);
let new_len = at.map_or(len + data_len, |at| (at + data_len).max(len));
// dbg!(new_len);
let write_start = at.unwrap_or(start + len);
if at.is_some_and(|at| at < start || at >= start + reserved) {
if at.is_some_and(|at| at >= start + reserved) {
return Err(Error::Str("Invalid at parameter"));
}
// Write to reserved space if possible
if new_len <= reserved {
// dbh!("Write to reserved space");
// dbg!(write_start);
// println!(
// "Write to {region_index} reserved space at {}",
// start + at.unwrap_or(len)
// );
self.write(write_start, data);
let regions = self.regions.read();
let mut region_lock = region.write();
if len != new_len {
region_lock.set_len(new_len);
if at.is_none() {
self.write(start + len, data);
}
regions.write_to_mmap(&region_lock, region_index);
let mut region = region_lock.write();
if let Some(at) = at {
self.write(start + at, data);
}
if len != new_len {
region.set_len(new_len);
}
regions.write_to_mmap(&region, region_index);
return Ok(());
}
let mut layout_lock = self.layout.write();
// let layouat_lock = self.layout.read();
debug_assert!(new_len > reserved);
assert!(new_len > reserved);
let mut new_reserved = reserved;
while new_len > new_reserved {
new_reserved *= 2;
}
debug_assert!(new_len <= new_reserved);
assert!(new_len <= new_reserved);
let added_reserve = new_reserved - reserved;
let mut layout = self.layout.write();
// If is last continue writing
if layout_lock.is_last_region(region_index) {
// dbg!("Append to file");
// dbg!(start, new_reserved, start + new_reserved);
if layout.is_last_region(region_index) {
// println!(
// "{region_index} Append to file at {}",
// start + at.unwrap_or(len)
// );
self.set_min_len(start + new_reserved)?;
self.write(write_start, data);
if at.is_none() {
self.write(start + len, data);
}
let mut region = region_lock.write();
region.set_reserved(new_reserved);
if let Some(at) = at {
self.write(start + at, data);
}
region.set_len(new_len);
regions.write_to_mmap(&region, region_index);
let regions = self.regions.read();
let mut region_lock = region.write();
region_lock.set_reserved(new_reserved);
region_lock.set_len(new_len);
regions.write_to_mmap(&region_lock, region_index);
return Ok(());
}
// Expand region to the right if gap is wide enough
let hole_start = start + reserved;
let gap = layout_lock.get_hole(hole_start);
if gap.is_some_and(|gap| gap >= added_reserve) {
// dbg!("Expand to hole");
if layout
.get_hole(hole_start)
.is_some_and(|gap| gap >= added_reserve)
{
// println!("Expand {region_index} to hole");
self.write(write_start, data);
layout.remove_or_compress_hole_to_right(hole_start, added_reserve);
drop(layout);
layout_lock.remove_or_compress_hole_to_right(hole_start, added_reserve);
drop(layout_lock);
let regions = self.regions.read();
let mut region_lock = region.write();
region_lock.set_reserved(new_reserved);
region_lock.set_len(new_len);
regions.write_to_mmap(&region_lock, region_index);
if at.is_none() {
self.write(start + len, data);
}
let mut region = region_lock.write();
region.set_reserved(new_reserved);
if let Some(at) = at {
self.write(start + at, data);
}
region.set_len(new_len);
regions.write_to_mmap(&region, region_index);
return Ok(());
}
// Find hole big enough to move the region
if let Some(hole_start) = layout_lock.find_smallest_adequate_hole(new_reserved) {
// dbg!("Move to hole");
if let Some(hole_start) = layout.find_smallest_adequate_hole(new_reserved) {
// println!("Move {region_index} to hole at {hole_start}");
layout.remove_or_compress_hole_to_right(hole_start, new_reserved);
// drop(layout);
self.write(
hole_start,
@@ -251,37 +275,40 @@ impl File {
);
self.write(hole_start + len, data);
let regions = self.regions.read();
let mut region_lock = region.write();
// let mut layout = self.layout.write();
let mut region = region_lock.write();
layout.move_region(hole_start, region_index, &region)?;
region.set_start(hole_start);
region.set_reserved(new_reserved);
region.set_len(new_len);
layout_lock.remove_or_compress_hole_to_right(hole_start, new_reserved);
layout_lock.move_region(hole_start, region_index, &region_lock)?;
drop(layout);
region_lock.set_start(hole_start);
region_lock.set_reserved(new_reserved);
region_lock.set_len(new_len);
regions.write_to_mmap(&region_lock, region_index);
drop(layout_lock);
self.punch_hole(start, reserved)?;
regions.write_to_mmap(&region, region_index);
return Ok(());
}
// Write at the end
// dbg!("Move and write at the end");
let regions = self.regions.read();
let mut region_lock = region.write();
let (last_region_start, last_region_index) = layout_lock.get_last_region().unwrap();
// let mut region_lock = region.write();
let (last_region_start, last_region_index) = layout.get_last_region().unwrap();
let new_start = last_region_start
+ regions
.get_region_from_index(last_region_index)
.unwrap()
.read()
.reserved();
// println!(
// "Move {region_index} to the end, from {start}..{} to {new_start}..{}",
// start + reserved,
// new_start + new_reserved
// );
self.set_min_len(new_start + new_reserved)?;
let mut region = region_lock.write();
layout.move_region(new_start, region_index, &region)?;
self.write(
new_start,
&self.mmap.read()[start as usize..(start + len) as usize],
@@ -290,23 +317,22 @@ impl File {
// dbg!(new_start, region_index, &region_lock, new_reserved, new_len);
layout_lock.move_region(new_start, region_index, &region_lock)?;
region.set_start(new_start);
region.set_reserved(new_reserved);
region.set_len(new_len);
region_lock.set_start(new_start);
region_lock.set_reserved(new_reserved);
region_lock.set_len(new_len);
regions.write_to_mmap(&region_lock, region_index);
drop(layout);
self.punch_hole(start, reserved)?;
regions.write_to_mmap(&region, region_index);
Ok(())
}
fn write(&self, start: u64, data: &[u8]) {
fn write(&self, at: u64, data: &[u8]) {
let mmap = self.mmap.read();
let data_len = data.len();
let start = start as usize;
let start = at as usize;
let end = start + data_len;
if end > mmap.len() {
@@ -376,11 +402,85 @@ impl File {
Ok(unsafe { MmapOptions::new().map_mut(file)? })
}
pub fn regions(&self) -> RwLockReadGuard<'_, Regions> {
self.regions.read()
}
pub fn layout(&self) -> RwLockReadGuard<'_, Layout> {
self.layout.read()
}
pub fn mmap(&self) -> RwLockReadGuard<'_, MmapMut> {
self.mmap.read()
}
fn ceil_number_to_page_size_multiple(num: u64) -> u64 {
(num + PAGE_SIZE_MINUS_1) & !PAGE_SIZE_MINUS_1
}
fn data_path(&self) -> PathBuf {
Self::data_path_(&self.path)
}
fn data_path_(path: &Path) -> PathBuf {
path.join("data")
}
pub fn disk_usage(&self) -> String {
let path = self.data_path();
let output = std::process::Command::new("du")
.arg("-h")
.arg(&path)
.output()
.expect("Failed to run du");
String::from_utf8_lossy(&output.stdout)
.replace(path.to_str().unwrap(), " ")
.trim()
.to_string()
}
pub fn flush(&self) -> Result<()> {
let mmap = self.mmap.read();
let regions = self.regions.read();
mmap.flush()?;
regions.flush()
}
pub fn punch_holes(&self) -> Result<()> {
let file = self.file.write();
let mmap = self.mmap.read();
let layout = self.layout.read();
Self::punch_holes_(&file, &mmap, &layout)
}
fn punch_holes_(file: &fs::File, mmap: &MmapMut, layout: &Layout) -> Result<()> {
layout
.start_to_hole()
.par_iter()
.try_for_each(|(&start, &hole)| -> Result<()> {
assert!(start % PAGE_SIZE == 0);
assert!(hole % PAGE_SIZE == 0);
let has_old_data = (((start / PAGE_SIZE) as usize)
..((start + hole) / PAGE_SIZE) as usize)
.any(|i| mmap[i * PAGE_SIZE as usize] != 0);
if has_old_data {
Self::punch_hole_(file, start, hole)
} else {
Ok(())
}
})
}
fn punch_hole(&self, start: u64, length: u64) -> Result<()> {
let file = self.file.write();
Self::punch_hole_macos(&file, start, length)
}
fn punch_hole_(file: &fs::File, start: u64, length: u64) -> Result<()> {
Self::punch_hole_macos(file, start, length)
}
#[cfg(target_os = "macos")]
fn punch_hole_macos(file: &fs::File, start: u64, length: u64) -> Result<()> {
let fpunchhole = FPunchhole {
@@ -405,56 +505,6 @@ impl File {
Ok(())
}
pub fn regions(&self) -> RwLockReadGuard<'_, Regions> {
self.regions.read()
}
pub fn layout(&self) -> RwLockReadGuard<'_, Layout> {
self.layout.read()
}
pub fn mmap(&self) -> RwLockReadGuard<'_, MmapMut> {
self.mmap.read()
}
fn ceil_number_to_page_size_multiple(num: u64) -> u64 {
(num + PAGE_SIZE_MINUS_1) & !PAGE_SIZE_MINUS_1
}
fn data_path(&self) -> PathBuf {
Self::data_path_(&self.path)
}
fn data_path_(path: &Path) -> PathBuf {
path.join("data")
}
pub fn flush(&self) -> Result<()> {
self.mmap.write().flush().map_err(|e| e.into())
}
pub fn sync_data(&self) -> Result<()> {
self.file.read().sync_data().map_err(|e| e.into())
}
pub fn sync_all(&self) -> Result<()> {
self.file.read().sync_all().map_err(|e| e.into())
}
pub fn disk_usage(&self) -> String {
let path = self.data_path();
let output = std::process::Command::new("du")
.arg("-h")
.arg(&path)
.output()
.expect("Failed to run du");
String::from_utf8_lossy(&output.stdout)
.replace(path.to_str().unwrap(), " ")
.trim()
.to_string()
}
}
#[repr(C)]

2
crates/brk_vecs/src/file/reader.rs
View File

@@ -18,7 +18,7 @@ impl<'a> Reader<'a> {
}
pub fn read(&self, offset: u64, len: u64) -> &[u8] {
debug_assert!(offset + len <= self.region.len());
assert!(offset + len <= self.region.len());
let start = self.region.start() + offset;
let end = start + len;
&self.mmap[start as usize..end as usize]

20
crates/brk_vecs/src/file/region.rs
View File

@@ -18,10 +18,10 @@ pub const SIZE_OF_REGION: usize = size_of::<Region>();
impl Region {
pub fn new(start: u64, len: u64, reserved: u64) -> Self {
debug_assert!(start % PAGE_SIZE == 0);
debug_assert!(reserved >= PAGE_SIZE);
debug_assert!(reserved % PAGE_SIZE == 0);
debug_assert!(len <= reserved);
assert!(start % PAGE_SIZE == 0);
assert!(reserved >= PAGE_SIZE);
assert!(reserved % PAGE_SIZE == 0);
assert!(len <= reserved);
Self {
start,
@@ -35,7 +35,7 @@ impl Region {
}
pub fn set_start(&mut self, start: u64) {
debug_assert!(start % PAGE_SIZE == 0);
assert!(start % PAGE_SIZE == 0);
self.start = start
}
@@ -44,7 +44,7 @@ impl Region {
}
pub fn set_len(&mut self, len: u64) {
debug_assert!(len <= self.reserved());
assert!(len <= self.reserved());
self.len = len
}
@@ -53,9 +53,9 @@ impl Region {
}
pub fn set_reserved(&mut self, reserved: u64) {
debug_assert!(self.len() <= reserved);
debug_assert!(reserved >= PAGE_SIZE);
debug_assert!(reserved % PAGE_SIZE == 0);
assert!(self.len() <= reserved);
assert!(reserved >= PAGE_SIZE);
assert!(reserved % PAGE_SIZE == 0);
self.reserved = reserved;
}
@@ -66,7 +66,7 @@ impl Region {
}
pub trait RegionReader {
fn create_reader(self, file: &File) -> Reader<'_>;
fn create_reader(self, file: &File) -> Reader;
}
impl<'a> RegionReader for RwLockReadGuard<'a, Region> {

98
crates/brk_vecs/src/file/regions.rs
View File

@@ -1,15 +1,14 @@
use std::{
collections::HashMap,
fs::{self, OpenOptions},
io::{BufReader, BufWriter},
path::Path,
io::{Cursor, Read},
path::{Path, PathBuf},
sync::Arc,
};
use bincode::{decode_from_std_read, encode_into_std_write};
use brk_core::{Error, Result};
use memmap2::MmapMut;
use parking_lot::RwLock;
use parking_lot::{RwLock, RwLockWriteGuard};
use zerocopy::{FromBytes, IntoBytes};
use super::{
@@ -20,7 +19,7 @@ use super::{
#[derive(Debug)]
pub struct Regions {
id_to_index: HashMap<String, usize>,
id_to_index_file: fs::File,
id_to_index_path: PathBuf,
index_to_region: Vec<Option<Arc<RwLock<Region>>>>,
index_to_region_file: fs::File,
index_to_region_mmap: MmapMut,
@@ -32,19 +31,11 @@ impl Regions {
fs::create_dir_all(&path)?;
let id_to_index_file = OpenOptions::new()
.read(true)
.create(true)
.write(true)
.truncate(false)
.open(path.join("id_to_index"))?;
let id_to_index_path = path.join("id_to_index");
let mut id_to_index: HashMap<String, usize> = HashMap::new();
if id_to_index_file.metadata()?.len() > 0 {
let mut reader = BufReader::new(&id_to_index_file);
id_to_index = decode_from_std_read(&mut reader, bincode::config::standard())?;
}
let id_to_index: HashMap<String, usize> =
deserialize_hashmap_binary(&fs::read(&id_to_index_path).unwrap_or_default())
.unwrap_or_default();
let index_to_region_file = OpenOptions::new()
.read(true)
@@ -77,7 +68,7 @@ impl Regions {
Ok(Self {
id_to_index,
id_to_index_file,
id_to_index_path,
index_to_region,
index_to_region_file,
index_to_region_mmap,
@@ -109,15 +100,18 @@ impl Regions {
self.set_min_len(((index + 1) * SIZE_OF_REGION) as u64)?;
self.write_to_mmap(&region, index);
let region_lock = RwLock::new(region);
let region_arc = Arc::new(RwLock::new(region));
self.write_to_mmap(&region_lock.write(), index);
let region_arc = Arc::new(region_lock);
let region_opt = Some(region_arc.clone());
if index < self.index_to_region.len() {
self.index_to_region[index] = region_opt
} else {
self.index_to_region.push(region_opt);
self.index_to_region_mmap.flush()?;
}
if self.id_to_index.insert(id, index).is_some() {
@@ -129,8 +123,10 @@ impl Regions {
}
fn flush_id_to_index(&mut self) -> Result<()> {
let mut writer = BufWriter::new(&mut self.id_to_index_file);
encode_into_std_write(&self.id_to_index, &mut writer, bincode::config::standard())?;
fs::write(
&self.id_to_index_path,
serialize_hashmap_binary(&self.id_to_index),
)?;
Ok(())
}
@@ -208,6 +204,8 @@ impl Regions {
return Ok(None);
};
self.index_to_region_mmap.flush()?;
self.id_to_index
.remove(&self.find_id_from_index(index).unwrap().to_owned());
@@ -216,10 +214,10 @@ impl Regions {
Ok(Some(region))
}
pub fn write_to_mmap(&self, region: &Region, index: usize) {
pub fn write_to_mmap(&self, region: &RwLockWriteGuard<Region>, index: usize) {
let mmap = &self.index_to_region_mmap;
let start = index * SIZE_OF_REGION;
let end = start + SIZE_OF_REGION;
let mmap = &self.index_to_region_mmap;
if end > mmap.len() {
unreachable!("Trying to write beyond mmap")
@@ -229,4 +227,56 @@ impl Regions {
slice[start..end].copy_from_slice(region.as_bytes());
}
pub fn flush(&self) -> Result<()> {
self.index_to_region_mmap.flush().map_err(|e| e.into())
}
}
fn serialize_hashmap_binary(map: &HashMap<String, usize>) -> Vec<u8> {
let mut buffer = Vec::new();
buffer.extend_from_slice(&map.len().to_ne_bytes());
for (key, value) in map {
buffer.extend_from_slice(&key.len().to_ne_bytes());
buffer.extend_from_slice(key.as_bytes());
buffer.extend_from_slice(&value.to_ne_bytes());
}
buffer
}
fn deserialize_hashmap_binary(data: &[u8]) -> Result<HashMap<String, usize>> {
let mut cursor = Cursor::new(data);
let mut buffer = [0u8; 8];
cursor
.read_exact(&mut buffer)
.map_err(|_| Error::Str("Failed to read entry count"))?;
let entry_count = usize::from_ne_bytes(buffer);
let mut map = HashMap::with_capacity(entry_count);
for _ in 0..entry_count {
cursor
.read_exact(&mut buffer)
.map_err(|_| Error::Str("Failed to read key length"))?;
let key_len = usize::from_ne_bytes(buffer);
let mut key_bytes = vec![0u8; key_len];
cursor
.read_exact(&mut key_bytes)
.map_err(|_| Error::Str("Failed to read key"))?;
let key = String::from_utf8(key_bytes).map_err(|_| Error::Str("Invalid UTF-8 in key"))?;
cursor
.read_exact(&mut buffer)
.map_err(|_| Error::Str("Failed to read value"))?;
let value = usize::from_ne_bytes(buffer);
map.insert(key, value);
}
Ok(map)
}

6
crates/brk_vecs/src/lib.rs
View File

@@ -6,4 +6,8 @@ use variants::*;
pub use file::{File, PAGE_SIZE, Reader};
pub use traits::*;
pub use variants::{AnyStampedVec, Format, RawVec, Stamp, StampedVec};
pub use variants::{
AnyStampedVec, CompressedVec, Computation, ComputedVec, ComputedVecFrom1, ComputedVecFrom2,
ComputedVecFrom3, EagerVec, Format, LazyVecFrom1, LazyVecFrom2, LazyVecFrom3, RawVec, Stamp,
StampedVec, StoredVec,
};

26
crates/brk_vecs/src/traits/any.rs
View File

@@ -1,7 +1,5 @@
use brk_core::{Height, Version};
use crate::{File, Reader};
use super::{BoxedVecIterator, StoredIndex, StoredType};
pub fn i64_to_usize(i: i64, len: usize) -> usize {
@@ -39,30 +37,6 @@ pub trait AnyVec: Send + Sync {
)
}
fn file(&self) -> &File;
fn region_index(&self) -> usize;
/// Be careful with deadlocks
///
/// You'll want to drop the reader before mutable ops
fn create_reader(&self) -> Reader<'_> {
self.create_static_reader()
}
/// Be careful with deadlocks
///
/// You'll want to drop the reader before mutable ops
fn create_static_reader(&self) -> Reader<'static> {
unsafe {
std::mem::transmute(
self.file()
.create_region_reader(self.region_index().into())
.unwrap(),
)
}
}
#[inline]
fn i64_to_usize(&self, i: i64) -> usize {
let len = self.len();

40
crates/brk_vecs/src/traits/generic.rs
View File

@@ -6,7 +6,7 @@ use std::{
use brk_core::{Error, Result};
use crate::{AnyVec, HEADER_OFFSET, Header, file::Reader};
use crate::{AnyVec, File, HEADER_OFFSET, Header, file::Reader};
use super::{StoredIndex, StoredType};
@@ -18,22 +18,46 @@ where
{
const SIZE_OF_T: usize = size_of::<T>();
fn file(&self) -> &File;
fn region_index(&self) -> usize;
/// Be careful with deadlocks
///
/// You'll want to drop the reader before mutable ops
fn create_reader(&self) -> Reader {
self.create_static_reader()
}
/// Be careful with deadlocks
///
/// You'll want to drop the reader before mutable ops
fn create_static_reader(&self) -> Reader<'static> {
unsafe {
std::mem::transmute(
self.file()
.create_region_reader(self.region_index().into())
.unwrap(),
)
}
}
#[inline]
fn unwrap_read(&self, index: I, reader: &Reader<'_>) -> T {
fn unwrap_read(&self, index: I, reader: &Reader) -> T {
self.read(index, reader).unwrap().unwrap()
}
#[inline]
fn read(&self, index: I, reader: &Reader<'_>) -> Result<Option<T>> {
fn read(&self, index: I, reader: &Reader) -> Result<Option<T>> {
self.read_(index.to_usize()?, reader)
}
fn read_(&self, index: usize, reader: &Reader<'_>) -> Result<Option<T>>;
fn read_(&self, index: usize, reader: &Reader) -> Result<Option<T>>;
#[inline]
fn get_or_read(&self, index: I, reader: &Reader<'_>) -> Result<Option<Cow<T>>> {
fn get_or_read(&self, index: I, reader: &Reader) -> Result<Option<Cow<T>>> {
self.get_or_read_(index.to_usize()?, reader)
}
#[inline]
fn get_or_read_(&self, index: usize, reader: &Reader<'_>) -> Result<Option<Cow<T>>> {
fn get_or_read_(&self, index: usize, reader: &Reader) -> Result<Option<Cow<T>>> {
let stored_len = self.stored_len();
if index >= stored_len {
@@ -117,7 +141,7 @@ where
fn holes(&self) -> &BTreeSet<usize>;
fn mut_holes(&mut self) -> &mut BTreeSet<usize>;
fn take(&mut self, index: I, reader: &Reader<'_>) -> Result<Option<T>> {
fn take(&mut self, index: I, reader: &Reader) -> Result<Option<T>> {
let opt = self.get_or_read(index, reader)?.map(|v| v.into_owned());
if opt.is_some() {
self.unchecked_delete(index);
@@ -205,7 +229,7 @@ where
Self::vec_region_name_(self.name())
}
fn vec_region_name_(name: &str) -> String {
format!("{name}_{}", I::to_string())
format!("{}_to_{name}", I::to_string())
}
fn holes_region_name(&self) -> String {

22
crates/brk_vecs/src/variants/compressed/mod.rs
View File

@@ -92,7 +92,7 @@ where
// })
}
fn decode_page(&self, page_index: usize, reader: &Reader<'_>) -> Result<Vec<T>> {
fn decode_page(&self, page_index: usize, reader: &Reader) -> Result<Vec<T>> {
Self::decode_page_(
self.stored_len(),
page_index,
@@ -104,7 +104,7 @@ where
fn decode_page_(
stored_len: usize,
page_index: usize,
reader: &Reader<'_>,
reader: &Reader,
compressed_pages_meta: &CompressedPagesMetadata,
) -> Result<Vec<T>> {
if Self::page_index_to_index(page_index) >= stored_len {
@@ -178,8 +178,16 @@ where
I: StoredIndex,
T: StoredType,
{
fn file(&self) -> &File {
self.inner.file()
}
fn region_index(&self) -> usize {
self.inner.region_index()
}
#[inline]
fn read_(&self, index: usize, reader: &Reader<'_>) -> Result<Option<T>> {
fn read_(&self, index: usize, reader: &Reader) -> Result<Option<T>> {
let page_index = Self::index_to_page_index(index);
let decoded_index = index % Self::PER_PAGE;
@@ -398,14 +406,6 @@ where
fn value_type_to_size_of(&self) -> usize {
size_of::<T>()
}
fn file(&self) -> &File {
self.inner.file()
}
fn region_index(&self) -> usize {
self.inner.region_index()
}
}
impl<I, T> Clone for CompressedVec<I, T> {

6
crates/brk_vecs/src/variants/mod.rs
View File

@@ -1,9 +1,15 @@
mod compressed;
mod computed;
mod eager;
mod lazy;
mod raw;
mod stamped;
mod stored;
pub use compressed::*;
pub use computed::*;
pub use eager::*;
pub use lazy::*;
pub use raw::*;
pub use stamped::*;
pub use stored::*;

27
crates/brk_vecs/src/variants/raw/mod.rs
View File

@@ -7,12 +7,13 @@ use std::{
};
use brk_core::{Error, Result, Version};
use parking_lot::RwLock;
use rayon::prelude::*;
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec,
File, GenericStoredVec, StoredIndex, StoredType,
file::{Reader, RegionReader},
File, GenericStoredVec, Reader, StoredIndex, StoredType,
file::{Region, RegionReader},
};
use super::Format;
@@ -28,6 +29,7 @@ const VERSION: Version = Version::ONE;
#[derive(Debug)]
pub struct RawVec<I, T> {
file: Arc<File>,
region: Arc<RwLock<Region>>,
region_index: usize,
header: Header,
@@ -68,7 +70,6 @@ where
if region_len > 0
&& (region_len < HEADER_OFFSET || (region_len - HEADER_OFFSET) % Self::SIZE_OF_T != 0)
{
dbg!(region_len);
return Err(Error::Str("Region was saved incorrectly"));
}
@@ -101,6 +102,7 @@ where
Ok(Self {
file: file.clone(),
region: region.clone(),
region_index,
header,
name: Box::leak(Box::new(name.to_string())),
@@ -143,7 +145,7 @@ where
T: StoredType,
{
#[inline]
fn read_(&self, index: usize, reader: &Reader<'_>) -> Result<Option<T>> {
fn read_(&self, index: usize, reader: &Reader) -> Result<Option<T>> {
let slice = reader.read(
(index * Self::SIZE_OF_T + HEADER_OFFSET) as u64,
(Self::SIZE_OF_T) as u64,
@@ -289,6 +291,14 @@ where
fn reset(&mut self) -> Result<()> {
self.reset_()
}
fn file(&self) -> &File {
&self.file
}
fn region_index(&self) -> usize {
self.region_index
}
}
impl<I, T> AnyVec for RawVec<I, T>
@@ -320,20 +330,13 @@ where
fn value_type_to_size_of(&self) -> usize {
size_of::<T>()
}
fn file(&self) -> &File {
&self.file
}
fn region_index(&self) -> usize {
self.region_index
}
}
impl<I, T> Clone for RawVec<I, T> {
fn clone(&self) -> Self {
Self {
file: self.file.clone(),
region: self.region.clone(),
region_index: self.region_index,
header: self.header.clone(),
name: self.name,

26
crates/brk_vecs/src/variants/stamped/mod.rs
View File

@@ -27,18 +27,16 @@ where
version: Version,
format: Format,
) -> Result<Self> {
Ok(Self(
StoredVec::forced_import(file, name, version, format).unwrap(),
))
Ok(Self(StoredVec::forced_import(file, name, version, format)?))
}
#[inline]
pub fn unwrap_read(&self, index: I, reader: &Reader<'_>) -> T {
pub fn unwrap_read(&self, index: I, reader: &Reader) -> T {
self.0.unwrap_read(index, reader)
}
#[inline]
pub fn get_or_read(&self, index: I, reader: &Reader<'_>) -> Result<Option<Cow<T>>> {
pub fn get_or_read(&self, index: I, reader: &Reader) -> Result<Option<Cow<T>>> {
self.0.get_or_read(index, reader)
}
@@ -95,7 +93,7 @@ where
self.0.update(index, value)
}
pub fn take(&mut self, index: I, reader: &Reader<'_>) -> Result<Option<T>> {
pub fn take(&mut self, index: I, reader: &Reader) -> Result<Option<T>> {
self.0.take(index, reader)
}
@@ -131,6 +129,14 @@ where
pub fn hasnt(&self, index: I) -> Result<bool> {
self.0.has(index).map(|b| !b)
}
pub fn create_reader(&self) -> Reader {
self.0.create_reader()
}
pub fn create_static_reader(&self) -> Reader<'static> {
self.0.create_static_reader()
}
}
impl<I, T> AnyVec for StampedVec<I, T>
@@ -162,14 +168,6 @@ where
fn value_type_to_size_of(&self) -> usize {
size_of::<T>()
}
fn file(&self) -> &File {
self.0.file()
}
fn region_index(&self) -> usize {
self.0.region_index()
}
}
pub trait AnyStampedVec: AnyVec {

4
crates/brk_vecs/src/variants/stamped/stamp.rs
View File

@@ -1,6 +1,8 @@
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Default, Clone, Copy, FromBytes, IntoBytes, Immutable, KnownLayout)]
#[derive(
Debug, Default, Clone, Copy, PartialEq, Eq, FromBytes, IntoBytes, Immutable, KnownLayout,
)]
pub struct Stamp(u64);
impl Stamp {

5
crates/brk_vecs/src/variants/stored/format.rs
View File

@@ -1,9 +1,12 @@
use std::{fs, io, path::Path};
use brk_core::{Error, Result};
use clap_derive::ValueEnum;
use serde::{Deserialize, Serialize};
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[derive(
Default, Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, ValueEnum,
)]
pub enum Format {
Compressed,
#[default]

41
crates/brk_vecs/src/variants/stored/mod.rs
View File

@@ -40,9 +40,10 @@ where
}
if format.is_compressed() {
Ok(Self::Compressed(CompressedVec::forced_import(
file, name, version,
)?))
todo!();
// Ok(Self::Compressed(CompressedVec::forced_import(
// file, name, version,
// )?))
} else {
Ok(Self::Raw(RawVec::forced_import(file, name, version)?))
}
@@ -55,7 +56,23 @@ where
T: StoredType,
{
#[inline]
fn read_(&self, index: usize, reader: &Reader<'_>) -> Result<Option<T>> {
fn file(&self) -> &File {
match self {
StoredVec::Raw(v) => v.file(),
StoredVec::Compressed(v) => v.file(),
}
}
#[inline]
fn region_index(&self) -> usize {
match self {
StoredVec::Raw(v) => v.region_index(),
StoredVec::Compressed(v) => v.region_index(),
}
}
#[inline]
fn read_(&self, index: usize, reader: &Reader) -> Result<Option<T>> {
match self {
StoredVec::Raw(v) => v.read_(index, reader),
StoredVec::Compressed(v) => v.read_(index, reader),
@@ -187,22 +204,6 @@ where
fn value_type_to_size_of(&self) -> usize {
size_of::<T>()
}
#[inline]
fn file(&self) -> &File {
match self {
StoredVec::Raw(v) => v.file(),
StoredVec::Compressed(v) => v.file(),
}
}
#[inline]
fn region_index(&self) -> usize {
match self {
StoredVec::Raw(v) => v.region_index(),
StoredVec::Compressed(v) => v.region_index(),
}
}
}
#[derive(Debug)]