Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-25 15:19:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: convert brk_vecs to its own crates and repo (seqdb/vecdb) + changes

Browse Source
This commit is contained in:
nym21
2025-08-10 12:49:41 +02:00
parent c85592eefe
commit 5e8c7da4df
168 changed files with 1779 additions and 9538 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
crates/brk_computer/src/blocks.rs
View File

@@ -1,4 +1,4 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
@@ -6,8 +6,8 @@ use brk_structs::{
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, Timestamp, Version,
Weight,
};
use brk_vecs::{
AnyCollectableVec, Computation, EagerVec, Exit, File, Format, PAGE_SIZE, VecIterator,
use vecdb::{
AnyCollectableVec, Computation, Database, EagerVec, Exit, Format, PAGE_SIZE, VecIterator,
};
use crate::grouped::Source;
@@ -22,7 +22,7 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub height_to_interval: EagerVec<Height, Timestamp>,
pub height_to_vbytes: EagerVec<Height, StoredU64>,
@@ -44,18 +44,18 @@ impl Vecs {
format: Format,
indexes: &indexes::Vecs,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("blocks"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
let db = Database::open(&parent.join("blocks"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
height_to_interval: EagerVec::forced_import(
&file,
&db,
"interval",
version + VERSION + Version::ZERO,
format,
)?,
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"timestamp",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -65,7 +65,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(),
)?,
indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
&file,
&db,
"block_interval",
Source::None,
version + VERSION + Version::ZERO,
@@ -78,7 +78,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_block_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"block_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -88,7 +88,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
&file,
&db,
"block_weight",
Source::None,
version + VERSION + Version::ZERO,
@@ -98,7 +98,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_block_size: ComputedVecsFromHeight::forced_import(
&file,
&db,
"block_size",
Source::None,
version + VERSION + Version::ZERO,
@@ -108,13 +108,13 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
height_to_vbytes: EagerVec::forced_import(
&file,
&db,
"vbytes",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
&file,
&db,
"block_vbytes",
Source::None,
version + VERSION + Version::ZERO,
@@ -124,19 +124,19 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
difficultyepoch_to_timestamp: EagerVec::forced_import(
&file,
&db,
"timestamp",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_timestamp: EagerVec::forced_import(
&file,
&db,
"timestamp",
version + VERSION + Version::ZERO,
format,
)?,
file,
db,
})
}
@@ -148,7 +148,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

60
crates/brk_computer/src/cointime.rs
View File

@@ -1,9 +1,9 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, CheckedSub, Dollars, StoredF64, Version};
use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, PAGE_SIZE, VecIterator};
use vecdb::{AnyCollectableVec, Computation, Database, Exit, Format, PAGE_SIZE, VecIterator};
use super::{
Indexes,
@@ -18,7 +18,7 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub indexes_to_coinblocks_created: ComputedVecsFromHeight<StoredF64>,
pub indexes_to_coinblocks_stored: ComputedVecsFromHeight<StoredF64>,
@@ -55,14 +55,14 @@ impl Vecs {
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("cointime"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
let db = Database::open(&parent.join("cointime"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
let compute_dollars = price.is_some();
Ok(Self {
indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import(
&file,
&db,
"coinblocks_created",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -72,7 +72,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import(
&file,
&db,
"coinblocks_stored",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -82,7 +82,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_liveliness: ComputedVecsFromHeight::forced_import(
&file,
&db,
"liveliness",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -92,7 +92,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaultedness: ComputedVecsFromHeight::forced_import(
&file,
&db,
"vaultedness",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -102,7 +102,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import(
&file,
&db,
"activity_to_vaultedness_ratio",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -112,7 +112,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"vaulted_supply",
Source::Compute,
version + VERSION + Version::ONE,
@@ -123,7 +123,7 @@ impl Vecs {
indexes,
)?,
indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"active_supply",
Source::Compute,
version + VERSION + Version::ONE,
@@ -134,7 +134,7 @@ impl Vecs {
indexes,
)?,
indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import(
&file,
&db,
"thermo_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -144,7 +144,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_investor_cap: ComputedVecsFromHeight::forced_import(
&file,
&db,
"investor_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -154,7 +154,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import(
&file,
&db,
"vaulted_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -164,7 +164,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_active_cap: ComputedVecsFromHeight::forced_import(
&file,
&db,
"active_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -174,7 +174,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import(
&file,
&db,
"vaulted_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -184,7 +184,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
&file,
&db,
"vaulted_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -194,7 +194,7 @@ impl Vecs {
true,
)?,
indexes_to_active_price: ComputedVecsFromHeight::forced_import(
&file,
&db,
"active_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -204,7 +204,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
&file,
&db,
"active_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -214,7 +214,7 @@ impl Vecs {
true,
)?,
indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import(
&file,
&db,
"true_market_mean",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -224,7 +224,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import(
&file,
&db,
"true_market_mean",
Source::None,
version + VERSION + Version::ZERO,
@@ -234,7 +234,7 @@ impl Vecs {
true,
)?,
indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import(
&file,
&db,
"cointime_value_destroyed",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -244,7 +244,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import(
&file,
&db,
"cointime_value_created",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -254,7 +254,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import(
&file,
&db,
"cointime_value_stored",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -264,7 +264,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_price: ComputedVecsFromHeight::forced_import(
&file,
&db,
"cointime_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -274,7 +274,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import(
&file,
&db,
"cointime_cap",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -284,7 +284,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
&file,
&db,
"cointime_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -294,7 +294,7 @@ impl Vecs {
true,
)?,
file,
db,
})
}
@@ -318,7 +318,7 @@ impl Vecs {
stateful,
exit,
)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

20
crates/brk_computer/src/constants.rs
View File

@@ -1,9 +1,9 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{StoredU16, Version};
use brk_vecs::{AnyCollectableVec, AnyVec, Computation, Exit, File, Format};
use vecdb::{AnyCollectableVec, AnyVec, Computation, Database, Exit, Format};
use crate::grouped::Source;
@@ -17,7 +17,7 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub constant_0: ComputedVecsFromHeight<StoredU16>,
pub constant_1: ComputedVecsFromHeight<StoredU16>,
@@ -33,11 +33,11 @@ impl Vecs {
format: Format,
indexes: &indexes::Vecs,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("constants"))?);
let db = Database::open(&parent.join("constants"))?;
Ok(Self {
constant_0: ComputedVecsFromHeight::forced_import(
&file,
&db,
"constant_0",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -47,7 +47,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_1: ComputedVecsFromHeight::forced_import(
&file,
&db,
"constant_1",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -57,7 +57,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_50: ComputedVecsFromHeight::forced_import(
&file,
&db,
"constant_50",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -67,7 +67,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_100: ComputedVecsFromHeight::forced_import(
&file,
&db,
"constant_100",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -77,7 +77,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
file,
db,
})
}
@@ -89,7 +89,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

20
crates/brk_computer/src/fetched.rs
View File

@@ -1,19 +1,19 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_structs::{DateIndex, Height, OHLCCents, Version};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Exit, File, GenericStoredVec, RawVec,
StoredIndex, VecIterator,
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec,
RawVec, StoredIndex, VecIterator,
};
use super::{Indexes, indexes};
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
fetcher: Fetcher,
pub dateindex_to_ohlc_in_cents: RawVec<DateIndex, OHLCCents>,
@@ -22,23 +22,23 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(parent: &Path, fetcher: Fetcher, version: Version) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("fetched"))?);
let db = Database::open(&parent.join("fetched"))?;
Ok(Self {
fetcher,
dateindex_to_ohlc_in_cents: RawVec::forced_import(
&file,
&db,
"ohlc_in_cents",
version + Version::ZERO,
)?,
height_to_ohlc_in_cents: RawVec::forced_import(
&file,
&db,
"ohlc_in_cents",
version + Version::ZERO,
)?,
file,
db,
})
}
@@ -50,7 +50,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

24
crates/brk_computer/src/grouped/builder_computed.rs
View File

@@ -1,11 +1,9 @@
use std::sync::Arc;
use brk_error::Result;
use brk_structs::Version;
use brk_vecs::{
use vecdb::{
AnyBoxedIterableVec, AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation,
ComputedVec, ComputedVecFrom2, Exit, File, Format, FromCoarserIndex, StoredIndex,
ComputedVec, ComputedVecFrom2, Database, Exit, Format, FromCoarserIndex, StoredIndex,
};
use crate::grouped::{EagerVecBuilder, VecBuilderOptions};
@@ -40,7 +38,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
version: Version,
format: Format,
@@ -67,7 +65,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
&maybe_suffix("first"),
version + VERSION + Version::ZERO,
format,
@@ -92,7 +90,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -101,7 +99,7 @@ where
source
.as_ref()
.unwrap_or_else(|| {
dbg!(file, name, I::to_string());
dbg!(db, name, I::to_string());
panic!()
})
.clone()
@@ -125,7 +123,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
&maybe_suffix("min"),
version + VERSION + Version::ZERO,
format,
@@ -150,7 +148,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
&maybe_suffix("max"),
version + VERSION + Version::ZERO,
format,
@@ -175,7 +173,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
&maybe_suffix("average"),
version + VERSION + Version::ZERO,
format,
@@ -207,7 +205,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
&(if !options.last && !options.average && !options.min && !options.max {
name.to_string()
} else {
@@ -242,7 +240,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
&suffix("cumulative"),
version + VERSION + Version::ZERO,
format,

102
crates/brk_computer/src/grouped/builder_eager.rs
View File

@@ -1,9 +1,7 @@
use std::sync::Arc;
use brk_error::{Error, Result};
use brk_structs::{CheckedSub, StoredU64, Version};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, EagerVec, Exit, File, Format,
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit, Format,
GenericStoredVec, StoredIndex, StoredRaw,
};
@@ -39,7 +37,7 @@ where
T: ComputedType,
{
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
version: Version,
format: Format,
@@ -61,7 +59,7 @@ where
first: options.first.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("first"),
version + VERSION + Version::ZERO,
format,
@@ -71,13 +69,13 @@ where
}),
last: options.last.then(|| {
Box::new(
EagerVec::forced_import(file, name, version + Version::ZERO, format).unwrap(),
EagerVec::forced_import(db, name, version + Version::ZERO, format).unwrap(),
)
}),
min: options.min.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("min"),
version + VERSION + Version::ZERO,
format,
@@ -88,7 +86,7 @@ where
max: options.max.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("max"),
version + VERSION + Version::ZERO,
format,
@@ -99,7 +97,7 @@ where
median: options.median.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("median"),
version + VERSION + Version::ZERO,
format,
@@ -110,7 +108,7 @@ where
average: options.average.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("average"),
version + VERSION + Version::ZERO,
format,
@@ -121,7 +119,7 @@ where
sum: options.sum.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&(if !options.last && !options.average && !options.min && !options.max {
name.to_string()
} else {
@@ -136,7 +134,7 @@ where
cumulative: options.cumulative.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&suffix("cumulative"),
version + VERSION + Version::ZERO,
format,
@@ -147,7 +145,7 @@ where
_90p: options._90p.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("90p"),
version + VERSION + Version::ZERO,
format,
@@ -158,7 +156,7 @@ where
_75p: options._75p.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("75p"),
version + VERSION + Version::ZERO,
format,
@@ -169,7 +167,7 @@ where
_25p: options._25p.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("25p"),
version + VERSION + Version::ZERO,
format,
@@ -180,7 +178,7 @@ where
_10p: options._10p.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("10p"),
version + VERSION + Version::ZERO,
format,
@@ -203,7 +201,7 @@ where
return Ok(());
};
self.validate_computed_version_or_reset_file(source.version())?;
self.validate_computed_version_or_reset(source.version())?;
let index = self.starting_index(max_from);
@@ -234,7 +232,7 @@ where
where
I2: StoredIndex + StoredRaw + CheckedSub<I2>,
{
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
source.version() + first_indexes.version() + count_indexes.version(),
)?;
@@ -253,10 +251,10 @@ where
first_indexes
.iter_at(index)
.try_for_each(|(i, first_index)| -> Result<()> {
.try_for_each(|(index, first_index)| -> Result<()> {
let first_index = first_index.into_owned();
let count_index = count_indexes_iter.unwrap_get_inner(i);
let count_index = count_indexes_iter.unwrap_get_inner(index);
if let Some(first) = self.first.as_mut() {
let f = source_iter
@@ -305,7 +303,7 @@ where
if let Some(max) = self.max.as_mut() {
max.forced_push_at(
i,
index,
*values
.last()
.ok_or(Error::Str("expect some"))
@@ -327,27 +325,27 @@ where
}
if let Some(_90p) = self._90p.as_mut() {
_90p.forced_push_at(i, get_percentile(&values, 0.90), exit)?;
_90p.forced_push_at(index, get_percentile(&values, 0.90), exit)?;
}
if let Some(_75p) = self._75p.as_mut() {
_75p.forced_push_at(i, get_percentile(&values, 0.75), exit)?;
_75p.forced_push_at(index, get_percentile(&values, 0.75), exit)?;
}
if let Some(median) = self.median.as_mut() {
median.forced_push_at(i, get_percentile(&values, 0.50), exit)?;
median.forced_push_at(index, get_percentile(&values, 0.50), exit)?;
}
if let Some(_25p) = self._25p.as_mut() {
_25p.forced_push_at(i, get_percentile(&values, 0.25), exit)?;
_25p.forced_push_at(index, get_percentile(&values, 0.25), exit)?;
}
if let Some(_10p) = self._10p.as_mut() {
_10p.forced_push_at(i, get_percentile(&values, 0.10), exit)?;
_10p.forced_push_at(index, get_percentile(&values, 0.10), exit)?;
}
if let Some(min) = self.min.as_mut() {
min.forced_push_at(i, *values.first().unwrap(), exit)?;
min.forced_push_at(index, *values.first().unwrap(), exit)?;
}
}
@@ -357,18 +355,18 @@ where
if let Some(average) = self.average.as_mut() {
let avg = sum / len;
average.forced_push_at(i, avg, exit)?;
average.forced_push_at(index, avg, exit)?;
}
if needs_sum_or_cumulative {
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum, exit)?;
sum_vec.forced_push_at(index, sum, exit)?;
}
if let Some(cumulative_vec) = self.cumulative.as_mut() {
let t = cumulative.unwrap() + sum;
cumulative.replace(t);
cumulative_vec.forced_push_at(i, t, exit)?;
cumulative_vec.forced_push_at(index, t, exit)?;
}
}
}
@@ -403,7 +401,7 @@ where
panic!("unsupported");
}
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
VERSION + first_indexes.version() + count_indexes.version(),
)?;
@@ -426,10 +424,10 @@ where
first_indexes
.iter_at(index)
.try_for_each(|(i, first_index, ..)| -> Result<()> {
.try_for_each(|(index, first_index, ..)| -> Result<()> {
let first_index = first_index.into_owned();
let count_index = count_indexes_iter.unwrap_get_inner(i);
let count_index = count_indexes_iter.unwrap_get_inner(index);
if let Some(first) = self.first.as_mut() {
let v = source_first_iter
@@ -468,7 +466,7 @@ where
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
values.sort_unstable();
max.forced_push_at(i, *values.last().unwrap(), exit)?;
max.forced_push_at(index, *values.last().unwrap(), exit)?;
}
if let Some(min) = self.min.as_mut() {
@@ -479,7 +477,7 @@ where
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
values.sort_unstable();
min.forced_push_at(i, *values.first().unwrap(), exit)?;
min.forced_push_at(index, *values.first().unwrap(), exit)?;
}
}
@@ -497,7 +495,7 @@ where
// TODO: Multiply by count then divide by cumulative
// Right now it's not 100% accurate as there could be more or less elements in the lower timeframe (28 days vs 31 days in a month for example)
let avg = cumulative / len;
average.forced_push_at(i, avg, exit)?;
average.forced_push_at(index, avg, exit)?;
}
if needs_sum_or_cumulative {
@@ -511,13 +509,13 @@ where
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum, exit)?;
sum_vec.forced_push_at(index, sum, exit)?;
}
if let Some(cumulative_vec) = self.cumulative.as_mut() {
let t = cumulative.unwrap() + sum;
cumulative.replace(t);
cumulative_vec.forced_push_at(i, t, exit)?;
cumulative_vec.forced_push_at(index, t, exit)?;
}
}
}
@@ -665,42 +663,42 @@ where
Ok(())
}
pub fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
pub fn validate_computed_version_or_reset(&mut self, version: Version) -> Result<()> {
if let Some(first) = self.first.as_mut() {
first.validate_computed_version_or_reset_file(Version::ZERO + version)?;
first.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(last) = self.last.as_mut() {
last.validate_computed_version_or_reset_file(Version::ZERO + version)?;
last.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(min) = self.min.as_mut() {
min.validate_computed_version_or_reset_file(Version::ZERO + version)?;
min.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(max) = self.max.as_mut() {
max.validate_computed_version_or_reset_file(Version::ZERO + version)?;
max.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(median) = self.median.as_mut() {
median.validate_computed_version_or_reset_file(Version::ZERO + version)?;
median.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(average) = self.average.as_mut() {
average.validate_computed_version_or_reset_file(Version::ZERO + version)?;
average.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(sum) = self.sum.as_mut() {
sum.validate_computed_version_or_reset_file(Version::ZERO + version)?;
sum.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(cumulative) = self.cumulative.as_mut() {
cumulative.validate_computed_version_or_reset_file(Version::ZERO + version)?;
cumulative.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(_90p) = self._90p.as_mut() {
_90p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
_90p.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(_75p) = self._75p.as_mut() {
_75p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
_75p.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(_25p) = self._25p.as_mut() {
_25p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
_25p.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(_10p) = self._10p.as_mut() {
_10p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
_10p.validate_computed_version_or_reset(Version::ZERO + version)?;
}
Ok(())

2
crates/brk_computer/src/grouped/computed.rs
View File

@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign, Div};
use brk_vecs::StoredCompressed;
use vecdb::StoredCompressed;
pub trait ComputedType
where

26
crates/brk_computer/src/grouped/from_dateindex.rs
View File

@@ -1,14 +1,12 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex,
};
use brk_vecs::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Exit, File,
Format,
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, Database, EagerVec,
Exit, Format,
};
use crate::{Indexes, grouped::ComputedVecBuilder, indexes};
@@ -38,7 +36,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<DateIndex, T>,
version: Version,
@@ -48,11 +46,11 @@ where
options: VecBuilderOptions,
) -> Result<Self> {
let dateindex = source.is_compute().then(|| {
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap()
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format).unwrap()
});
let dateindex_extra = EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -65,7 +63,7 @@ where
Ok(Self {
weekindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -76,7 +74,7 @@ where
options.into(),
)?,
monthindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -87,7 +85,7 @@ where
options.into(),
)?,
quarterindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -98,7 +96,7 @@ where
options.into(),
)?,
semesterindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -109,7 +107,7 @@ where
options.into(),
)?,
yearindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -120,7 +118,7 @@ where
options.into(),
)?,
decadeindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,

32
crates/brk_computer/src/grouped/from_height.rs
View File

@@ -1,5 +1,3 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
@@ -7,9 +5,9 @@ use brk_structs::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex,
Version, WeekIndex, YearIndex,
};
use brk_vecs::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Exit, File,
Format,
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, Database, EagerVec,
Exit, Format,
};
use crate::{
@@ -47,7 +45,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<Height, T>,
version: Version,
@@ -57,11 +55,11 @@ where
options: VecBuilderOptions,
) -> Result<Self> {
let height = source.is_compute().then(|| {
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap()
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format).unwrap()
});
let height_extra = EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -69,7 +67,7 @@ where
)?;
let dateindex = EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -80,7 +78,7 @@ where
Ok(Self {
weekindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -91,7 +89,7 @@ where
options.into(),
)?,
monthindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -102,7 +100,7 @@ where
options.into(),
)?,
quarterindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -113,7 +111,7 @@ where
options.into(),
)?,
semesterindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -124,7 +122,7 @@ where
options.into(),
)?,
yearindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -135,7 +133,7 @@ where
options.into(),
)?,
decadeindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -145,12 +143,12 @@ where
indexes.decadeindex_to_decadeindex.boxed_clone(),
options.into(),
)?,
// halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
height,
height_extra,
dateindex,
difficultyepoch: EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,

15
crates/brk_computer/src/grouped/from_height_strict.rs
View File

@@ -1,10 +1,8 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{DifficultyEpoch, Height, Version};
use brk_vecs::{AnyCollectableVec, EagerVec, Exit, File, Format};
use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, Format};
use crate::{Indexes, indexes};
@@ -29,17 +27,16 @@ where
f64: From<T>,
{
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
version: Version,
format: Format,
options: VecBuilderOptions,
) -> Result<Self> {
let height =
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)?;
let height = EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)?;
let height_extra = EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -52,13 +49,13 @@ where
height,
height_extra,
difficultyepoch: EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
})
}

36
crates/brk_computer/src/grouped/from_txindex.rs
View File

@@ -1,14 +1,12 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex,
};
use brk_vecs::{
AnyCloneableIterableVec, AnyCollectableVec, AnyVec, CollectableVec, Computation, EagerVec,
Exit, File, Format, GenericStoredVec, StoredIndex, VecIterator,
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyVec, CollectableVec, Computation, Database,
EagerVec, Exit, Format, GenericStoredVec, StoredIndex, VecIterator,
};
use crate::{
@@ -46,7 +44,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<TxIndex, T>,
version: Version,
@@ -57,13 +55,13 @@ where
) -> Result<Self> {
let txindex = source.is_compute().then(|| {
Box::new(
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)
.unwrap(),
)
});
let height = EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -73,7 +71,7 @@ where
let options = options.remove_percentiles();
let dateindex = EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -82,7 +80,7 @@ where
Ok(Self {
weekindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -93,7 +91,7 @@ where
options.into(),
)?,
monthindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -104,7 +102,7 @@ where
options.into(),
)?,
quarterindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -115,7 +113,7 @@ where
options.into(),
)?,
semesterindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -126,7 +124,7 @@ where
options.into(),
)?,
yearindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -137,7 +135,7 @@ where
options.into(),
)?,
decadeindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -152,13 +150,13 @@ where
height,
dateindex,
difficultyepoch: EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
})
}
@@ -325,7 +323,7 @@ impl ComputedVecsFromTxindex<Bitcoin> {
};
self.height
.validate_computed_version_or_reset_file(txindex_version)?;
.validate_computed_version_or_reset(txindex_version)?;
let starting_index = self.height.starting_index(starting_indexes.height);
@@ -504,7 +502,7 @@ impl ComputedVecsFromTxindex<Dollars> {
};
self.height
.validate_computed_version_or_reset_file(txindex_version)?;
.validate_computed_version_or_reset(txindex_version)?;
let starting_index = self.height.starting_index(starting_indexes.height);

88
crates/brk_computer/src/grouped/ratio_from_dateindex.rs
View File

@@ -1,11 +1,9 @@
use std::{f32, sync::Arc};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Date, DateIndex, Dollars, StoredF32, Version};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, EagerVec,
Exit, File, Format, GenericStoredVec, StoredIndex, VecIterator,
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, Database,
EagerVec, Exit, Format, GenericStoredVec, StoredIndex, VecIterator,
};
use crate::{Indexes, grouped::source::Source, indexes, price, utils::get_percentile};
@@ -60,7 +58,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedRatioVecsFromDateIndex {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<DateIndex, Dollars>,
version: Version,
@@ -74,7 +72,7 @@ impl ComputedRatioVecsFromDateIndex {
Ok(Self {
price: source.is_compute().then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
name,
Source::Compute,
version + VERSION,
@@ -86,7 +84,7 @@ impl ComputedRatioVecsFromDateIndex {
.unwrap()
}),
ratio: ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -97,7 +95,7 @@ impl ComputedRatioVecsFromDateIndex {
)?,
ratio_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -110,7 +108,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1w_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_1w_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -123,7 +121,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1m_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_1m_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -136,7 +134,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_1y_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -149,7 +147,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_4y_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_4y_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -162,7 +160,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_sma_momentum_oscillator: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_1y_sma_momentum_oscillator"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -175,7 +173,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -188,7 +186,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_4y_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_4y_sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -201,7 +199,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_1y_sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -214,7 +212,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_9: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p99_9"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -227,7 +225,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_5: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p99_5"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -240,7 +238,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p99"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -253,7 +251,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p1"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -266,7 +264,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_5: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p0_5"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -279,7 +277,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_1: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p0_1"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -292,7 +290,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p1sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -305,7 +303,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p2sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p2sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -318,7 +316,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p3sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p3sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -331,7 +329,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m1sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_m1sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -344,7 +342,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m2sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_m2sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -357,7 +355,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m3sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_m3sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -370,7 +368,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_9_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p99_9_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -383,7 +381,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_5_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p99_5_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -396,7 +394,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p99_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -409,7 +407,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p1_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -422,7 +420,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_5_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p0_5_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -435,7 +433,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_1_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p0_1_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -448,7 +446,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p1sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -461,7 +459,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p2sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p2sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -474,7 +472,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p3sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p3sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -487,7 +485,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m1sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_m1sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -500,7 +498,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m2sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_m2sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -513,7 +511,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m3sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_m3sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -526,7 +524,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_zscore"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -539,7 +537,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_4y_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_4y_zscore"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -552,7 +550,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_1y_zscore"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -771,7 +769,7 @@ impl ComputedRatioVecsFromDateIndex {
self.mut_ratio_vecs()
.iter_mut()
.try_for_each(|v| -> Result<()> {
v.validate_computed_version_or_reset_file(
v.validate_computed_version_or_reset(
Version::ZERO + v.inner_version() + ratio_version,
)?;
Ok(())

2
crates/brk_computer/src/grouped/source.rs
View File

@@ -1,4 +1,4 @@
use brk_vecs::AnyBoxedIterableVec;
use vecdb::AnyBoxedIterableVec;
#[derive(Clone)]
pub enum Source<I, T> {

14
crates/brk_computer/src/grouped/value_from_dateindex.rs
View File

@@ -1,10 +1,8 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, DateIndex, Dollars, Sats, Version};
use brk_vecs::{
AnyCollectableVec, CollectableVec, Computation, EagerVec, Exit, File, Format, StoredVec,
use vecdb::{
AnyCollectableVec, CollectableVec, Computation, Database, EagerVec, Exit, Format, StoredVec,
};
use crate::{
@@ -28,7 +26,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromDateIndex {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<DateIndex, Sats>,
version: Version,
@@ -40,7 +38,7 @@ impl ComputedValueVecsFromDateIndex {
) -> Result<Self> {
Ok(Self {
sats: ComputedVecsFromDateIndex::forced_import(
file,
db,
name,
source,
version + VERSION,
@@ -50,7 +48,7 @@ impl ComputedValueVecsFromDateIndex {
options,
)?,
bitcoin: ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_in_btc"),
Source::Compute,
version + VERSION,
@@ -61,7 +59,7 @@ impl ComputedValueVecsFromDateIndex {
)?,
dollars: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_in_usd"),
Source::Compute,
version + VERSION,

14
crates/brk_computer/src/grouped/value_from_height.rs
View File

@@ -1,10 +1,8 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, Dollars, Height, Sats, Version};
use brk_vecs::{
AnyCollectableVec, CollectableVec, Computation, EagerVec, Exit, File, Format, StoredVec,
use vecdb::{
AnyCollectableVec, CollectableVec, Computation, Database, EagerVec, Exit, Format, StoredVec,
};
use crate::{
@@ -28,7 +26,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromHeight {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<Height, Sats>,
version: Version,
@@ -40,7 +38,7 @@ impl ComputedValueVecsFromHeight {
) -> Result<Self> {
Ok(Self {
sats: ComputedVecsFromHeight::forced_import(
file,
db,
name,
source,
version + VERSION,
@@ -50,7 +48,7 @@ impl ComputedValueVecsFromHeight {
options,
)?,
bitcoin: ComputedVecsFromHeight::forced_import(
file,
db,
&format!("{name}_in_btc"),
Source::Compute,
version + VERSION,
@@ -61,7 +59,7 @@ impl ComputedValueVecsFromHeight {
)?,
dollars: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
file,
db,
&format!("{name}_in_usd"),
Source::Compute,
version + VERSION,

16
crates/brk_computer/src/grouped/value_from_txindex.rs
View File

@@ -1,11 +1,9 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use brk_vecs::{
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, CollectableVec, Computation, ComputedVecFrom3,
Exit, File, Format, LazyVecFrom1, StoredIndex, StoredVec,
Database, Exit, Format, LazyVecFrom1, StoredIndex, StoredVec,
};
use crate::{Indexes, grouped::Source, indexes, price};
@@ -38,7 +36,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromTxindex {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
indexes: &indexes::Vecs,
source: Source<TxIndex, Sats>,
@@ -54,7 +52,7 @@ impl ComputedValueVecsFromTxindex {
let name_in_usd = format!("{name}_in_usd");
let sats = ComputedVecsFromTxindex::forced_import(
file,
db,
name,
source.clone(),
version + VERSION,
@@ -79,7 +77,7 @@ impl ComputedValueVecsFromTxindex {
);
let bitcoin = ComputedVecsFromTxindex::forced_import(
file,
db,
&name_in_btc,
Source::None,
version + VERSION,
@@ -92,7 +90,7 @@ impl ComputedValueVecsFromTxindex {
let dollars_txindex = price.map(|price| {
ComputedVecFrom3::forced_import_or_init_from_3(
computation,
file,
db,
&name_in_usd,
version + VERSION,
format,
@@ -127,7 +125,7 @@ impl ComputedValueVecsFromTxindex {
dollars_txindex,
dollars: compute_dollars.then(|| {
ComputedVecsFromTxindex::forced_import(
file,
db,
&name_in_usd,
Source::None,
version + VERSION,

12
crates/brk_computer/src/grouped/value_height.rs
View File

@@ -1,9 +1,7 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, Dollars, Height, Sats, Version};
use brk_vecs::{AnyCollectableVec, CollectableVec, EagerVec, Exit, File, Format, StoredVec};
use vecdb::{AnyCollectableVec, CollectableVec, Database, EagerVec, Exit, Format, StoredVec};
use crate::{
Indexes,
@@ -23,7 +21,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedHeightValueVecs {
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<Height, Sats>,
version: Version,
@@ -32,18 +30,18 @@ impl ComputedHeightValueVecs {
) -> Result<Self> {
Ok(Self {
sats: source.is_compute().then(|| {
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)
.unwrap()
}),
bitcoin: EagerVec::forced_import(
file,
db,
&format!("{name}_in_btc"),
version + VERSION + Version::ZERO,
format,
)?,
dollars: compute_dollars.then(|| {
EagerVec::forced_import(
file,
db,
&format!("{name}_in_usd"),
version + VERSION + Version::ZERO,
format,

136
crates/brk_computer/src/indexes.rs
View File

@@ -1,4 +1,4 @@
use std::{ops::Deref, path::Path, sync::Arc};
use std::{ops::Deref, path::Path};
use brk_error::Result;
use brk_indexer::Indexer;
@@ -10,16 +10,16 @@ use brk_structs::{
P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredU64,
Timestamp, TxIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex,
};
use brk_vecs::{
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, Computation, ComputedVec, ComputedVecFrom1,
ComputedVecFrom2, EagerVec, Exit, File, Format, PAGE_SIZE, StoredIndex, VecIterator,
ComputedVecFrom2, Database, EagerVec, Exit, Format, PAGE_SIZE, StoredIndex, VecIterator,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub dateindex_to_date: EagerVec<DateIndex, Date>,
pub dateindex_to_dateindex: EagerVec<DateIndex, DateIndex>,
@@ -105,12 +105,12 @@ impl Vecs {
computation: Computation,
format: Format,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("indexes"))?);
file.set_min_len(PAGE_SIZE * 10_000_000)?;
let db = Database::open(&parent.join("indexes"))?;
db.set_min_len(PAGE_SIZE * 10_000_000)?;
let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"outputindex",
version + VERSION + Version::ZERO,
format,
@@ -120,7 +120,7 @@ impl Vecs {
let inputindex_to_inputindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"inputindex",
version + VERSION + Version::ZERO,
format,
@@ -130,7 +130,7 @@ impl Vecs {
let txindex_to_txindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"txindex",
version + VERSION + Version::ZERO,
format,
@@ -140,7 +140,7 @@ impl Vecs {
let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2(
computation,
&file,
&db,
"input_count",
version + VERSION + Version::ZERO,
format,
@@ -163,7 +163,7 @@ impl Vecs {
let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2(
computation,
&file,
&db,
"output_count",
version + VERSION + Version::ZERO,
format,
@@ -186,7 +186,7 @@ impl Vecs {
let p2pk33addressindex_to_p2pk33addressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2pk33addressindex",
version + VERSION + Version::ZERO,
format,
@@ -195,7 +195,7 @@ impl Vecs {
)?;
let p2pk65addressindex_to_p2pk65addressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2pk65addressindex",
version + VERSION + Version::ZERO,
format,
@@ -204,7 +204,7 @@ impl Vecs {
)?;
let p2pkhaddressindex_to_p2pkhaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2pkhaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -213,7 +213,7 @@ impl Vecs {
)?;
let p2shaddressindex_to_p2shaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2shaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -222,7 +222,7 @@ impl Vecs {
)?;
let p2traddressindex_to_p2traddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2traddressindex",
version + VERSION + Version::ZERO,
format,
@@ -231,7 +231,7 @@ impl Vecs {
)?;
let p2wpkhaddressindex_to_p2wpkhaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2wpkhaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -240,7 +240,7 @@ impl Vecs {
)?;
let p2wshaddressindex_to_p2wshaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2wshaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -249,7 +249,7 @@ impl Vecs {
)?;
let p2aaddressindex_to_p2aaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2aaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -258,7 +258,7 @@ impl Vecs {
)?;
let p2msoutputindex_to_p2msoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2msoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -267,7 +267,7 @@ impl Vecs {
)?;
let emptyoutputindex_to_emptyoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"emptyoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -276,7 +276,7 @@ impl Vecs {
)?;
let unknownoutputindex_to_unknownoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"unknownoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -285,7 +285,7 @@ impl Vecs {
)?;
let opreturnindex_to_opreturnindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"opreturnindex",
version + VERSION + Version::ZERO,
format,
@@ -313,265 +313,265 @@ impl Vecs {
unknownoutputindex_to_unknownoutputindex,
dateindex_to_date: EagerVec::forced_import(
&file,
&db,
"date",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_dateindex: EagerVec::forced_import(
&file,
&db,
"dateindex",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_first_height: EagerVec::forced_import(
&file,
&db,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_monthindex: EagerVec::forced_import(
&file,
&db,
"monthindex",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_weekindex: EagerVec::forced_import(
&file,
&db,
"weekindex",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_decadeindex: EagerVec::forced_import(
&file,
&db,
"decadeindex",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_first_yearindex: EagerVec::forced_import(
&file,
&db,
"first_yearindex",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_difficultyepoch: EagerVec::forced_import(
&file,
&db,
"difficultyepoch",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_first_height: EagerVec::forced_import(
&file,
&db,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_first_height: EagerVec::forced_import(
&file,
&db,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_halvingepoch: EagerVec::forced_import(
&file,
&db,
"halvingepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_date: EagerVec::forced_import(
&file,
&db,
"date",
version + VERSION + Version::ZERO,
format,
)?,
height_to_difficultyepoch: EagerVec::forced_import(
&file,
&db,
"difficultyepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_halvingepoch: EagerVec::forced_import(
&file,
&db,
"halvingepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_height: EagerVec::forced_import(
&file,
&db,
"height",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_first_dateindex: EagerVec::forced_import(
&file,
&db,
"first_dateindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_monthindex: EagerVec::forced_import(
&file,
&db,
"monthindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_quarterindex: EagerVec::forced_import(
&file,
&db,
"quarterindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_semesterindex: EagerVec::forced_import(
&file,
&db,
"semesterindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_yearindex: EagerVec::forced_import(
&file,
&db,
"yearindex",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_first_monthindex: EagerVec::forced_import(
&file,
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_first_monthindex: EagerVec::forced_import(
&file,
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_first_dateindex: EagerVec::forced_import(
&file,
&db,
"first_dateindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_first_monthindex: EagerVec::forced_import(
&file,
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_quarterindex: EagerVec::forced_import(
&file,
&db,
"quarterindex",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_semesterindex: EagerVec::forced_import(
&file,
&db,
"semesterindex",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_weekindex: EagerVec::forced_import(
&file,
&db,
"weekindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_decadeindex: EagerVec::forced_import(
&file,
&db,
"decadeindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_yearindex: EagerVec::forced_import(
&file,
&db,
"yearindex",
version + VERSION + Version::ZERO,
format,
)?,
height_to_date_fixed: EagerVec::forced_import(
&file,
&db,
"date_fixed",
version + VERSION + Version::ZERO,
format,
)?,
height_to_dateindex: EagerVec::forced_import(
&file,
&db,
"dateindex",
version + VERSION + Version::ZERO,
format,
)?,
txindex_to_height: EagerVec::forced_import(
&file,
&db,
"height",
version + VERSION + Version::ZERO,
format,
)?,
height_to_timestamp_fixed: EagerVec::forced_import(
&file,
&db,
"timestamp_fixed",
version + VERSION + Version::ZERO,
format,
)?,
height_to_txindex_count: EagerVec::forced_import(
&file,
&db,
"txindex_count",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_height_count: EagerVec::forced_import(
&file,
&db,
"height_count",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_dateindex_count: EagerVec::forced_import(
&file,
&db,
"dateindex_count",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_height_count: EagerVec::forced_import(
&file,
&db,
"height_count",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_dateindex_count: EagerVec::forced_import(
&file,
&db,
"dateindex_count",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_monthindex_count: EagerVec::forced_import(
&file,
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_monthindex_count: EagerVec::forced_import(
&file,
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_monthindex_count: EagerVec::forced_import(
&file,
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_yearindex_count: EagerVec::forced_import(
&file,
&db,
"yearindex_count",
version + VERSION + Version::ZERO,
format,
)?,
outputindex_to_txindex: EagerVec::forced_import(
&file,
&db,
"txindex",
version + VERSION + Version::ZERO,
format,
)?,
file,
db,
})
}
@@ -582,7 +582,7 @@ impl Vecs {
exit: &Exit,
) -> Result<Indexes> {
let idxs = self.compute_(indexer, starting_indexes, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(idxs)
}

2
crates/brk_computer/src/lib.rs
View File

@@ -9,8 +9,8 @@ use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_structs::Version;
use brk_vecs::{AnyCollectableVec, Computation, Exit, Format};
use log::info;
use vecdb::{AnyCollectableVec, Computation, Exit, Format};
mod blocks;
mod cointime;

284
crates/brk_computer/src/market.rs
View File

File diff suppressed because it is too large Load Diff

20
crates/brk_computer/src/mining.rs
View File

@@ -1,9 +1,9 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, PAGE_SIZE, VecIterator};
use vecdb::{AnyCollectableVec, Computation, Database, Exit, Format, PAGE_SIZE, VecIterator};
use crate::grouped::Source;
@@ -17,7 +17,7 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub indexes_to_difficulty: ComputedVecsFromHeight<StoredF64>,
pub indexes_to_difficultyepoch: ComputedVecsFromDateIndex<DifficultyEpoch>,
@@ -32,12 +32,12 @@ impl Vecs {
format: Format,
indexes: &indexes::Vecs,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("mining"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
let db = Database::open(&parent.join("mining"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
indexes_to_difficulty: ComputedVecsFromHeight::forced_import(
&file,
&db,
"difficulty",
Source::None,
version + VERSION + Version::ZERO,
@@ -47,7 +47,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"difficultyepoch",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -57,7 +57,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"halvingepoch",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -67,7 +67,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
file,
db,
})
}
@@ -79,7 +79,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

106
crates/brk_computer/src/price.rs
View File

@@ -1,4 +1,4 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
@@ -6,8 +6,8 @@ use brk_structs::{
Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex,
OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, SemesterIndex, Version, WeekIndex, YearIndex,
};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, EagerVec, Exit, File,
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, Database, EagerVec, Exit,
Format, GenericStoredVec, PAGE_SIZE, RawVec,
};
@@ -21,7 +21,7 @@ use super::{
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub dateindex_to_close_in_cents: EagerVec<DateIndex, Close<Cents>>,
pub dateindex_to_high_in_cents: EagerVec<DateIndex, High<Cents>>,
@@ -80,80 +80,76 @@ impl Vecs {
format: Format,
indexes: &indexes::Vecs,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("price"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
let db = Database::open(&parent.join("price"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
dateindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
dateindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
dateindex_to_close_in_cents: EagerVec::forced_import(
&file,
&db,
"close_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_high_in_cents: EagerVec::forced_import(
&file,
&db,
"high_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_low_in_cents: EagerVec::forced_import(
&file,
&db,
"low_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_open_in_cents: EagerVec::forced_import(
&file,
&db,
"open_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_ohlc: RawVec::forced_import(
&file,
"ohlc",
version + VERSION + Version::ZERO,
)?,
height_to_ohlc: RawVec::forced_import(&db, "ohlc", version + VERSION + Version::ZERO)?,
height_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
height_to_close_in_cents: EagerVec::forced_import(
&file,
&db,
"close_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_high_in_cents: EagerVec::forced_import(
&file,
&db,
"high_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_low_in_cents: EagerVec::forced_import(
&file,
&db,
"low_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_open_in_cents: EagerVec::forced_import(
&file,
&db,
"open_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
timeindexes_to_open: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"open",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -163,7 +159,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(),
)?,
timeindexes_to_high: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"high",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -173,7 +169,7 @@ impl Vecs {
VecBuilderOptions::default().add_max(),
)?,
timeindexes_to_low: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"low",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -183,7 +179,7 @@ impl Vecs {
VecBuilderOptions::default().add_min(),
)?,
timeindexes_to_close: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"close",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -193,7 +189,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"open_in_sats",
Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -203,7 +199,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(),
)?,
timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"high_in_sats",
Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -213,7 +209,7 @@ impl Vecs {
VecBuilderOptions::default().add_max(),
)?,
timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"low_in_sats",
Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -223,7 +219,7 @@ impl Vecs {
VecBuilderOptions::default().add_min(),
)?,
timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"close_in_sats",
Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -233,135 +229,135 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"open",
version + VERSION + Version::ZERO,
format,
VecBuilderOptions::default().add_first(),
)?,
chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"high",
version + VERSION + Version::ZERO,
format,
VecBuilderOptions::default().add_max(),
)?,
chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"low",
version + VERSION + Version::ZERO,
format,
VecBuilderOptions::default().add_min(),
)?,
chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"close",
version + VERSION + Version::ZERO,
format,
VecBuilderOptions::default().add_last(),
)?,
chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"open_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
VecBuilderOptions::default().add_first(),
)?,
chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"high_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
VecBuilderOptions::default().add_max(),
)?,
chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"low_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
VecBuilderOptions::default().add_min(),
)?,
chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"close_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
VecBuilderOptions::default().add_last(),
)?,
weekindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
weekindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
difficultyepoch_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
difficultyepoch_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
monthindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
monthindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
quarterindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
quarterindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
semesterindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
semesterindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
yearindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
yearindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
// halvingepoch_to_ohlc: StorableVec::forced_import(file,
// halvingepoch_to_ohlc: StorableVec::forced_import(db,
// "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?,
decadeindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
decadeindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
file,
db,
})
}
@@ -374,7 +370,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, fetched, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

55
crates/brk_computer/src/stateful/address_cohort.rs
View File

@@ -1,10 +1,10 @@
use std::{ops::Deref, path::Path, sync::Arc};
use std::{ops::Deref, path::Path};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, EagerVec, Exit, File,
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, Database, EagerVec, Exit,
Format, GenericStoredVec, VecIterator,
};
@@ -25,7 +25,7 @@ const VERSION: Version = Version::ZERO;
pub struct Vecs {
starting_height: Height,
pub state: AddressCohortState,
pub state: Option<AddressCohortState>,
pub inner: common::Vecs,
@@ -36,14 +36,14 @@ pub struct Vecs {
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
cohort_name: Option<&str>,
computation: Computation,
format: Format,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
states_path: Option<&Path>,
compute_relative_to_all: bool,
) -> Result<Self> {
let compute_dollars = price.is_some();
@@ -52,19 +52,22 @@ impl Vecs {
Ok(Self {
starting_height: Height::ZERO,
state: AddressCohortState::default_and_import(
states_path,
cohort_name.unwrap_or_default(),
compute_dollars,
)?,
state: states_path.map(|states_path| {
AddressCohortState::default_and_import(
states_path,
cohort_name.unwrap_or_default(),
compute_dollars,
)
.unwrap()
}),
height_to_address_count: EagerVec::forced_import(
file,
db,
&suffix("address_count"),
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_address_count: ComputedVecsFromHeight::forced_import(
file,
db,
&suffix("address_count"),
Source::None,
version + VERSION + Version::ZERO,
@@ -74,7 +77,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
inner: common::Vecs::forced_import(
file,
db,
cohort_name,
computation,
format,
@@ -83,6 +86,7 @@ impl Vecs {
price,
compute_relative_to_all,
false,
false,
)?,
})
}
@@ -91,7 +95,9 @@ impl Vecs {
impl DynCohortVecs for Vecs {
fn starting_height(&self) -> Height {
[
self.state.height().map_or(Height::MAX, |h| h.incremented()),
self.state.as_ref().map_or(Height::MAX, |state| {
state.height().map_or(Height::MAX, |h| h.incremented())
}),
self.height_to_address_count.len().into(),
self.inner.starting_height(),
]
@@ -108,19 +114,21 @@ impl DynCohortVecs for Vecs {
self.starting_height = starting_height;
if let Some(prev_height) = starting_height.decremented() {
self.state.address_count = *self
self.state.as_mut().unwrap().address_count = *self
.height_to_address_count
.into_iter()
.unwrap_get_inner(prev_height);
}
self.inner
.init(&mut self.starting_height, &mut self.state.inner);
self.inner.init(
&mut self.starting_height,
&mut self.state.as_mut().unwrap().inner,
);
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.height_to_address_count
.validate_computed_version_or_reset_file(
.validate_computed_version_or_reset(
base_version + self.height_to_address_count.inner_version(),
)?;
@@ -134,11 +142,12 @@ impl DynCohortVecs for Vecs {
self.height_to_address_count.forced_push_at(
height,
self.state.address_count.into(),
self.state.as_ref().unwrap().address_count.into(),
exit,
)?;
self.inner.forced_pushed_at(height, exit, &self.state.inner)
self.inner
.forced_pushed_at(height, exit, &self.state.as_ref().unwrap().inner)
}
fn compute_then_force_push_unrealized_states(
@@ -155,7 +164,7 @@ impl DynCohortVecs for Vecs {
dateindex,
date_price,
exit,
&self.state.inner,
&self.state.as_ref().unwrap().inner,
)
}
@@ -163,7 +172,7 @@ impl DynCohortVecs for Vecs {
self.height_to_address_count.safe_flush(exit)?;
self.inner
.safe_flush_stateful_vecs(height, exit, &mut self.state.inner)
.safe_flush_stateful_vecs(height, exit, &mut self.state.as_mut().unwrap().inner)
}
#[allow(clippy::too_many_arguments)]

170
crates/brk_computer/src/stateful/address_cohorts.rs
View File

@@ -1,4 +1,4 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
@@ -6,8 +6,8 @@ use brk_structs::{
AddressGroups, Bitcoin, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, DateIndex,
Dollars, GroupFilter, Height, Version,
};
use brk_vecs::{AnyIterableVec, Computation, Exit, File, Format};
use derive_deref::{Deref, DerefMut};
use vecdb::{AnyIterableVec, Computation, Database, Exit, Format};
use crate::{
Indexes, indexes, market, price,
@@ -24,7 +24,7 @@ pub struct Vecs(AddressGroups<(GroupFilter, address_cohort::Vecs)>);
impl Vecs {
pub fn forced_import(
file: &Arc<File>,
db: &Database,
version: Version,
_computation: Computation,
format: Format,
@@ -36,458 +36,458 @@ impl Vecs {
AddressGroups {
amount_range: ByAmountRange {
_0sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_with_0sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_1sat_to_10sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1sat_under_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_10sats_to_100sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10sats_under_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_100sats_to_1k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100sats_under_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_1k_sats_to_10k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1k_sats_under_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_10k_sats_to_100k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10k_sats_under_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_100k_sats_to_1m_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100k_sats_under_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_1m_sats_to_10m_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1m_sats_under_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_10m_sats_to_1btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10m_sats_under_1btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_1btc_to_10btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1btc_under_10btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_10btc_to_100btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10btc_under_100btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_100btc_to_1k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100btc_under_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_1k_btc_to_10k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1k_btc_under_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_10k_btc_to_100k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10k_btc_under_100k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_100k_btc_or_more: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
},
lt_amount: ByLowerThanAmount {
_10sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1m_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10m_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_1btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_10btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_100btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_100k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
},
ge_amount: ByGreatEqualAmount {
_1sat: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1sat"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1m_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10m_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
},

2
crates/brk_computer/src/stateful/addresstype_to_addresscount.rs
View File

@@ -1,6 +1,6 @@
use brk_structs::{ByAddressType, Height};
use brk_vecs::VecIterator;
use derive_deref::{Deref, DerefMut};
use vecdb::VecIterator;
use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;

2
crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs
View File

@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_structs::{ByAddressType, Height, StoredU64};
use brk_vecs::{EagerVec, Exit, GenericStoredVec};
use vecdb::{EagerVec, Exit, GenericStoredVec};
use derive_deref::{Deref, DerefMut};
use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;

2
crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs
View File

@@ -1,7 +1,7 @@
use brk_error::Result;
use brk_structs::{ByAddressType, StoredU64};
use brk_vecs::{AnyCollectableVec, Exit};
use derive_deref::{Deref, DerefMut};
use vecdb::{AnyCollectableVec, Exit};
use crate::{
Indexes, grouped::ComputedVecsFromHeight, indexes,

450
crates/brk_computer/src/stateful/common.rs
View File

File diff suppressed because it is too large Load Diff

185
crates/brk_computer/src/stateful/mod.rs
View File

@@ -1,4 +1,4 @@
use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, sync::Arc, thread};
use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread};
use brk_error::Result;
use brk_indexer::Indexer;
@@ -9,12 +9,12 @@ use brk_structs::{
P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex,
P2WSHAddressIndex, Sats, StoredU64, Timestamp, TypeIndex, Version,
};
use brk_vecs::{
AnyCollectableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, EagerVec, Exit, File,
Format, GenericStoredVec, PAGE_SIZE, RawVec, Reader, Stamp, StoredIndex, VecIterator,
};
use log::info;
use rayon::prelude::*;
use vecdb::{
AnyCollectableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, Database, EagerVec, Exit,
Format, GenericStoredVec, PAGE_SIZE, RawVec, Reader, Stamp, StoredIndex, VecIterator,
};
use crate::{
BlockState, Indexes, SupplyState, Transacted,
@@ -52,7 +52,7 @@ const VERSION: Version = Version::new(21);
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub chain_state: RawVec<Height, SupplyState>,
@@ -92,29 +92,29 @@ impl Vecs {
price: Option<&price::Vecs>,
states_path: &Path,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("stateful"))?);
file.set_min_len(PAGE_SIZE * 20_000_000)?;
file.set_min_regions(50_000)?;
let db = Database::open(&parent.join("stateful"))?;
db.set_min_len(PAGE_SIZE * 20_000_000)?;
db.set_min_regions(50_000)?;
let compute_dollars = price.is_some();
let chain_file = Arc::new(File::open(&parent.join("chain"))?);
let chain_db = Database::open(&parent.join("chain"))?;
Ok(Self {
chain_state: RawVec::forced_import(
&chain_file,
&chain_db,
"chain",
version + VERSION + Version::ZERO,
)?,
height_to_unspendable_supply: EagerVec::forced_import(
&file,
&db,
"unspendable_supply",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"unspendable_supply",
Source::None,
version + VERSION + Version::ZERO,
@@ -125,13 +125,13 @@ impl Vecs {
indexes,
)?,
height_to_opreturn_supply: EagerVec::forced_import(
&file,
&db,
"opreturn_supply",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"opreturn_supply",
Source::None,
version + VERSION + Version::ZERO,
@@ -142,7 +142,7 @@ impl Vecs {
indexes,
)?,
indexes_to_address_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"address_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -152,7 +152,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_empty_address_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"empty_address_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -164,49 +164,49 @@ impl Vecs {
addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount::from(
ByAddressType {
p2pk65: EagerVec::forced_import(
&file,
&db,
"p2pk65_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
&file,
&db,
"p2pk33_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
&file,
&db,
"p2pkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
&file,
&db,
"p2sh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
&file,
&db,
"p2wpkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
&file,
&db,
"p2wsh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
&file,
&db,
"p2tr_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
&file,
&db,
"p2a_address_count",
version + VERSION + Version::ZERO,
format,
@@ -216,49 +216,49 @@ impl Vecs {
addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount::from(
ByAddressType {
p2pk65: EagerVec::forced_import(
&file,
&db,
"p2pk65_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
&file,
&db,
"p2pk33_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
&file,
&db,
"p2pkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
&file,
&db,
"p2sh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
&file,
&db,
"p2wpkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
&file,
&db,
"p2wsh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
&file,
&db,
"p2tr_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
&file,
&db,
"p2a_empty_address_count",
version + VERSION + Version::ZERO,
format,
@@ -268,7 +268,7 @@ impl Vecs {
addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount::from(
ByAddressType {
p2pk65: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pk65_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -278,7 +278,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pk33: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pk33_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -288,7 +288,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pkh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pkh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -298,7 +298,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2sh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2sh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -308,7 +308,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wpkh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2wpkh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -318,7 +318,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wsh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2wsh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -328,7 +328,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2tr: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2tr_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -338,7 +338,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2a: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2a_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -352,7 +352,7 @@ impl Vecs {
addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount::from(
ByAddressType {
p2pk65: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pk65_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -362,7 +362,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pk33: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pk33_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -372,7 +372,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pkh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pkh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -382,7 +382,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2sh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2sh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -392,7 +392,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wpkh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2wpkh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -402,7 +402,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wsh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2wsh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -412,7 +412,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2tr: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2tr_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -422,7 +422,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2a: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2a_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -434,7 +434,7 @@ impl Vecs {
},
),
utxo_cohorts: utxo_cohorts::Vecs::forced_import(
&file,
&db,
version,
computation,
format,
@@ -443,7 +443,7 @@ impl Vecs {
states_path,
)?,
address_cohorts: address_cohorts::Vecs::forced_import(
&file,
&db,
version,
computation,
format,
@@ -453,58 +453,58 @@ impl Vecs {
)?,
p2aaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2pk33addressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2pk65addressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2pkhaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2shaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2traddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2wpkhaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2wshaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
loadedaddressindex_to_loadedaddressdata: RawVec::forced_import(
&file,
&db,
"loadedaddressdata",
version + VERSION + Version::ZERO,
)?,
emptyaddressindex_to_emptyaddressdata: RawVec::forced_import(
&file,
&db,
"emptyaddressdata",
version + VERSION + Version::ZERO,
)?,
file,
db,
})
}
@@ -529,7 +529,7 @@ impl Vecs {
starting_indexes,
exit,
)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}
@@ -624,11 +624,11 @@ impl Vecs {
.par_iter_mut()
.try_for_each(|(_, v)| v.validate_computed_versions(base_version))?;
self.height_to_unspendable_supply
.validate_computed_version_or_reset_file(
.validate_computed_version_or_reset(
base_version + self.height_to_unspendable_supply.inner_version(),
)?;
self.height_to_opreturn_supply
.validate_computed_version_or_reset_file(
.validate_computed_version_or_reset(
base_version + self.height_to_opreturn_supply.inner_version(),
)?;
@@ -712,13 +712,14 @@ impl Vecs {
separate_utxo_vecs
.par_iter_mut()
.try_for_each(|(_, v)| v.state.reset_price_to_amount())?;
.flat_map(|(_, v)| v.state.as_mut())
.try_for_each(|state| state.reset_price_to_amount())?;
info!("Resetting address price maps...");
separate_address_vecs
.par_iter_mut()
.try_for_each(|(_, v)| v.state.reset_price_to_amount())?;
.try_for_each(|(_, v)| v.state.as_mut().unwrap().reset_price_to_amount())?;
};
let last_height = Height::from(indexer.vecs.height_to_blockhash.stamp());
@@ -820,12 +821,12 @@ impl Vecs {
self.utxo_cohorts
.as_mut_separate_vecs()
.iter_mut()
.for_each(|(_, v)| v.state.reset_single_iteration_values());
.for_each(|(_, v)| v.state.as_mut().unwrap().reset_single_iteration_values());
self.address_cohorts
.as_mut_separate_vecs()
.iter_mut()
.for_each(|(_, v)| v.state.reset_single_iteration_values());
.for_each(|(_, v)| v.state.as_mut().unwrap().reset_single_iteration_values());
let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height);
let price = height_to_close_iter
@@ -1874,17 +1875,27 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
.get_mut(prev_amount)
.1
.state
.as_mut()
.unwrap()
.subtract(addressdata);
}
addressdata.receive(value, price);
vecs.amount_range.get_mut(amount).1.state.add(addressdata);
vecs.amount_range
.get_mut(amount)
.1
.state
.as_mut()
.unwrap()
.add(addressdata);
} else {
vecs.amount_range
.get_mut(amount)
.1
.state
.as_mut()
.unwrap()
.receive(addressdata, value, price);
}
});
@@ -1965,6 +1976,8 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
.get_mut(prev_amount)
.1
.state
.as_mut()
.unwrap()
.subtract(addressdata);
addressdata.send(value, prev_price)?;
@@ -1985,18 +1998,30 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
.unwrap()
.insert(type_index, addressdata.into());
} else {
vecs.amount_range.get_mut(amount).1.state.add(addressdata);
vecs.amount_range
.get_mut(amount)
.1
.state
.as_mut()
.unwrap()
.add(addressdata);
}
} else {
vecs.amount_range.get_mut(amount).1.state.send(
addressdata,
value,
price,
prev_price,
blocks_old,
days_old,
older_than_hour,
)?;
vecs.amount_range
.get_mut(amount)
.1
.state
.as_mut()
.unwrap()
.send(
addressdata,
value,
price,
prev_price,
blocks_old,
days_old,
older_than_hour,
)?;
}
Ok(())

2
crates/brk_computer/src/stateful/range_map.rs
View File

@@ -1,6 +1,6 @@
use std::collections::BTreeMap;
use brk_vecs::{CompressedVec, RawVec, StoredCompressed, StoredIndex, StoredRaw};
use vecdb::{CompressedVec, RawVec, StoredCompressed, StoredIndex, StoredRaw};
#[derive(Debug)]
pub struct RangeMap<I, T>(BTreeMap<I, T>);

2
crates/brk_computer/src/stateful/trait.rs
View File

@@ -1,7 +1,7 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version};
use brk_vecs::{AnyCollectableVec, AnyIterableVec, Exit};
use vecdb::{AnyCollectableVec, AnyIterableVec, Exit};
use crate::{Indexes, indexes, market, price};

41
crates/brk_computer/src/stateful/utxo_cohort.rs
View File

@@ -1,9 +1,9 @@
use std::{ops::Deref, path::Path, sync::Arc};
use std::{ops::Deref, path::Path};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version};
use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, Exit, File, Format};
use vecdb::{AnyCollectableVec, AnyIterableVec, Computation, Database, Exit, Format};
use crate::{
Indexes, UTXOCohortState, indexes, market, price,
@@ -17,7 +17,7 @@ use crate::{
pub struct Vecs {
starting_height: Height,
pub state: UTXOCohortState,
pub state: Option<UTXOCohortState>,
inner: common::Vecs,
}
@@ -25,30 +25,34 @@ pub struct Vecs {
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
cohort_name: Option<&str>,
computation: Computation,
format: Format,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
states_path: Option<&Path>,
compute_relative_to_all: bool,
ratio_extended: bool,
compute_adjusted: bool,
) -> Result<Self> {
let compute_dollars = price.is_some();
Ok(Self {
starting_height: Height::ZERO,
state: UTXOCohortState::default_and_import(
states_path,
cohort_name.unwrap_or_default(),
compute_dollars,
)?,
state: states_path.map(|states_path| {
UTXOCohortState::default_and_import(
states_path,
cohort_name.unwrap_or_default(),
compute_dollars,
)
.unwrap()
}),
inner: common::Vecs::forced_import(
file,
db,
cohort_name,
computation,
format,
@@ -57,6 +61,7 @@ impl Vecs {
price,
compute_relative_to_all,
ratio_extended,
compute_adjusted,
)?,
})
}
@@ -65,7 +70,9 @@ impl Vecs {
impl DynCohortVecs for Vecs {
fn starting_height(&self) -> Height {
[
self.state.height().map_or(Height::MAX, |h| h.incremented()),
self.state.as_ref().map_or(Height::MAX, |state| {
state.height().map_or(Height::MAX, |h| h.incremented())
}),
self.inner.starting_height(),
]
.into_iter()
@@ -80,7 +87,8 @@ impl DynCohortVecs for Vecs {
self.starting_height = starting_height;
self.inner.init(&mut self.starting_height, &mut self.state);
self.inner
.init(&mut self.starting_height, self.state.as_mut().unwrap());
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
@@ -92,7 +100,8 @@ impl DynCohortVecs for Vecs {
return Ok(());
}
self.inner.forced_pushed_at(height, exit, &self.state)
self.inner
.forced_pushed_at(height, exit, self.state.as_ref().unwrap())
}
fn compute_then_force_push_unrealized_states(
@@ -109,13 +118,13 @@ impl DynCohortVecs for Vecs {
dateindex,
date_price,
exit,
&self.state,
self.state.as_mut().unwrap(),
)
}
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.inner
.safe_flush_stateful_vecs(height, exit, &mut self.state)
.safe_flush_stateful_vecs(height, exit, self.state.as_mut().unwrap())
}
#[allow(clippy::too_many_arguments)]

653
crates/brk_computer/src/stateful/utxo_cohorts.rs
View File

File diff suppressed because it is too large Load Diff

16
crates/brk_computer/src/traits.rs
View File

@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_structs::{Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Sats, StoredF32};
use brk_vecs::{
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, StoredIndex,
VecIterator, Version,
};
@@ -32,7 +32,7 @@ impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
len: usize,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + closes.version(),
)?;
@@ -83,7 +83,7 @@ impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
from: DateIndex,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + closes.version(),
)?;
@@ -143,7 +143,7 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
len: usize,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ONE + self.inner_version() + stacks.version(),
)?;
@@ -176,7 +176,7 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
from: DateIndex,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + stacks.version(),
)?;
@@ -218,7 +218,7 @@ where
sats: &impl AnyIterableVec<I, Sats>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + sats.version(),
)?;
@@ -254,7 +254,7 @@ where
price: &impl AnyIterableVec<I, Close<Dollars>>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + bitcoin.version(),
)?;
@@ -292,7 +292,7 @@ where
ath: &impl AnyIterableVec<I, Dollars>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + ath.version() + close.version(),
)?;

88
crates/brk_computer/src/transactions.rs
View File

@@ -1,4 +1,4 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
@@ -6,9 +6,9 @@ use brk_structs::{
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredBool,
StoredU32, StoredU64, TxIndex, TxVersion, Version, Weight,
};
use brk_vecs::{
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, ComputedVec,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Exit, File, Format, PAGE_SIZE,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Database, Exit, Format, PAGE_SIZE,
StoredIndex, VecIterator,
};
@@ -23,7 +23,7 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
// pub txindex_to_is_v1: LazyVec<Txindex, bool>,
// pub txindex_to_is_v2: LazyVec<Txindex, bool>,
@@ -90,14 +90,14 @@ impl Vecs {
format: Format,
price: Option<&price::Vecs>,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("transactions"))?);
file.set_min_len(PAGE_SIZE * 10_000_000)?;
let db = Database::open(&parent.join("transactions"))?;
db.set_min_len(PAGE_SIZE * 10_000_000)?;
let compute_dollars = price.is_some();
let inputindex_to_value = ComputedVec::forced_import_or_init_from_2(
computation,
&file,
&db,
"value",
version + VERSION + Version::ZERO,
format,
@@ -124,7 +124,7 @@ impl Vecs {
let txindex_to_weight = ComputedVec::forced_import_or_init_from_2(
computation,
&file,
&db,
"weight",
version + VERSION + Version::ZERO,
format,
@@ -152,7 +152,7 @@ impl Vecs {
let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"vsize",
version + VERSION + Version::ZERO,
format,
@@ -169,7 +169,7 @@ impl Vecs {
let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2(
computation,
&file,
&db,
"is_coinbase",
version + VERSION + Version::ZERO,
format,
@@ -192,7 +192,7 @@ impl Vecs {
let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3(
computation,
&file,
&db,
"input_value",
version + VERSION + Version::ZERO,
format,
@@ -228,7 +228,7 @@ impl Vecs {
// let indexes_to_input_value: ComputedVecsFromTxindex<Sats> =
// ComputedVecsFromTxindex::forced_import(
// file,
// db,
// "input_value",
// true,
// version + VERSION + Version::ZERO,
@@ -242,7 +242,7 @@ impl Vecs {
let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3(
computation,
&file,
&db,
"output_value",
version + VERSION + Version::ZERO,
format,
@@ -278,7 +278,7 @@ impl Vecs {
// let indexes_to_output_value: ComputedVecsFromTxindex<Sats> =
// ComputedVecsFromTxindex::forced_import(
// file,
// db,
// "output_value",
// true,
// version + VERSION + Version::ZERO,
@@ -292,7 +292,7 @@ impl Vecs {
let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2(
Computation::Eager,
&file,
&db,
"fee",
version + VERSION + Version::ZERO,
format,
@@ -315,7 +315,7 @@ impl Vecs {
let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2(
Computation::Eager,
&file,
&db,
"feerate",
version + VERSION + Version::ZERO,
format,
@@ -335,7 +335,7 @@ impl Vecs {
Ok(Self {
indexes_to_tx_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"tx_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -350,7 +350,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_input_count: ComputedVecsFromTxindex::forced_import(
&file,
&db,
"input_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -365,7 +365,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_output_count: ComputedVecsFromTxindex::forced_import(
&file,
&db,
"output_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -380,7 +380,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
&file,
&db,
"tx_v1",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -390,7 +390,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_tx_v2: ComputedVecsFromHeight::forced_import(
&file,
&db,
"tx_v2",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -400,7 +400,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_tx_v3: ComputedVecsFromHeight::forced_import(
&file,
&db,
"tx_v3",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -410,7 +410,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_fee: ComputedValueVecsFromTxindex::forced_import(
&file,
&db,
"fee",
indexes,
Source::Vec(txindex_to_fee.boxed_clone()),
@@ -426,7 +426,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_feerate: ComputedVecsFromTxindex::forced_import(
&file,
&db,
"feerate",
Source::None,
version + VERSION + Version::ZERO,
@@ -439,7 +439,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import(
&file,
&db,
"tx_vsize",
Source::None,
version + VERSION + Version::ZERO,
@@ -452,7 +452,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import(
&file,
&db,
"tx_weight",
Source::None,
version + VERSION + Version::ZERO,
@@ -465,7 +465,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"subsidy",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -481,7 +481,7 @@ impl Vecs {
indexes,
)?,
indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"coinbase",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -497,7 +497,7 @@ impl Vecs {
indexes,
)?,
indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"unclaimed_rewards",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -508,7 +508,7 @@ impl Vecs {
indexes,
)?,
indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2a_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -523,7 +523,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2ms_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -538,7 +538,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pk33_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -553,7 +553,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pk65_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -568,7 +568,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pkh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -583,7 +583,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2sh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -598,7 +598,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2tr_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -613,7 +613,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2wpkh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -628,7 +628,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2wsh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -643,7 +643,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"opreturn_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -658,7 +658,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"unknownoutput_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -673,7 +673,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"emptyoutput_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -688,7 +688,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"exact_utxo_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -708,7 +708,7 @@ impl Vecs {
txindex_to_vsize,
txindex_to_weight,
file,
db,
})
}
@@ -721,7 +721,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, price, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}