global: snapshot

This commit is contained in:
nym21
2025-10-23 18:30:29 +02:00
parent 6cd60a064b
commit 4f1653b086
234 changed files with 758 additions and 722 deletions
+6 -8
View File
@@ -3,14 +3,13 @@ use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_reader::Reader;
use brk_structs::{BlkPosition, Height, TxIndex, Version};
use brk_traversable::Traversable;
use brk_types::{BlkPosition, Height, TxIndex, Version};
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, CompressedVec, Database, Exit, GenericStoredVec,
PAGE_SIZE, VecIterator,
AnyStoredVec, AnyVec, CompressedVec, Database, Exit, GenericStoredVec, PAGE_SIZE, VecIterator,
};
use super::{Indexes, indexes};
use super::Indexes;
#[derive(Clone, Traversable)]
pub struct Vecs {
@@ -54,12 +53,11 @@ impl Vecs {
pub fn compute(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
reader: &Reader,
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, reader, exit)?;
self.compute_(indexer, starting_indexes, reader, exit)?;
self.db.flush_then_punch()?;
Ok(())
}
@@ -67,7 +65,6 @@ impl Vecs {
fn compute_(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
parser: &Reader,
exit: &Exit,
@@ -75,7 +72,8 @@ impl Vecs {
let min_txindex =
TxIndex::from(self.txindex_to_position.len()).min(starting_indexes.txindex);
let Some(min_height) = indexes
let Some(min_height) = indexer
.vecs
.txindex_to_height
.iter()
.get_inner(min_txindex)
+3 -2
View File
@@ -3,13 +3,13 @@ use std::path::Path;
use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
use brk_traversable::Traversable;
use brk_types::{
Bitcoin, CheckedSub, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, FeeRate, HalvingEpoch,
Height, MonthIndex, ONE_DAY_IN_SEC_F64, QuarterIndex, Sats, SemesterIndex, StoredBool,
StoredF32, StoredF64, StoredU32, StoredU64, Timestamp, TxInIndex, TxIndex, TxOutIndex,
TxVersion, Version, WeekIndex, Weight, YearIndex,
};
use brk_traversable::Traversable;
use vecdb::{
AnyCloneableIterableVec, AnyIterableVec, Database, EagerVec, Exit, LazyVecFrom1, LazyVecFrom2,
LazyVecFrom3, PAGE_SIZE, StoredIndex, VecIterator,
@@ -627,6 +627,7 @@ impl Vecs {
indexes_to_fee: ComputedValueVecsFromTxindex::forced_import(
&db,
"fee",
indexer,
indexes,
Source::Vec(txindex_to_fee.boxed_clone()),
version + Version::ZERO,
+1 -1
View File
@@ -1,8 +1,8 @@
use std::path::Path;
use brk_error::Result;
use brk_structs::{Bitcoin, CheckedSub, Dollars, StoredF32, StoredF64, Version};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, CheckedSub, Dollars, StoredF32, StoredF64, Version};
use vecdb::{Database, Exit, PAGE_SIZE, VecIterator};
use crate::grouped::ComputedVecsFromDateIndex;
+3 -3
View File
@@ -1,9 +1,9 @@
use std::path::Path;
use brk_error::Result;
use brk_structs::{StoredF32, StoredI16, StoredU16, Version};
use brk_traversable::Traversable;
use vecdb::{AnyVec, Database, Exit};
use brk_types::{StoredF32, StoredI16, StoredU16, Version};
use vecdb::{AnyVec, Database, Exit, PAGE_SIZE};
use crate::grouped::Source;
@@ -40,7 +40,7 @@ impl Vecs {
indexes: &indexes::Vecs,
) -> Result<Self> {
let db = Database::open(&parent_path.join("constants"))?;
db.set_min_len(PAGE_SIZE * 10_000_000)?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
let version = parent_version + Version::ZERO;
+3 -3
View File
@@ -3,11 +3,11 @@ use std::path::Path;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_structs::{DateIndex, Height, OHLCCents, Version};
use brk_traversable::Traversable;
use brk_types::{DateIndex, Height, OHLCCents, Version};
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec, RawVec, StoredIndex,
VecIterator,
AnyIterableVec, AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec, PAGE_SIZE, RawVec,
StoredIndex, VecIterator,
};
use super::{Indexes, indexes};
@@ -1,7 +1,7 @@
use allocative::Allocative;
use brk_error::{Error, Result};
use brk_structs::{CheckedSub, StoredU64, Version};
use brk_traversable::Traversable;
use brk_types::{CheckedSub, StoredU64, Version};
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit, Format, GenericStoredVec,
StoredIndex, StoredRaw,
@@ -1,6 +1,6 @@
use allocative::Allocative;
use brk_structs::Version;
use brk_traversable::Traversable;
use brk_types::Version;
use vecdb::{
AnyBoxedIterableVec, AnyCloneableIterableVec, FromCoarserIndex, LazyVecFrom2, StoredIndex,
};
@@ -1,10 +1,10 @@
use allocative::Allocative;
use brk_error::Result;
use brk_structs::{
use brk_traversable::Traversable;
use brk_types::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex,
};
use brk_traversable::Traversable;
use vecdb::{AnyCloneableIterableVec, AnyIterableVec, Database, EagerVec, Exit};
use crate::{Indexes, grouped::LazyVecsBuilder, indexes};
@@ -1,11 +1,11 @@
use allocative::Allocative;
use brk_error::Result;
use brk_structs::{
use brk_traversable::Traversable;
use brk_types::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex,
Version, WeekIndex, YearIndex,
};
use brk_traversable::Traversable;
use vecdb::{AnyCloneableIterableVec, AnyIterableVec, Database, EagerVec, Exit};
use crate::{
@@ -1,7 +1,7 @@
use brk_error::Result;
use brk_structs::{DifficultyEpoch, Height, Version};
use brk_traversable::Traversable;
use brk_types::{DifficultyEpoch, Height, Version};
use vecdb::{Database, EagerVec, Exit};
use crate::{Indexes, indexes};
@@ -1,11 +1,11 @@
use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
use brk_traversable::Traversable;
use brk_types::{
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex,
};
use brk_traversable::Traversable;
use vecdb::{
AnyCloneableIterableVec, AnyVec, CollectableVec, Database, EagerVec, Exit, GenericStoredVec,
StoredIndex, VecIterator,
@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_structs::{Date, DateIndex, Dollars, StoredF32, Version};
use brk_traversable::Traversable;
use brk_types::{Date, DateIndex, Dollars, StoredF32, Version};
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, CollectableVec, Database, EagerVec, Exit,
GenericStoredVec, StoredIndex, VecIterator,
@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_structs::{CheckedSub, Date, DateIndex, Dollars, StoredF32, Version};
use brk_traversable::Traversable;
use brk_types::{CheckedSub, Date, DateIndex, Dollars, StoredF32, Version};
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, BoxedVecIterator, CollectableVec, Database, EagerVec,
Exit, GenericStoredVec, StoredIndex,
@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_structs::{Bitcoin, DateIndex, Dollars, Sats, Version};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Sats, Version};
use vecdb::{CollectableVec, Database, EagerVec, Exit, StoredVec};
use crate::{
@@ -1,7 +1,7 @@
use allocative::Allocative;
use brk_error::Result;
use brk_structs::{Bitcoin, Dollars, Height, Sats, Version};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, Dollars, Height, Sats, Version};
use vecdb::{CollectableVec, Database, EagerVec, Exit, StoredVec};
use crate::{
@@ -1,8 +1,8 @@
use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use vecdb::{
AnyCloneableIterableVec, CollectableVec, Database, Exit, LazyVecFrom1, LazyVecFrom3,
StoredIndex, StoredVec,
@@ -31,6 +31,7 @@ impl ComputedValueVecsFromTxindex {
pub fn forced_import(
db: &Database,
name: &str,
indexer: &Indexer,
indexes: &indexes::Vecs,
source: Source<TxIndex, Sats>,
version: Version,
@@ -79,7 +80,7 @@ impl ComputedValueVecsFromTxindex {
&name_usd,
version + VERSION,
bitcoin_txindex.boxed_clone(),
indexes.txindex_to_height.boxed_clone(),
indexer.vecs.txindex_to_height.boxed_clone(),
price.chainindexes_to_price_close.height.boxed_clone(),
|txindex: TxIndex,
txindex_to_btc_iter,
@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_structs::{Bitcoin, Dollars, Height, Sats, Version};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, Dollars, Height, Sats, Version};
use vecdb::{CollectableVec, Database, EagerVec, Exit, Format, StoredVec};
use crate::{
+101 -71
View File
@@ -2,15 +2,16 @@ use std::{ops::Deref, path::Path};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
Date, DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height,
MonthIndex, OpReturnIndex, P2AAddressIndex, P2ABytes, P2MSOutputIndex, P2PK33AddressIndex,
P2PK33Bytes, P2PK65AddressIndex, P2PK65Bytes, P2PKHAddressIndex, P2PKHBytes, P2SHAddressIndex,
P2SHBytes, P2TRAddressIndex, P2TRBytes, P2WPKHAddressIndex, P2WPKHBytes, P2WSHAddressIndex,
P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredU64, Timestamp, TxInIndex, TxIndex,
TxOutIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex,
};
use brk_traversable::Traversable;
use brk_types::{
Date, DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height,
MonthIndex, OpReturnIndex, OutPoint, P2AAddressIndex, P2ABytes, P2MSOutputIndex,
P2PK33AddressIndex, P2PK33Bytes, P2PK65AddressIndex, P2PK65Bytes, P2PKHAddressIndex,
P2PKHBytes, P2SHAddressIndex, P2SHBytes, P2TRAddressIndex, P2TRBytes, P2WPKHAddressIndex,
P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredU64,
Timestamp, TxInIndex, TxIndex, TxOutIndex, Txid, UnknownOutputIndex, Version, WeekIndex,
YearIndex,
};
use vecdb::{
AnyCloneableIterableVec, Database, EagerVec, Exit, LazyVecFrom1, LazyVecFrom2, PAGE_SIZE,
StoredIndex, VecIterator,
@@ -46,7 +47,6 @@ pub struct Vecs {
pub height_to_height: EagerVec<Height, Height>,
pub height_to_timestamp_fixed: EagerVec<Height, Timestamp>,
pub height_to_txindex_count: EagerVec<Height, StoredU64>,
pub txinindex_to_txinindex: LazyVecFrom1<TxInIndex, TxInIndex, TxInIndex, TxOutIndex>,
pub monthindex_to_dateindex_count: EagerVec<MonthIndex, StoredU64>,
pub monthindex_to_first_dateindex: EagerVec<MonthIndex, DateIndex>,
pub monthindex_to_monthindex: EagerVec<MonthIndex, MonthIndex>,
@@ -55,7 +55,6 @@ pub struct Vecs {
pub monthindex_to_yearindex: EagerVec<MonthIndex, YearIndex>,
pub opreturnindex_to_opreturnindex:
LazyVecFrom1<OpReturnIndex, OpReturnIndex, OpReturnIndex, TxIndex>,
pub txoutindex_to_txoutindex: LazyVecFrom1<TxOutIndex, TxOutIndex, TxOutIndex, Sats>,
pub p2aaddressindex_to_p2aaddressindex:
LazyVecFrom1<P2AAddressIndex, P2AAddressIndex, P2AAddressIndex, P2ABytes>,
pub p2msoutputindex_to_p2msoutputindex:
@@ -85,6 +84,10 @@ pub struct Vecs {
pub txindex_to_output_count:
LazyVecFrom2<TxIndex, StoredU64, TxIndex, TxOutIndex, TxOutIndex, Sats>,
pub txindex_to_txindex: LazyVecFrom1<TxIndex, TxIndex, TxIndex, Txid>,
pub txinindex_to_txinindex: LazyVecFrom1<TxInIndex, TxInIndex, TxInIndex, OutPoint>,
pub txinindex_to_txoutindex:
LazyVecFrom2<TxInIndex, TxOutIndex, TxInIndex, OutPoint, TxIndex, TxOutIndex>,
pub txoutindex_to_txoutindex: LazyVecFrom1<TxOutIndex, TxOutIndex, TxOutIndex, Sats>,
pub unknownoutputindex_to_unknownoutputindex:
LazyVecFrom1<UnknownOutputIndex, UnknownOutputIndex, UnknownOutputIndex, TxIndex>,
pub weekindex_to_dateindex_count: EagerVec<WeekIndex, StoredU64>,
@@ -97,36 +100,62 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(parent: &Path, version: Version, indexer: &Indexer) -> Result<Self> {
pub fn forced_import(
parent: &Path,
parent_version: Version,
indexer: &Indexer,
) -> Result<Self> {
let db = Database::open(&parent.join("indexes"))?;
db.set_min_len(PAGE_SIZE * 10_000_000)?;
let version = parent_version + VERSION;
let txinindex_to_txoutindex = LazyVecFrom2::init(
"txoutindex",
version,
indexer.vecs.txinindex_to_outpoint.boxed_clone(),
indexer.vecs.txindex_to_first_txoutindex.boxed_clone(),
|index: TxInIndex, txinindex_to_outpoint_iter, txindex_to_first_txoutindex_iter| {
txinindex_to_outpoint_iter
.next_at(index.to_usize())
.map(|(_, outpoint)| {
let outpoint = outpoint.into_owned();
txindex_to_first_txoutindex_iter
.next_at(outpoint.txindex().to_usize())
.unwrap()
.1
.into_owned()
+ outpoint.vout()
})
},
);
let txoutindex_to_txoutindex = LazyVecFrom1::init(
"txoutindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.txoutindex_to_value.boxed_clone(),
|index, _| Some(index),
);
let txinindex_to_txinindex = LazyVecFrom1::init(
"txinindex",
version + VERSION + Version::ZERO,
indexer.vecs.txinindex_to_txoutindex.boxed_clone(),
version + Version::ZERO,
indexer.vecs.txinindex_to_outpoint.boxed_clone(),
|index, _| Some(index),
);
let txindex_to_txindex = LazyVecFrom1::init(
"txindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.txindex_to_txid.boxed_clone(),
|index, _| Some(index),
);
let txindex_to_input_count = LazyVecFrom2::init(
"input_count",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.txindex_to_first_txinindex.boxed_clone(),
indexer.vecs.txinindex_to_txoutindex.boxed_clone(),
txinindex_to_txoutindex.boxed_clone(),
|index: TxIndex, txindex_to_first_txinindex_iter, txinindex_to_txoutindex_iter| {
let txindex = index.to_usize();
txindex_to_first_txinindex_iter
@@ -144,7 +173,7 @@ impl Vecs {
let txindex_to_output_count = LazyVecFrom2::init(
"output_count",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.txindex_to_first_txoutindex.boxed_clone(),
indexer.vecs.txoutindex_to_value.boxed_clone(),
|index: TxIndex, txindex_to_first_txoutindex_iter, txoutindex_to_value_iter| {
@@ -164,78 +193,79 @@ impl Vecs {
let p2pk33addressindex_to_p2pk33addressindex = LazyVecFrom1::init(
"p2pk33addressindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.p2pk33addressindex_to_p2pk33bytes.boxed_clone(),
|index, _| Some(index),
);
let p2pk65addressindex_to_p2pk65addressindex = LazyVecFrom1::init(
"p2pk65addressindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.p2pk65addressindex_to_p2pk65bytes.boxed_clone(),
|index, _| Some(index),
);
let p2pkhaddressindex_to_p2pkhaddressindex = LazyVecFrom1::init(
"p2pkhaddressindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.p2pkhaddressindex_to_p2pkhbytes.boxed_clone(),
|index, _| Some(index),
);
let p2shaddressindex_to_p2shaddressindex = LazyVecFrom1::init(
"p2shaddressindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.p2shaddressindex_to_p2shbytes.boxed_clone(),
|index, _| Some(index),
);
let p2traddressindex_to_p2traddressindex = LazyVecFrom1::init(
"p2traddressindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.p2traddressindex_to_p2trbytes.boxed_clone(),
|index, _| Some(index),
);
let p2wpkhaddressindex_to_p2wpkhaddressindex = LazyVecFrom1::init(
"p2wpkhaddressindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.p2wpkhaddressindex_to_p2wpkhbytes.boxed_clone(),
|index, _| Some(index),
);
let p2wshaddressindex_to_p2wshaddressindex = LazyVecFrom1::init(
"p2wshaddressindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.p2wshaddressindex_to_p2wshbytes.boxed_clone(),
|index, _| Some(index),
);
let p2aaddressindex_to_p2aaddressindex = LazyVecFrom1::init(
"p2aaddressindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.p2aaddressindex_to_p2abytes.boxed_clone(),
|index, _| Some(index),
);
let p2msoutputindex_to_p2msoutputindex = LazyVecFrom1::init(
"p2msoutputindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.p2msoutputindex_to_txindex.boxed_clone(),
|index, _| Some(index),
);
let emptyoutputindex_to_emptyoutputindex = LazyVecFrom1::init(
"emptyoutputindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.emptyoutputindex_to_txindex.boxed_clone(),
|index, _| Some(index),
);
let unknownoutputindex_to_unknownoutputindex = LazyVecFrom1::init(
"unknownoutputindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.unknownoutputindex_to_txindex.boxed_clone(),
|index, _| Some(index),
);
let opreturnindex_to_opreturnindex = LazyVecFrom1::init(
"opreturnindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
indexer.vecs.opreturnindex_to_txindex.boxed_clone(),
|index, _| Some(index),
);
let this = Self {
txinindex_to_txoutindex,
emptyoutputindex_to_emptyoutputindex,
txinindex_to_txinindex,
opreturnindex_to_opreturnindex,
@@ -257,207 +287,207 @@ impl Vecs {
dateindex_to_date: EagerVec::forced_import_compressed(
&db,
"date",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
dateindex_to_dateindex: EagerVec::forced_import_compressed(
&db,
"dateindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
dateindex_to_first_height: EagerVec::forced_import_compressed(
&db,
"first_height",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
dateindex_to_monthindex: EagerVec::forced_import_compressed(
&db,
"monthindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
dateindex_to_weekindex: EagerVec::forced_import_compressed(
&db,
"weekindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
decadeindex_to_decadeindex: EagerVec::forced_import_compressed(
&db,
"decadeindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
decadeindex_to_first_yearindex: EagerVec::forced_import_compressed(
&db,
"first_yearindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
difficultyepoch_to_difficultyepoch: EagerVec::forced_import_compressed(
&db,
"difficultyepoch",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
difficultyepoch_to_first_height: EagerVec::forced_import_compressed(
&db,
"first_height",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
halvingepoch_to_first_height: EagerVec::forced_import_compressed(
&db,
"first_height",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
halvingepoch_to_halvingepoch: EagerVec::forced_import_compressed(
&db,
"halvingepoch",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
height_to_date: EagerVec::forced_import_compressed(
&db,
"date",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
height_to_difficultyepoch: EagerVec::forced_import_compressed(
&db,
"difficultyepoch",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
height_to_halvingepoch: EagerVec::forced_import_compressed(
&db,
"halvingepoch",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
height_to_height: EagerVec::forced_import_compressed(
&db,
"height",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
monthindex_to_first_dateindex: EagerVec::forced_import_compressed(
&db,
"first_dateindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
monthindex_to_monthindex: EagerVec::forced_import_compressed(
&db,
"monthindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
monthindex_to_quarterindex: EagerVec::forced_import_compressed(
&db,
"quarterindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
monthindex_to_semesterindex: EagerVec::forced_import_compressed(
&db,
"semesterindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
monthindex_to_yearindex: EagerVec::forced_import_compressed(
&db,
"yearindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
quarterindex_to_first_monthindex: EagerVec::forced_import_compressed(
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
semesterindex_to_first_monthindex: EagerVec::forced_import_compressed(
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
weekindex_to_first_dateindex: EagerVec::forced_import_compressed(
&db,
"first_dateindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
yearindex_to_first_monthindex: EagerVec::forced_import_compressed(
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
quarterindex_to_quarterindex: EagerVec::forced_import_compressed(
&db,
"quarterindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
semesterindex_to_semesterindex: EagerVec::forced_import_compressed(
&db,
"semesterindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
weekindex_to_weekindex: EagerVec::forced_import_compressed(
&db,
"weekindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
yearindex_to_decadeindex: EagerVec::forced_import_compressed(
&db,
"decadeindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
yearindex_to_yearindex: EagerVec::forced_import_compressed(
&db,
"yearindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
height_to_date_fixed: EagerVec::forced_import_compressed(
&db,
"date_fixed",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
height_to_dateindex: EagerVec::forced_import_compressed(
&db,
"dateindex",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
height_to_timestamp_fixed: EagerVec::forced_import_compressed(
&db,
"timestamp_fixed",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
height_to_txindex_count: EagerVec::forced_import_compressed(
&db,
"txindex_count",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
dateindex_to_height_count: EagerVec::forced_import_compressed(
&db,
"height_count",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
weekindex_to_dateindex_count: EagerVec::forced_import_compressed(
&db,
"dateindex_count",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
difficultyepoch_to_height_count: EagerVec::forced_import_compressed(
&db,
"height_count",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
monthindex_to_dateindex_count: EagerVec::forced_import_compressed(
&db,
"dateindex_count",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
quarterindex_to_monthindex_count: EagerVec::forced_import_compressed(
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
semesterindex_to_monthindex_count: EagerVec::forced_import_compressed(
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
yearindex_to_monthindex_count: EagerVec::forced_import_compressed(
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
decadeindex_to_yearindex_count: EagerVec::forced_import_compressed(
&db,
"yearindex_count",
version + VERSION + Version::ZERO,
version + Version::ZERO,
)?,
db,
};
+70 -45
View File
@@ -1,13 +1,13 @@
#![doc = include_str!("../README.md")]
use std::path::Path;
use std::{path::Path, thread};
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_reader::Reader;
use brk_structs::Version;
use brk_traversable::Traversable;
use brk_types::Version;
use log::info;
use vecdb::{Exit, Format};
@@ -59,53 +59,78 @@ impl Computer {
let computed_path = outputs_path.join("computed");
let indexes =
indexes::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, indexer)?;
let (indexes, fetched, blks) = thread::scope(|s| -> Result<_> {
let fetched_handle = fetcher.map(|fetcher| {
s.spawn(move || fetched::Vecs::forced_import(outputs_path, fetcher, VERSION))
});
let fetched = fetcher.map(|fetcher| {
fetched::Vecs::forced_import(outputs_path, fetcher, VERSION + Version::ZERO).unwrap()
});
let blks_handle = s.spawn(|| blks::Vecs::forced_import(&computed_path, VERSION));
let price = fetched.is_some().then(|| {
price::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, &indexes).unwrap()
});
let indexes = indexes::Vecs::forced_import(&computed_path, VERSION, indexer)?;
let fetched = fetched_handle.map(|h| h.join().unwrap()).transpose()?;
let blks = blks_handle.join().unwrap()?;
import in theads
Ok((indexes, fetched, blks))
})?;
let (price, constants, market) = thread::scope(|s| -> Result<_> {
let constants_handle =
s.spawn(|| constants::Vecs::forced_import(&computed_path, VERSION, &indexes));
let market_handle =
s.spawn(|| market::Vecs::forced_import(&computed_path, VERSION, &indexes));
let price = fetched
.is_some()
.then(|| price::Vecs::forced_import(&computed_path, VERSION, &indexes).unwrap());
let constants = constants_handle.join().unwrap()?;
let market = market_handle.join().unwrap()?;
Ok((price, constants, market))
})?;
let (chain, pools, cointime) = thread::scope(|s| -> Result<_> {
let chain_handle = s.spawn(|| {
chain::Vecs::forced_import(
&computed_path,
VERSION,
indexer,
&indexes,
price.as_ref(),
)
});
let pools_handle = s.spawn(|| {
pools::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref())
});
let cointime =
cointime::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref())?;
let chain = chain_handle.join().unwrap()?;
let pools = pools_handle.join().unwrap()?;
Ok((chain, pools, cointime))
})?;
// Threads inside
let stateful = stateful::Vecs::forced_import(
&computed_path,
VERSION,
Format::Compressed,
&indexes,
price.as_ref(),
)?;
Ok(Self {
constants: constants::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
&indexes,
)?,
market: market::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, &indexes)?,
stateful: stateful::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
Format::Compressed,
&indexes,
price.as_ref(),
)?,
chain: chain::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
indexer,
&indexes,
price.as_ref(),
)?,
blks: blks::Vecs::forced_import(&computed_path, VERSION + Version::ZERO)?,
pools: pools::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
&indexes,
price.as_ref(),
)?,
cointime: cointime::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
&indexes,
price.as_ref(),
)?,
constants,
market,
stateful,
chain,
blks,
pools,
cointime,
indexes,
fetched,
price,
@@ -137,7 +162,7 @@ impl Computer {
info!("Computing BLKs metadata...");
self.blks
.compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
.compute(indexer, &starting_indexes, parser, exit)?;
std::thread::scope(|scope| -> Result<()> {
let constants = scope.spawn(|| -> Result<()> {
+1 -1
View File
@@ -1,8 +1,8 @@
use std::{path::Path, thread};
use brk_error::Result;
use brk_structs::{Date, DateIndex, Dollars, Height, Sats, StoredF32, StoredU16, Version};
use brk_traversable::Traversable;
use brk_types::{Date, DateIndex, Dollars, Height, Sats, StoredF32, StoredU16, Version};
use vecdb::{Database, EagerVec, Exit, PAGE_SIZE, StoredIndex, VecIterator};
use crate::{
+1 -1
View File
@@ -4,8 +4,8 @@ use allocative::Allocative;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_store::AnyStore;
use brk_structs::{Address, AddressBytes, Height, OutputType, PoolId, Pools, TxOutIndex, pools};
use brk_traversable::Traversable;
use brk_types::{Address, AddressBytes, Height, OutputType, PoolId, Pools, TxOutIndex, pools};
use rayon::prelude::*;
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec, PAGE_SIZE, RawVec,
+1 -1
View File
@@ -1,7 +1,7 @@
use allocative::Allocative;
use brk_error::Result;
use brk_structs::{Height, PoolId, Pools, Sats, StoredF32, StoredU16, StoredU32};
use brk_traversable::Traversable;
use brk_types::{Height, PoolId, Pools, Sats, StoredF32, StoredU16, StoredU32};
use vecdb::{AnyIterableVec, Database, Exit, StoredIndex, VecIterator, Version};
use crate::{
+2 -2
View File
@@ -1,11 +1,11 @@
use std::path::Path;
use brk_error::Result;
use brk_structs::{
use brk_traversable::Traversable;
use brk_types::{
Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex,
OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, SemesterIndex, Version, WeekIndex, YearIndex,
};
use brk_traversable::Traversable;
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, PAGE_SIZE,
RawVec,
@@ -1,8 +1,8 @@
use std::path::Path;
use brk_error::Result;
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit, Format, GenericStoredVec,
VecIterator,
@@ -2,8 +2,8 @@ use std::path::Path;
use brk_error::Result;
use brk_grouper::{AddressGroups, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, Filtered};
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use derive_deref::{Deref, DerefMut};
use vecdb::{AnyIterableVec, Database, Exit, Format};
@@ -1,5 +1,5 @@
use brk_grouper::ByAddressType;
use brk_structs::Height;
use brk_types::Height;
use derive_deref::{Deref, DerefMut};
use vecdb::VecIterator;
@@ -1,7 +1,7 @@
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_structs::{Height, StoredU64};
use brk_traversable::Traversable;
use brk_types::{Height, StoredU64};
use derive_deref::{Deref, DerefMut};
use vecdb::{EagerVec, Exit, GenericStoredVec};
@@ -1,6 +1,6 @@
use std::collections::BTreeMap;
use brk_structs::Height;
use brk_types::Height;
use derive_deref::{Deref, DerefMut};
use crate::stateful::AddressTypeToVec;
@@ -1,7 +1,7 @@
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_structs::StoredU64;
use brk_traversable::Traversable;
use brk_types::StoredU64;
use derive_deref::{Deref, DerefMut};
use vecdb::Exit;
@@ -1,7 +1,7 @@
use std::{collections::BTreeMap, mem};
use brk_grouper::ByAddressType;
use brk_structs::TypeIndex;
use brk_types::TypeIndex;
use derive_deref::{Deref, DerefMut};
#[derive(Debug, Deref, DerefMut)]
+2 -2
View File
@@ -1,8 +1,8 @@
use brk_error::{Error, Result};
use brk_structs::{
use brk_traversable::Traversable;
use brk_types::{
Bitcoin, DateIndex, Dollars, Height, Sats, StoredF32, StoredF64, StoredU64, Version,
};
use brk_traversable::Traversable;
use vecdb::{
AnyCloneableIterableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit,
Format, GenericStoredVec, VecIterator,
+15 -15
View File
@@ -9,14 +9,14 @@ use std::{
use brk_error::Result;
use brk_grouper::{ByAddressType, ByAnyAddress, Filtered};
use brk_indexer::Indexer;
use brk_structs::{
use brk_traversable::Traversable;
use brk_types::{
AnyAddressDataIndexEnum, AnyAddressIndex, CheckedSub, DateIndex, Dollars, EmptyAddressData,
EmptyAddressIndex, Height, LoadedAddressData, LoadedAddressIndex, OutputType, P2AAddressIndex,
P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex,
P2WPKHAddressIndex, P2WSHAddressIndex, Sats, StoredU64, Timestamp, TxInIndex, TxOutIndex,
TypeIndex, Version,
};
use brk_traversable::Traversable;
use log::info;
use rayon::prelude::*;
use vecdb::{
@@ -544,9 +544,9 @@ impl Vecs {
let height_to_input_count = chain.indexes_to_input_count.height.unwrap_sum();
let txinindex_to_txoutindex = &indexer.vecs.txinindex_to_txoutindex;
let txoutindex_to_value = &indexer.vecs.txoutindex_to_value;
let txindex_to_height = &indexes.txindex_to_height;
let txindex_to_height = &indexer.vecs.txindex_to_height;
let height_to_timestamp_fixed = &indexes.height_to_timestamp_fixed;
let txoutindex_to_txindex = &indexes.txoutindex_to_txindex;
let txoutindex_to_txindex = &indexer.vecs.txoutindex_to_txindex;
let txoutindex_to_outputtype = &indexer.vecs.txoutindex_to_outputtype;
let txoutindex_to_typeindex = &indexer.vecs.txoutindex_to_typeindex;
let height_to_unclaimed_rewards = chain
@@ -1506,28 +1506,28 @@ impl Vecs {
let anyaddressindex = match address_type {
OutputType::P2PK33 => {
p2pk33addressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
p2pk33addressindex_to_anyaddressindex.get_any_or_read(typeindex.into(), reader)
}
OutputType::P2PK65 => {
p2pk65addressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
p2pk65addressindex_to_anyaddressindex.get_any_or_read(typeindex.into(), reader)
}
OutputType::P2PKH => {
p2pkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
p2pkhaddressindex_to_anyaddressindex.get_any_or_read(typeindex.into(), reader)
}
OutputType::P2SH => {
p2shaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
p2shaddressindex_to_anyaddressindex.get_any_or_read(typeindex.into(), reader)
}
OutputType::P2TR => {
p2traddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
p2traddressindex_to_anyaddressindex.get_any_or_read(typeindex.into(), reader)
}
OutputType::P2WPKH => {
p2wpkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
p2wpkhaddressindex_to_anyaddressindex.get_any_or_read(typeindex.into(), reader)
}
OutputType::P2WSH => {
p2wshaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
p2wshaddressindex_to_anyaddressindex.get_any_or_read(typeindex.into(), reader)
}
OutputType::P2A => {
p2aaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
p2aaddressindex_to_anyaddressindex.get_any_or_read(typeindex.into(), reader)
}
_ => unreachable!(),
}
@@ -1543,7 +1543,7 @@ impl Vecs {
.unwrap();
let loadedaddressdata = loadedaddressindex_to_loadedaddressdata
.get_or_read(loadedaddressindex, mmap)
.get_any_or_read(loadedaddressindex, mmap)
.unwrap()
.unwrap()
.into_owned();
@@ -1560,7 +1560,7 @@ impl Vecs {
.unwrap();
let emptyaddressdata = emptyaddressindex_to_emptyaddressdata
.get_or_read(emtpyaddressindex, mmap)
.get_any_or_read(emtpyaddressindex, mmap)
.unwrap()
.unwrap()
.into_owned();
@@ -1956,7 +1956,7 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
price: Option<Dollars>,
addresstype_to_addr_count: &mut ByAddressType<u64>,
addresstype_to_empty_addr_count: &mut ByAddressType<u64>,
height_to_price_close_vec: Option<&Vec<brk_structs::Close<Dollars>>>,
height_to_price_close_vec: Option<&Vec<brk_types::Close<Dollars>>>,
height_to_timestamp_fixed_vec: &[Timestamp],
height: Height,
timestamp: Timestamp,
+1 -1
View File
@@ -1,5 +1,5 @@
use brk_error::Result;
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version};
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use vecdb::{AnyIterableVec, Exit};
use crate::{Indexes, indexes, price};
@@ -1,8 +1,8 @@
use std::{ops::Deref, path::Path};
use brk_error::Result;
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version};
use brk_traversable::Traversable;
use brk_types::{Bitcoin, DateIndex, Dollars, Height, Version};
use vecdb::{AnyIterableVec, Database, Exit, Format};
use crate::{
@@ -5,10 +5,10 @@ use brk_grouper::{
ByAgeRange, ByAmountRange, ByEpoch, ByGreatEqualAmount, ByLowerThanAmount, ByMaxAge, ByMinAge,
BySpendableType, ByTerm, Filter, Filtered, UTXOGroups,
};
use brk_structs::{
use brk_traversable::Traversable;
use brk_types::{
Bitcoin, CheckedSub, DateIndex, Dollars, HalvingEpoch, Height, Timestamp, Version,
};
use brk_traversable::Traversable;
use derive_deref::{Deref, DerefMut};
use vecdb::{AnyIterableVec, Database, Exit, Format, StoredIndex};
@@ -1,4 +1,4 @@
use brk_structs::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex};
use brk_types::{EmptyAddressData, EmptyAddressIndex, LoadedAddressData, LoadedAddressIndex};
#[derive(Debug)]
pub enum WithAddressDataSource<T> {
+1 -1
View File
@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign, SubAssign};
use brk_structs::{Dollars, Timestamp};
use brk_types::{Dollars, Timestamp};
use serde::Serialize;
use super::SupplyState;
@@ -1,7 +1,7 @@
use std::path::Path;
use brk_error::Result;
use brk_structs::{Dollars, Height, LoadedAddressData, Sats};
use brk_types::{Dollars, Height, LoadedAddressData, Sats};
use crate::SupplyState;
@@ -1,7 +1,7 @@
use std::{cmp::Ordering, path::Path};
use brk_error::Result;
use brk_structs::{CheckedSub, Dollars, Height, Sats};
use brk_types::{CheckedSub, Dollars, Height, Sats};
use crate::{PriceToAmount, RealizedState, SupplyState, UnrealizedState};
@@ -5,7 +5,7 @@ use std::{
};
use brk_error::{Error, Result};
use brk_structs::{Dollars, Height, Sats};
use brk_types::{Dollars, Height, Sats};
use derive_deref::{Deref, DerefMut};
use pco::standalone::{simple_decompress, simpler_compress};
use serde::{Deserialize, Serialize};
+1 -1
View File
@@ -1,6 +1,6 @@
use std::cmp::Ordering;
use brk_structs::{CheckedSub, Dollars};
use brk_types::{CheckedSub, Dollars};
use super::SupplyState;
+1 -1
View File
@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign, SubAssign};
use brk_structs::{CheckedSub, LoadedAddressData, Sats};
use brk_types::{CheckedSub, LoadedAddressData, Sats};
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
+1 -1
View File
@@ -1,7 +1,7 @@
use std::ops::{Add, AddAssign};
use brk_grouper::{ByAmountRange, GroupedByType};
use brk_structs::{OutputType, Sats};
use brk_types::{OutputType, Sats};
use super::SupplyState;
+1 -1
View File
@@ -1,4 +1,4 @@
use brk_structs::{Dollars, Sats};
use brk_types::{Dollars, Sats};
#[derive(Debug, Default, Clone)]
pub struct UnrealizedState {
+1 -1
View File
@@ -1,5 +1,5 @@
use brk_error::Result;
use brk_structs::{Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Sats, StoredF32};
use brk_types::{Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Sats, StoredF32};
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, StoredIndex,
VecIterator, Version,