Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-11 19:18:14 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer: cleanup

Browse Source
This commit is contained in:
nym21
2025-08-16 16:42:01 +02:00
parent 8648d3131a
commit 009fb35c4c
18 changed files with 225 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_computer/src/blocks.rs
+6 -15
View File
@@ -6,7 +6,7 @@ use brk_structs::{
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, Timestamp, Version,
Weight,
};
use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, Format, PAGE_SIZE, VecIterator};
use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, PAGE_SIZE, VecIterator};
use crate::grouped::Source;
@@ -35,21 +35,15 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(
parent: &Path,
version: Version,
format: Format,
indexes: &indexes::Vecs,
) -> Result<Self> {
pub fn forced_import(parent: &Path, version: Version, indexes: &indexes::Vecs) -> Result<Self> {
let db = Database::open(&parent.join("blocks"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
height_to_interval: EagerVec::forced_import(
height_to_interval: EagerVec::forced_import_compressed(
&db,
"interval",
version + VERSION + Version::ZERO,
format,
)?,
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
&db,
@@ -94,11 +88,10 @@ impl Vecs {
indexes,
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
height_to_vbytes: EagerVec::forced_import(
height_to_vbytes: EagerVec::forced_import_compressed(
&db,
"vbytes",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
&db,
@@ -108,17 +101,15 @@ impl Vecs {
indexes,
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
difficultyepoch_to_timestamp: EagerVec::forced_import(
difficultyepoch_to_timestamp: EagerVec::forced_import_compressed(
&db,
"timestamp",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_timestamp: EagerVec::forced_import(
halvingepoch_to_timestamp: EagerVec::forced_import_compressed(
&db,
"timestamp",
version + VERSION + Version::ZERO,
format,
)?,
db,
crates/brk_computer/src/grouped/builder_eager.rs
+9
View File
@@ -36,6 +36,15 @@ where
I: StoredIndex,
T: ComputedType,
{
pub fn forced_import_compressed(
db: &Database,
name: &str,
version: Version,
options: VecBuilderOptions,
) -> Result<Self> {
Self::forced_import(db, name, version, Format::Compressed, options)
}
pub fn forced_import(
db: &Database,
name: &str,
crates/brk_computer/src/grouped/from_dateindex.rs
+3 -8
View File
@@ -4,9 +4,7 @@ use brk_indexer::Indexer;
use brk_structs::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex,
};
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit, Format,
};
use vecdb::{AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit};
use crate::{Indexes, grouped::LazyVecBuilder, indexes};
@@ -42,17 +40,14 @@ where
indexes: &indexes::Vecs,
options: VecBuilderOptions,
) -> Result<Self> {
let format = Format::Compressed;
let dateindex = source.is_compute().then(|| {
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format).unwrap()
EagerVec::forced_import_compressed(db, name, version + VERSION + Version::ZERO).unwrap()
});
let dateindex_extra = EagerVecBuilder::forced_import(
let dateindex_extra = EagerVecBuilder::forced_import_compressed(
db,
name,
version + VERSION + Version::ZERO,
format,
options.copy_self_extra(),
)?;
crates/brk_computer/src/grouped/from_height.rs
+5 -12
View File
@@ -5,9 +5,7 @@ use brk_structs::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex,
Version, WeekIndex, YearIndex,
};
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit, Format,
};
use vecdb::{AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit};
use crate::{
Indexes,
@@ -51,25 +49,21 @@ where
indexes: &indexes::Vecs,
options: VecBuilderOptions,
) -> Result<Self> {
let format = Format::Compressed;
let height = source.is_compute().then(|| {
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format).unwrap()
EagerVec::forced_import_compressed(db, name, version + VERSION + Version::ZERO).unwrap()
});
let height_extra = EagerVecBuilder::forced_import(
let height_extra = EagerVecBuilder::forced_import_compressed(
db,
name,
version + VERSION + Version::ZERO,
format,
options.copy_self_extra(),
)?;
let dateindex = EagerVecBuilder::forced_import(
let dateindex = EagerVecBuilder::forced_import_compressed(
db,
name,
version + VERSION + Version::ZERO,
format,
options,
)?;
@@ -128,11 +122,10 @@ where
height,
height_extra,
dateindex,
difficultyepoch: EagerVecBuilder::forced_import(
difficultyepoch: EagerVecBuilder::forced_import_compressed(
db,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
})
crates/brk_computer/src/grouped/from_height_strict.rs
+5 -8
View File
@@ -2,7 +2,7 @@ use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{DifficultyEpoch, Height, Version};
use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, Format};
use vecdb::{AnyCollectableVec, Database, EagerVec, Exit};
use crate::{Indexes, indexes};
@@ -32,15 +32,13 @@ where
version: Version,
options: VecBuilderOptions,
) -> Result<Self> {
let format = Format::Compressed;
let height =
EagerVec::forced_import_compressed(db, name, version + VERSION + Version::ZERO)?;
let height = EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)?;
let height_extra = EagerVecBuilder::forced_import(
let height_extra = EagerVecBuilder::forced_import_compressed(
db,
name,
version + VERSION + Version::ZERO,
format,
options.copy_self_extra(),
)?;
@@ -49,11 +47,10 @@ where
Ok(Self {
height,
height_extra,
difficultyepoch: EagerVecBuilder::forced_import(
difficultyepoch: EagerVecBuilder::forced_import_compressed(
db,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
crates/brk_computer/src/grouped/from_txindex.rs
+5 -9
View File
@@ -6,7 +6,7 @@ use brk_structs::{
};
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyVec, CollectableVec, Database, EagerVec, Exit,
Format, GenericStoredVec, StoredIndex, VecIterator,
GenericStoredVec, StoredIndex, VecIterator,
};
use crate::{
@@ -48,32 +48,29 @@ where
name: &str,
source: Source<TxIndex, T>,
version: Version,
format: Format,
indexes: &indexes::Vecs,
options: VecBuilderOptions,
) -> Result<Self> {
let txindex = source.is_compute().then(|| {
Box::new(
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)
EagerVec::forced_import_compressed(db, name, version + VERSION + Version::ZERO)
.unwrap(),
)
});
let height = EagerVecBuilder::forced_import(
let height = EagerVecBuilder::forced_import_compressed(
db,
name,
version + VERSION + Version::ZERO,
format,
options,
)?;
let options = options.remove_percentiles();
let dateindex = EagerVecBuilder::forced_import(
let dateindex = EagerVecBuilder::forced_import_compressed(
db,
name,
version + VERSION + Version::ZERO,
format,
options,
)?;
@@ -130,11 +127,10 @@ where
txindex,
height,
dateindex,
difficultyepoch: EagerVecBuilder::forced_import(
difficultyepoch: EagerVecBuilder::forced_import_compressed(
db,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
crates/brk_computer/src/grouped/value_from_txindex.rs
+2 -6
View File
@@ -2,8 +2,8 @@ use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, CollectableVec, Database, Exit, Format,
LazyVecFrom1, LazyVecFrom3, StoredIndex, StoredVec,
AnyCloneableIterableVec, AnyCollectableVec, CollectableVec, Database, Exit, LazyVecFrom1,
LazyVecFrom3, StoredIndex, StoredVec,
};
use crate::{Indexes, grouped::Source, indexes, price};
@@ -32,7 +32,6 @@ impl ComputedValueVecsFromTxindex {
indexes: &indexes::Vecs,
source: Source<TxIndex, Sats>,
version: Version,
format: Format,
price: Option<&price::Vecs>,
options: VecBuilderOptions,
) -> Result<Self> {
@@ -46,7 +45,6 @@ impl ComputedValueVecsFromTxindex {
name,
source.clone(),
version + VERSION,
format,
indexes,
options,
)?;
@@ -70,7 +68,6 @@ impl ComputedValueVecsFromTxindex {
&name_in_btc,
Source::None,
version + VERSION,
format,
indexes,
options,
)?;
@@ -113,7 +110,6 @@ impl ComputedValueVecsFromTxindex {
&name_in_usd,
Source::None,
version + VERSION,
format,
indexes,
options,
)
crates/brk_computer/src/indexes.rs
+44 -87
View File
@@ -11,7 +11,7 @@ use brk_structs::{
Timestamp, TxIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex,
};
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, Database, EagerVec, Exit, Format, LazyVecFrom1,
AnyCloneableIterableVec, AnyCollectableVec, Database, EagerVec, Exit, LazyVecFrom1,
LazyVecFrom2, PAGE_SIZE, StoredIndex, VecIterator,
};
@@ -255,263 +255,220 @@ impl Vecs {
txindex_to_txindex,
unknownoutputindex_to_unknownoutputindex,
dateindex_to_date: EagerVec::forced_import(
dateindex_to_date: EagerVec::forced_import_compressed(
&db,
"date",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
dateindex_to_dateindex: EagerVec::forced_import(
dateindex_to_dateindex: EagerVec::forced_import_compressed(
&db,
"dateindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
dateindex_to_first_height: EagerVec::forced_import(
dateindex_to_first_height: EagerVec::forced_import_compressed(
&db,
"first_height",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
dateindex_to_monthindex: EagerVec::forced_import(
dateindex_to_monthindex: EagerVec::forced_import_compressed(
&db,
"monthindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
dateindex_to_weekindex: EagerVec::forced_import(
dateindex_to_weekindex: EagerVec::forced_import_compressed(
&db,
"weekindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
decadeindex_to_decadeindex: EagerVec::forced_import(
decadeindex_to_decadeindex: EagerVec::forced_import_compressed(
&db,
"decadeindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
decadeindex_to_first_yearindex: EagerVec::forced_import(
decadeindex_to_first_yearindex: EagerVec::forced_import_compressed(
&db,
"first_yearindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
difficultyepoch_to_difficultyepoch: EagerVec::forced_import(
difficultyepoch_to_difficultyepoch: EagerVec::forced_import_compressed(
&db,
"difficultyepoch",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
difficultyepoch_to_first_height: EagerVec::forced_import(
difficultyepoch_to_first_height: EagerVec::forced_import_compressed(
&db,
"first_height",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
halvingepoch_to_first_height: EagerVec::forced_import(
halvingepoch_to_first_height: EagerVec::forced_import_compressed(
&db,
"first_height",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
halvingepoch_to_halvingepoch: EagerVec::forced_import(
halvingepoch_to_halvingepoch: EagerVec::forced_import_compressed(
&db,
"halvingepoch",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
height_to_date: EagerVec::forced_import(
height_to_date: EagerVec::forced_import_compressed(
&db,
"date",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
height_to_difficultyepoch: EagerVec::forced_import(
height_to_difficultyepoch: EagerVec::forced_import_compressed(
&db,
"difficultyepoch",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
height_to_halvingepoch: EagerVec::forced_import(
height_to_halvingepoch: EagerVec::forced_import_compressed(
&db,
"halvingepoch",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
height_to_height: EagerVec::forced_import(
height_to_height: EagerVec::forced_import_compressed(
&db,
"height",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
monthindex_to_first_dateindex: EagerVec::forced_import(
monthindex_to_first_dateindex: EagerVec::forced_import_compressed(
&db,
"first_dateindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
monthindex_to_monthindex: EagerVec::forced_import(
monthindex_to_monthindex: EagerVec::forced_import_compressed(
&db,
"monthindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
monthindex_to_quarterindex: EagerVec::forced_import(
monthindex_to_quarterindex: EagerVec::forced_import_compressed(
&db,
"quarterindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
monthindex_to_semesterindex: EagerVec::forced_import(
monthindex_to_semesterindex: EagerVec::forced_import_compressed(
&db,
"semesterindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
monthindex_to_yearindex: EagerVec::forced_import(
monthindex_to_yearindex: EagerVec::forced_import_compressed(
&db,
"yearindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
quarterindex_to_first_monthindex: EagerVec::forced_import(
quarterindex_to_first_monthindex: EagerVec::forced_import_compressed(
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
semesterindex_to_first_monthindex: EagerVec::forced_import(
semesterindex_to_first_monthindex: EagerVec::forced_import_compressed(
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
weekindex_to_first_dateindex: EagerVec::forced_import(
weekindex_to_first_dateindex: EagerVec::forced_import_compressed(
&db,
"first_dateindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
yearindex_to_first_monthindex: EagerVec::forced_import(
yearindex_to_first_monthindex: EagerVec::forced_import_compressed(
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
quarterindex_to_quarterindex: EagerVec::forced_import(
quarterindex_to_quarterindex: EagerVec::forced_import_compressed(
&db,
"quarterindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
semesterindex_to_semesterindex: EagerVec::forced_import(
semesterindex_to_semesterindex: EagerVec::forced_import_compressed(
&db,
"semesterindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
weekindex_to_weekindex: EagerVec::forced_import(
weekindex_to_weekindex: EagerVec::forced_import_compressed(
&db,
"weekindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
yearindex_to_decadeindex: EagerVec::forced_import(
yearindex_to_decadeindex: EagerVec::forced_import_compressed(
&db,
"decadeindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
yearindex_to_yearindex: EagerVec::forced_import(
yearindex_to_yearindex: EagerVec::forced_import_compressed(
&db,
"yearindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
height_to_date_fixed: EagerVec::forced_import(
height_to_date_fixed: EagerVec::forced_import_compressed(
&db,
"date_fixed",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
height_to_dateindex: EagerVec::forced_import(
height_to_dateindex: EagerVec::forced_import_compressed(
&db,
"dateindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
txindex_to_height: EagerVec::forced_import(
txindex_to_height: EagerVec::forced_import_compressed(
&db,
"height",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
height_to_timestamp_fixed: EagerVec::forced_import(
height_to_timestamp_fixed: EagerVec::forced_import_compressed(
&db,
"timestamp_fixed",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
height_to_txindex_count: EagerVec::forced_import(
height_to_txindex_count: EagerVec::forced_import_compressed(
&db,
"txindex_count",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
dateindex_to_height_count: EagerVec::forced_import(
dateindex_to_height_count: EagerVec::forced_import_compressed(
&db,
"height_count",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
weekindex_to_dateindex_count: EagerVec::forced_import(
weekindex_to_dateindex_count: EagerVec::forced_import_compressed(
&db,
"dateindex_count",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
difficultyepoch_to_height_count: EagerVec::forced_import(
difficultyepoch_to_height_count: EagerVec::forced_import_compressed(
&db,
"height_count",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
monthindex_to_dateindex_count: EagerVec::forced_import(
monthindex_to_dateindex_count: EagerVec::forced_import_compressed(
&db,
"dateindex_count",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
quarterindex_to_monthindex_count: EagerVec::forced_import(
quarterindex_to_monthindex_count: EagerVec::forced_import_compressed(
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
semesterindex_to_monthindex_count: EagerVec::forced_import(
semesterindex_to_monthindex_count: EagerVec::forced_import_compressed(
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
yearindex_to_monthindex_count: EagerVec::forced_import(
yearindex_to_monthindex_count: EagerVec::forced_import_compressed(
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
decadeindex_to_yearindex_count: EagerVec::forced_import(
decadeindex_to_yearindex_count: EagerVec::forced_import_compressed(
&db,
"yearindex_count",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
outputindex_to_txindex: EagerVec::forced_import(
outputindex_to_txindex: EagerVec::forced_import_compressed(
&db,
"txindex",
version + VERSION + Version::ZERO,
Format::Compressed,
)?,
db,
crates/brk_computer/src/lib.rs
+4 -18
View File
@@ -60,36 +60,23 @@ impl Computer {
fetched::Vecs::forced_import(outputs_path, fetcher, VERSION + Version::ZERO).unwrap()
});
let format = Format::Compressed;
let price = fetched.is_some().then(|| {
price::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, format, &indexes)
.unwrap()
price::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, &indexes).unwrap()
});
Ok(Self {
blocks: blocks::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
format,
&indexes,
)?,
blocks: blocks::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, &indexes)?,
mining: mining::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, &indexes)?,
constants: constants::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
&indexes,
)?,
market: market::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
format,
&indexes,
)?,
market: market::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, &indexes)?,
stateful: stateful::Vecs::forced_import(
&computed_path,
VERSION + Version::ZERO,
format,
Format::Compressed,
&indexes,
price.as_ref(),
&computed_path.join("states"),
@@ -99,7 +86,6 @@ impl Computer {
VERSION + Version::ZERO,
indexer,
&indexes,
format,
price.as_ref(),
)?,
cointime: cointime::Vecs::forced_import(
crates/brk_computer/src/market.rs
+5 -15
View File
@@ -3,9 +3,7 @@ use std::{path::Path, thread};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Date, DateIndex, Dollars, Height, Sats, StoredF32, StoredU16, Version};
use vecdb::{
AnyCollectableVec, Database, EagerVec, Exit, Format, PAGE_SIZE, StoredIndex, VecIterator,
};
use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, PAGE_SIZE, StoredIndex, VecIterator};
use crate::{
grouped::Source,
@@ -170,33 +168,25 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(
parent: &Path,
version: Version,
format: Format,
indexes: &indexes::Vecs,
) -> Result<Self> {
pub fn forced_import(parent: &Path, version: Version, indexes: &indexes::Vecs) -> Result<Self> {
let db = Database::open(&parent.join("market"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
height_to_marketcap: EagerVec::forced_import(
height_to_marketcap: EagerVec::forced_import_compressed(
&db,
"marketcap",
version + VERSION + Version::ZERO,
format,
)?,
height_to_ath: EagerVec::forced_import(
height_to_ath: EagerVec::forced_import_compressed(
&db,
"ath",
version + VERSION + Version::ZERO,
format,
)?,
height_to_drawdown: EagerVec::forced_import(
height_to_drawdown: EagerVec::forced_import_compressed(
&db,
"drawdown",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_marketcap: ComputedVecsFromDateIndex::forced_import(
&db,
crates/brk_computer/src/price.rs
+10 -23
View File
@@ -7,7 +7,7 @@ use brk_structs::{
OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, SemesterIndex, Version, WeekIndex, YearIndex,
};
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit, Format,
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit,
GenericStoredVec, PAGE_SIZE, RawVec,
};
@@ -73,12 +73,7 @@ const VERSION: Version = Version::ZERO;
const VERSION_IN_SATS: Version = Version::ZERO;
impl Vecs {
pub fn forced_import(
parent: &Path,
version: Version,
format: Format,
indexes: &indexes::Vecs,
) -> Result<Self> {
pub fn forced_import(parent: &Path, version: Version, indexes: &indexes::Vecs) -> Result<Self> {
let db = Database::open(&parent.join("price"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
@@ -93,29 +88,25 @@ impl Vecs {
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
dateindex_to_close_in_cents: EagerVec::forced_import(
dateindex_to_close_in_cents: EagerVec::forced_import_compressed(
&db,
"close_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_high_in_cents: EagerVec::forced_import(
dateindex_to_high_in_cents: EagerVec::forced_import_compressed(
&db,
"high_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_low_in_cents: EagerVec::forced_import(
dateindex_to_low_in_cents: EagerVec::forced_import_compressed(
&db,
"low_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_open_in_cents: EagerVec::forced_import(
dateindex_to_open_in_cents: EagerVec::forced_import_compressed(
&db,
"open_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_ohlc: RawVec::forced_import(&db, "ohlc", version + VERSION + Version::ZERO)?,
height_to_ohlc_in_sats: RawVec::forced_import(
@@ -123,29 +114,25 @@ impl Vecs {
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
height_to_close_in_cents: EagerVec::forced_import(
height_to_close_in_cents: EagerVec::forced_import_compressed(
&db,
"close_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_high_in_cents: EagerVec::forced_import(
height_to_high_in_cents: EagerVec::forced_import_compressed(
&db,
"high_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_low_in_cents: EagerVec::forced_import(
height_to_low_in_cents: EagerVec::forced_import_compressed(
&db,
"low_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_open_in_cents: EagerVec::forced_import(
height_to_open_in_cents: EagerVec::forced_import_compressed(
&db,
"open_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
timeindexes_to_open: ComputedVecsFromDateIndex::forced_import(
&db,
crates/brk_computer/src/stateful/mod.rs
+18 -36
View File
@@ -105,11 +105,10 @@ impl Vecs {
version + VERSION + Version::ZERO,
)?,
height_to_unspendable_supply: EagerVec::forced_import(
height_to_unspendable_supply: EagerVec::forced_import_compressed(
&db,
"unspendable_supply",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
&db,
@@ -120,11 +119,10 @@ impl Vecs {
compute_dollars,
indexes,
)?,
height_to_opreturn_supply: EagerVec::forced_import(
height_to_opreturn_supply: EagerVec::forced_import_compressed(
&db,
"opreturn_supply",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import(
&db,
@@ -153,105 +151,89 @@ impl Vecs {
)?,
addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount::from(
ByAddressType {
p2pk65: EagerVec::forced_import(
p2pk65: EagerVec::forced_import_compressed(
&db,
"p2pk65_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
p2pk33: EagerVec::forced_import_compressed(
&db,
"p2pk33_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
p2pkh: EagerVec::forced_import_compressed(
&db,
"p2pkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
p2sh: EagerVec::forced_import_compressed(
&db,
"p2sh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
p2wpkh: EagerVec::forced_import_compressed(
&db,
"p2wpkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
p2wsh: EagerVec::forced_import_compressed(
&db,
"p2wsh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
p2tr: EagerVec::forced_import_compressed(
&db,
"p2tr_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
p2a: EagerVec::forced_import_compressed(
&db,
"p2a_address_count",
version + VERSION + Version::ZERO,
format,
)?,
},
),
addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount::from(
ByAddressType {
p2pk65: EagerVec::forced_import(
p2pk65: EagerVec::forced_import_compressed(
&db,
"p2pk65_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
p2pk33: EagerVec::forced_import_compressed(
&db,
"p2pk33_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
p2pkh: EagerVec::forced_import_compressed(
&db,
"p2pkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
p2sh: EagerVec::forced_import_compressed(
&db,
"p2sh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
p2wpkh: EagerVec::forced_import_compressed(
&db,
"p2wpkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
p2wsh: EagerVec::forced_import_compressed(
&db,
"p2wsh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
p2tr: EagerVec::forced_import_compressed(
&db,
"p2tr_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
p2a: EagerVec::forced_import_compressed(
&db,
"p2a_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
},
),
crates/brk_computer/src/states/price_to_amount.rs
+21 -13
View File
@@ -2,7 +2,6 @@ use std::{
collections::BTreeMap,
fs,
path::{Path, PathBuf},
time::SystemTime,
};
use brk_error::Result;
@@ -10,6 +9,7 @@ use brk_structs::{Dollars, Height, Sats};
use derive_deref::{Deref, DerefMut};
use pco::standalone::{simple_decompress, simpler_compress};
use serde::{Deserialize, Serialize};
use zerocopy::{FromBytes, IntoBytes};
use crate::states::SupplyState;
@@ -78,23 +78,31 @@ impl PriceToAmount {
}
pub fn flush(&mut self, height: Height) -> Result<()> {
let mut files: Vec<(SystemTime, PathBuf)> = fs::read_dir(&self.pathbuf)?
let files: BTreeMap<Height, PathBuf> = fs::read_dir(&self.pathbuf)?
.filter_map(|entry| {
let path = entry.ok()?.path();
let name = path.file_name()?.to_str()?;
if name.starts_with(STATE_AT_) && name[STATE_AT_.len() + 1..].parse::<u64>().is_ok()
{
let modified = fs::metadata(&path).ok()?.modified().ok()?;
Some((modified, path))
if let Some(height_str) = name.strip_prefix(STATE_AT_) {
if let Ok(h) = height_str.parse::<u64>().map(Height::from) {
if h < height {
Some((h, path))
} else {
let _ = fs::remove_file(path);
None
}
} else {
None
}
} else {
None
}
})
.collect();
files.sort_unstable_by_key(|(time, _)| *time);
for (_, path) in files.iter().take(files.len().saturating_sub(STATE_TO_KEEP)) {
for (_, path) in files
.iter()
.take(files.len().saturating_sub(STATE_TO_KEEP - 1))
{
fs::remove_file(path)?;
}
@@ -128,8 +136,8 @@ impl State {
let compressed_values = simpler_compress(&values, COMPRESSION_LEVEL)?;
let mut buffer = Vec::new();
buffer.extend(&(keys.len() as u64).to_ne_bytes());
buffer.extend(&(compressed_keys.len() as u64).to_ne_bytes());
buffer.extend(keys.len().as_bytes());
buffer.extend(compressed_keys.len().as_bytes());
buffer.extend(compressed_keys);
buffer.extend(compressed_values);
@@ -137,8 +145,8 @@ impl State {
}
fn deserialize(data: &[u8]) -> vecdb::Result<Self> {
let entry_count = u64::from_ne_bytes(data[0..8].try_into().unwrap()) as usize;
let keys_len = u64::from_ne_bytes(data[8..16].try_into().unwrap()) as usize;
let entry_count = usize::read_from_bytes(&data[0..8])?;
let keys_len = usize::read_from_bytes(&data[8..16])?;
let keys: Vec<f64> = simple_decompress(&data[16..16 + keys_len])?;
let values: Vec<u64> = simple_decompress(&data[16 + keys_len..])?;
crates/brk_computer/src/transactions.rs
+5 -20
View File
@@ -7,7 +7,7 @@ use brk_structs::{
StoredU32, StoredU64, TxIndex, TxVersion, Version, Weight,
};
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit, Format,
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit,
LazyVecFrom1, LazyVecFrom2, LazyVecFrom3, PAGE_SIZE, StoredIndex, VecIterator,
};
@@ -75,7 +75,6 @@ impl Vecs {
version: Version,
indexer: &Indexer,
indexes: &indexes::Vecs,
format: Format,
price: Option<&price::Vecs>,
) -> Result<Self> {
let db = Database::open(&parent.join("transactions"))?;
@@ -260,19 +259,11 @@ impl Vecs {
// .add_cumulative(),
// )?;
let txindex_to_fee = EagerVec::forced_import(
&db,
"fee",
version + VERSION + Version::ZERO,
Format::Compressed,
)?;
let txindex_to_fee =
EagerVec::forced_import_compressed(&db, "fee", version + VERSION + Version::ZERO)?;
let txindex_to_feerate = EagerVec::forced_import(
&db,
"feerate",
version + VERSION + Version::ZERO,
Format::Compressed,
)?;
let txindex_to_feerate =
EagerVec::forced_import_compressed(&db, "feerate", version + VERSION + Version::ZERO)?;
Ok(Self {
indexes_to_tx_count: ComputedVecsFromHeight::forced_import(
@@ -293,7 +284,6 @@ impl Vecs {
"input_count",
Source::None,
version + VERSION + Version::ZERO,
format,
indexes,
VecBuilderOptions::default()
.add_average()
@@ -307,7 +297,6 @@ impl Vecs {
"output_count",
Source::None,
version + VERSION + Version::ZERO,
format,
indexes,
VecBuilderOptions::default()
.add_average()
@@ -346,7 +335,6 @@ impl Vecs {
indexes,
Source::Vec(txindex_to_fee.boxed_clone()),
version + VERSION + Version::ZERO,
format,
price,
VecBuilderOptions::default()
.add_sum()
@@ -360,7 +348,6 @@ impl Vecs {
"feerate",
Source::None,
version + VERSION + Version::ZERO,
format,
indexes,
VecBuilderOptions::default()
.add_percentiles()
@@ -372,7 +359,6 @@ impl Vecs {
"tx_vsize",
Source::None,
version + VERSION + Version::ZERO,
format,
indexes,
VecBuilderOptions::default()
.add_percentiles()
@@ -384,7 +370,6 @@ impl Vecs {
"tx_weight",
Source::None,
version + VERSION + Version::ZERO,
format,
indexes,
VecBuilderOptions::default()
.add_percentiles()
crates/brk_indexer/src/lib.rs
+10 -9
View File
@@ -782,16 +782,17 @@ impl Indexer {
},
)?;
txindex_to_first_outputindex_reader_opt.take();
p2pk65addressindex_to_p2pk65bytes_reader_opt.take();
p2pk33addressindex_to_p2pk33bytes_reader_opt.take();
p2pkhaddressindex_to_p2pkhbytes_reader_opt.take();
p2shaddressindex_to_p2shbytes_reader_opt.take();
p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take();
p2wshaddressindex_to_p2wshbytes_reader_opt.take();
p2traddressindex_to_p2trbytes_reader_opt.take();
p2aaddressindex_to_p2abytes_reader_opt.take();
if should_export(idxs.height, true) {
txindex_to_first_outputindex_reader_opt.take();
p2pk65addressindex_to_p2pk65bytes_reader_opt.take();
p2pk33addressindex_to_p2pk33bytes_reader_opt.take();
p2pkhaddressindex_to_p2pkhbytes_reader_opt.take();
p2shaddressindex_to_p2shbytes_reader_opt.take();
p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take();
p2wshaddressindex_to_p2wshbytes_reader_opt.take();
p2traddressindex_to_p2trbytes_reader_opt.take();
p2aaddressindex_to_p2abytes_reader_opt.take();
export(stores, vecs, idxs.height, exit)?;
}
crates/brk_server/src/api/interface.rs
+1 -1
View File
@@ -62,7 +62,7 @@ fn req_to_response_res(
if weight > MAX_WEIGHT {
return Err(Error::Str(
"Request is too heavy, max weight is {MAX_WEIGHT}",
"Request is too heavy, max weight is {MAX_WEIGHT} bytes",
));
}