Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-25 07:09:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

storable_vec: add modes

Browse Source
This commit is contained in:
nym21
2025-02-04 20:56:48 +01:00
parent 42c996e16e
commit d11a1622f8
16 changed files with 696 additions and 495 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
computer/src/lib.rs
View File

@@ -7,16 +7,17 @@ use exit::Exit;
mod storage;
mod structs;
use storable_vec::{CACHED_GETS, SINGLE_THREAD};
use storage::{Fjalls, StorableVecs};
use structs::*;
pub struct Computer {
pub struct Computer<const MODE: u8> {
outputs_dir: PathBuf,
vecs: StorableVecs,
vecs: StorableVecs<MODE>,
trees: Fjalls,
}
impl Computer {
impl<const MODE: u8> Computer<MODE> {
pub fn import(outputs_dir: &Path) -> color_eyre::Result<Self> {
let outputs_dir = outputs_dir.to_owned();
let computed_dir = outputs_dir.join("computed");
@@ -29,56 +30,63 @@ impl Computer {
})
}
pub fn compute(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
let mut indexer = Indexer::import(&self.outputs_dir.join("indexes"))?;
fn open_indexer<const MODE_IDX: u8>(&self) -> color_eyre::Result<Indexer<MODE_IDX>> {
Indexer::import(&self.outputs_dir.join("indexes"))
}
}
impl Computer<SINGLE_THREAD> {
pub fn compute(&mut self, bitcoin_dir: &Path, rpc: rpc::Client, exit: &Exit) -> color_eyre::Result<()> {
if false {
let mut indexer: Indexer<CACHED_GETS> = self.open_indexer()?;
indexer.index(bitcoin_dir, rpc, exit)?;
}
let height_count = indexer.vecs().height_to_size.len();
let txindexes_count = indexer.vecs().txindex_to_txid.len();
let txinindexes_count = indexer.vecs().txinindex_to_txoutindex.len();
let txoutindexes_count = indexer.vecs().txoutindex_to_addressindex.len();
let mut indexer: Indexer<SINGLE_THREAD> = self.open_indexer()?;
let height_count = indexer.vecs.height_to_size.len();
let txindexes_count = indexer.vecs.txindex_to_txid.len();
let txinindexes_count = indexer.vecs.txinindex_to_txoutindex.len();
let txoutindexes_count = indexer.vecs.txoutindex_to_addressindex.len();
// TODO: Remove all outdated
self.vecs
.txindex_to_last_txinindex
.compute_last_index_from_first(&indexer.vecs().txindex_to_first_txinindex, txinindexes_count)?;
.compute_last_index_from_first(&indexer.vecs.txindex_to_first_txinindex, txinindexes_count)?;
self.vecs.txindex_to_inputcount.compute_count_from_indexes(
&indexer.vecs().txindex_to_first_txinindex,
&indexer.vecs.txindex_to_first_txinindex,
&self.vecs.txindex_to_last_txinindex,
)?;
self.vecs
.txindex_to_last_txoutindex
.compute_last_index_from_first(&indexer.vecs().txindex_to_first_txoutindex, txoutindexes_count)?;
.compute_last_index_from_first(&indexer.vecs.txindex_to_first_txoutindex, txoutindexes_count)?;
self.vecs.txindex_to_outputcount.compute_count_from_indexes(
&indexer.vecs().txindex_to_first_txoutindex,
&indexer.vecs.txindex_to_first_txoutindex,
&self.vecs.txindex_to_last_txoutindex,
)?;
self.vecs
.height_to_date
.compute_transform(&indexer.vecs().height_to_timestamp, |timestamp| Date::from(timestamp))?;
.compute_transform(&mut indexer.vecs.height_to_timestamp, |timestamp| Date::from(timestamp))?;
self.vecs
.height_to_last_txindex
.compute_last_index_from_first(&indexer.vecs().height_to_first_txindex, height_count)?;
.compute_last_index_from_first(&indexer.vecs.height_to_first_txindex, height_count)?;
self.vecs.txindex_to_height.compute_inverse_less_to_more(
&indexer.vecs().height_to_first_txindex,
&self.vecs.height_to_last_txindex,
&mut indexer.vecs.height_to_first_txindex,
&mut self.vecs.height_to_last_txindex,
)?;
let date_count = self.vecs.height_to_date.len();
self.vecs
.date_to_first_height
.compute_inverse_more_to_less(&self.vecs.height_to_date)?;
.compute_inverse_more_to_less(&mut self.vecs.height_to_date)?;
// ---
// Date to X

3
computer/src/main.rs
View File

@@ -3,6 +3,7 @@ use std::path::Path;
use biter::rpc;
use bomputer::Computer;
use exit::Exit;
use storable_vec::SINGLE_THREAD;
mod structs;
@@ -18,7 +19,7 @@ pub fn main() -> color_eyre::Result<()> {
let i = std::time::Instant::now();
let mut computer = Computer::import(Path::new("../_outputs"))?;
let mut computer: Computer<SINGLE_THREAD> = Computer::import(Path::new("../_outputs"))?;
computer.compute(data_dir, rpc, &exit)?;

73
computer/src/storage/storable_vecs/base.rs
View File

@@ -7,22 +7,28 @@ use std::{
};
use derive_deref::{Deref, DerefMut};
use storable_vec::{StorableVecIndex, StorableVecType, Version};
use storable_vec::{StorableVecIndex, StorableVecType, Version, SINGLE_THREAD};
#[derive(Debug, Deref, DerefMut)]
pub struct StorableVec<I, T>(storable_vec::StorableVec<I, T>);
pub struct StorableVec<I, T, const MODE: u8>(storable_vec::StorableVec<I, T, MODE>);
const FLUSH_EVERY: usize = 10_000;
impl<I, T> StorableVec<I, T>
impl<I, T, const MODE: u8> StorableVec<I, T, MODE>
where
I: StorableVecIndex,
T: StorableVecType,
{
pub fn import(path: &Path, version: Version) -> io::Result<Self> {
Ok(Self(storable_vec::StorableVec::import(path, version)?))
pub fn import(path: &Path, version: Version) -> storable_vec::Result<Self> {
Ok(Self(storable_vec::StorableVec::forced_import(path, version)?))
}
}
impl<I, T> StorableVec<I, T, SINGLE_THREAD>
where
I: StorableVecIndex,
T: StorableVecType,
{
fn flush_vec_if_needed(&mut self) -> io::Result<()> {
if self.pushed_len() == FLUSH_EVERY {
self.flush()
@@ -31,53 +37,10 @@ where
}
}
pub fn compute_inverse_more_to_less(&mut self, other: &storable_vec::StorableVec<T, I>) -> storable_vec::Result<()>
where
I: StorableVecType,
T: StorableVecIndex,
{
other.iter_from(self.last()?.map(|v| *v).unwrap_or_default(), |(v, i)| {
self.push_if_needed(*i, v)
})
}
pub fn compute_inverse_less_to_more(
&mut self,
first_indexes: &storable_vec::StorableVec<T, I>,
last_indexes: &storable_vec::StorableVec<T, I>,
) -> color_eyre::Result<()>
where
I: StorableVecType,
T: StorableVecIndex,
{
let (mut file_last, mut buf_last) = last_indexes.prepare_to_read_at_(self.len())?;
first_indexes.iter_from(T::from(self.len()), |(value, first_index)| {
let first_index: usize = (*first_index)
.try_into()
.map_err(|_| storable_vec::Error::FailedKeyTryIntoUsize)?;
let last_index = last_indexes.read_exact(&mut file_last, &mut buf_last)?;
let last_index: usize = (*last_index)
.try_into()
.map_err(|_| storable_vec::Error::FailedKeyTryIntoUsize)?;
(first_index..last_index).try_for_each(|index| self.push_if_needed(I::from(index), value))?;
Ok(())
})?;
self.flush()?;
Ok(())
}
pub fn compute_transform<A, F>(&mut self, other: &storable_vec::StorableVec<I, A>, t: F) -> storable_vec::Result<()>
where
A: StorableVecType,
F: Fn(&A) -> T,
{
other.iter_from(I::from(self.len()), |(i, a)| self.push_if_needed(i, t(a)))
}
pub fn compute_is_first_ordered<A>(
&mut self,
self_to_other: &storable_vec::StorableVec<I, A>,
other_to_self: &storable_vec::StorableVec<A, I>,
self_to_other: &storable_vec::StorableVec<I, A, SINGLE_THREAD>,
other_to_self: &storable_vec::StorableVec<A, I, SINGLE_THREAD>,
) -> storable_vec::Result<()>
where
A: StorableVecIndex + StorableVecType,
@@ -100,7 +63,7 @@ where
pub fn compute_last_index_from_first(
&mut self,
first_index_vec: &storable_vec::StorableVec<I, T>,
first_index_vec: &storable_vec::StorableVec<I, T, SINGLE_THREAD>,
final_len: usize,
) -> color_eyre::Result<()>
where
@@ -112,7 +75,7 @@ where
self.push_if_needed(prev_index, *v - T::from(1))?;
}
prev_index.replace(i);
self.flush_vec_if_needed().map_err(storable_vec::Error::IO)
Ok(self.flush_vec_if_needed()?)
})?;
if let Some(prev_index) = prev_index {
self.push_if_needed(prev_index, T::from(final_len) - T::from(1))?;
@@ -123,8 +86,8 @@ where
pub fn compute_count_from_indexes<T2>(
&mut self,
first_indexes: &storable_vec::StorableVec<I, T2>,
last_indexes: &storable_vec::StorableVec<I, T2>,
first_indexes: &storable_vec::StorableVec<I, T2, SINGLE_THREAD>,
last_indexes: &storable_vec::StorableVec<I, T2, SINGLE_THREAD>,
) -> color_eyre::Result<()>
where
T: From<T2>,
@@ -136,7 +99,7 @@ where
let last_index = last_indexes.read_exact(&mut file_last, &mut buf_last)?;
let count = *last_index + 1_usize - *first_index;
self.push_if_needed(i, count.into())?;
self.flush_vec_if_needed().map_err(storable_vec::Error::IO)
Ok(self.flush_vec_if_needed()?)
})?;
self.flush()?;
Ok(())

50
computer/src/storage/storable_vecs/mod.rs
View File

@@ -9,33 +9,33 @@ mod base;
use base::*;
pub struct StorableVecs {
pub date_to_first_height: StorableVec<Date, Height>,
// pub height_to_block_interval: StorableVec<Height, Timestamp>,
pub height_to_date: StorableVec<Height, Date>,
// pub height_to_fee: StorableVec<Txindex, Amount>,
// pub height_to_inputcount: StorableVec<Txindex, u32>,
// pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
pub height_to_last_txindex: StorableVec<Height, Txindex>,
// pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
// pub height_to_maxfeerate: StorableVec<Txindex, Feerate>,
// pub height_to_medianfeerate: StorableVec<Txindex, Feerate>,
// pub height_to_minfeerate: StorableVec<Txindex, Feerate>,
// pub height_to_outputcount: StorableVec<Txindex, u32>,
// pub height_to_subsidy: StorableVec<Txindex, u32>,
// pub height_to_totalfees: StorableVec<Height, Amount>,
// pub height_to_txcount: StorableVec<Txindex, u32>,
pub txindex_to_fee: StorableVec<Txindex, Amount>,
pub txindex_to_height: StorableVec<Txindex, Height>,
pub txindex_to_is_coinbase: StorableVec<Txindex, bool>,
// pub txindex_to_feerate: StorableVec<Txindex, Feerate>,
pub txindex_to_inputcount: StorableVec<Txindex, u32>,
pub txindex_to_last_txinindex: StorableVec<Txindex, Txinindex>,
pub txindex_to_last_txoutindex: StorableVec<Txindex, Txoutindex>,
pub txindex_to_outputcount: StorableVec<Txindex, u32>,
pub struct StorableVecs<const MODE: u8> {
pub date_to_first_height: StorableVec<Date, Height, MODE>,
// pub height_to_block_interval: StorableVec<Height, Timestamp, MODE>,
pub height_to_date: StorableVec<Height, Date, MODE>,
// pub height_to_fee: StorableVec<Txindex, Amount, MODE>,
// pub height_to_inputcount: StorableVec<Txindex, u32, MODE>,
// pub height_to_last_addressindex: StorableVec<Height, Addressindex, MODE>,
pub height_to_last_txindex: StorableVec<Height, Txindex, MODE>,
// pub height_to_last_txoutindex: StorableVec<Height, Txoutindex, MODE>,
// pub height_to_maxfeerate: StorableVec<Txindex, Feerate, MODE>,
// pub height_to_medianfeerate: StorableVec<Txindex, Feerate, MODE>,
// pub height_to_minfeerate: StorableVec<Txindex, Feerate, MODE>,
// pub height_to_outputcount: StorableVec<Txindex, u32, MODE>,
// pub height_to_subsidy: StorableVec<Txindex, u32, MODE>,
// pub height_to_totalfees: StorableVec<Height, Amount, MODE>,
// pub height_to_txcount: StorableVec<Txindex, u32, MODE>,
pub txindex_to_fee: StorableVec<Txindex, Amount, MODE>,
pub txindex_to_height: StorableVec<Txindex, Height, MODE>,
pub txindex_to_is_coinbase: StorableVec<Txindex, bool, MODE>,
// pub txindex_to_feerate: StorableVec<Txindex, Feerate, MODE>,
pub txindex_to_inputcount: StorableVec<Txindex, u32, MODE>,
pub txindex_to_last_txinindex: StorableVec<Txindex, Txinindex, MODE>,
pub txindex_to_last_txoutindex: StorableVec<Txindex, Txoutindex, MODE>,
pub txindex_to_outputcount: StorableVec<Txindex, u32, MODE>,
}
impl StorableVecs {
impl<const MODE: u8> StorableVecs<MODE> {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;

8
computer/src/structs/date.rs
View File

@@ -5,9 +5,15 @@ use color_eyre::eyre::eyre;
use derive_deref::Deref;
use jiff::{civil::Date as _Date, tz::TimeZone, Span};
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)]
pub struct Date(_Date);
impl Default for Date {
fn default() -> Self {
Self::INDEX_ZERO
}
}
impl Date {
const INDEX_ZERO: Self = Self(_Date::constant(2009, 1, 3));
const INDEX_ONE: Self = Self(_Date::constant(2009, 1, 9));