Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-27 08:09:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-02-02 23:28:03 +01:00
parent ad34d9d402
commit 1e37d75e49
28 changed files with 660 additions and 359 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
computer/src/lib.rs
View File

@@ -1,6 +1,6 @@
use std::path::{Path, PathBuf};
use bindex::{Height, Indexer};
use bindex::Indexer;
use biter::rpc;
use exit::Exit;
@@ -36,34 +36,63 @@ impl Computer {
indexer.index(bitcoin_dir, rpc, exit)?;
}
let height_count = indexer.vecs().height_to_size.len();
let txindexes_count = indexer.vecs().txindex_to_txid.len();
let txinindexes_count = indexer.vecs().txinindex_to_txoutindex.len();
let txoutindexes_count = indexer.vecs().txoutindex_to_addressindex.len();
// TODO: Remove all outdated
// Compute txindex to X
self.vecs
.txindex_to_last_txinindex
.compute_last_index_from_first(&indexer.vecs().txindex_to_first_txinindex, txinindexes_count)?;
self.vecs.txindex_to_inputcount.compute_count_from_indexes(
&indexer.vecs().txindex_to_first_txinindex,
&self.vecs.txindex_to_last_txinindex,
)?;
self.vecs
.txindex_to_last_txoutindex
.compute_last_index_from_first(&indexer.vecs().txindex_to_first_txoutindex, txoutindexes_count)?;
self.vecs.txindex_to_outputcount.compute_count_from_indexes(
&indexer.vecs().txindex_to_first_txoutindex,
&self.vecs.txindex_to_last_txoutindex,
)?;
// Compute height to X
indexer
.vecs()
.height_to_timestamp
.read_from_(self.vecs.height_to_date.len(), |(_height, timestamp)| {
self.vecs
.height_to_date
.push_if_needed(Height::from(_height), Date::from(timestamp))
})?;
self.vecs
.height_to_date
.read_from_(self.vecs.date_to_first_height.len(), |(_height, date)| {
self.vecs
.date_to_first_height
.push_if_needed(*date, Height::from(_height))
})?;
.compute_transform(&indexer.vecs().height_to_timestamp, |timestamp| Date::from(timestamp))?;
// Compute date to X
self.vecs
.height_to_last_txindex
.compute_last_index_from_first(&indexer.vecs().height_to_first_txindex, height_count)?;
self.vecs.txindex_to_height.compute_inverse_less_to_more(
&indexer.vecs().height_to_first_txindex,
&self.vecs.height_to_last_txindex,
)?;
let date_count = self.vecs.height_to_date.len();
self.vecs
.date_to_first_height
.compute_inverse_more_to_less(&self.vecs.height_to_date)?;
// ---
// Date to X
// ---
// ...
// Compute month to X
// ---
// Month to X
// ---
// ...
// Compute year to X
// ---
// Year to X
// ---
// ...
Ok(())

12
computer/src/storage/fjalls.rs
View File

@@ -5,18 +5,18 @@ use bindex::{Store, Version};
use crate::structs::{AddressindexTxoutindex, Unit};
pub struct Fjalls {
pub address_txoutindex_in: Store<AddressindexTxoutindex, Unit>,
pub address_txoutindex_out: Store<AddressindexTxoutindex, Unit>,
pub address_to_utxos_received: Store<AddressindexTxoutindex, Unit>,
pub address_to_utxos_spent: Store<AddressindexTxoutindex, Unit>,
}
impl Fjalls {
pub fn import(path: &Path) -> color_eyre::Result<Self> {
let address_txoutindex_in = Store::import(&path.join("address_txoutindex_in"), Version::from(1))?;
let address_txoutindex_out = Store::import(&path.join("address_txoutindex_out"), Version::from(1))?;
let address_to_utxos_received = Store::import(&path.join("address_to_utxos_received"), Version::from(1))?;
let address_to_utxos_spent = Store::import(&path.join("address_to_utxos_spent"), Version::from(1))?;
Ok(Self {
address_txoutindex_in,
address_txoutindex_out,
address_to_utxos_received,
address_to_utxos_spent,
})
}
}

176
computer/src/storage/storable_vecs/base.rs
View File

@@ -1,74 +1,144 @@
use std::{
error,
fmt::Debug,
io,
ops::{Deref, DerefMut},
ops::{Add, Sub},
path::Path,
};
use bindex::{Indexer, Version};
use derive_deref::{Deref, DerefMut};
use storable_vec::{StorableVecIndex, StorableVecType, Version};
use crate::Computer;
#[derive(Debug, Deref, DerefMut)]
pub struct StorableVec<I, T>(storable_vec::StorableVec<I, T>);
pub struct StorableVec<I, T> {
vec: bindex::StorableVec<I, T>,
f: Box<dyn Fn(&Indexer, &Computer) -> storable_vec::Result<Vec<(I, T)>>>,
}
const FLUSH_EVERY: usize = 10_000;
impl<I, T> StorableVec<I, T>
where
I: TryInto<usize>,
T: Sized + Debug + Clone,
I: StorableVecIndex,
T: StorableVecType,
{
pub fn import<F>(path: &Path, version: Version, f: F) -> io::Result<Self>
pub fn import(path: &Path, version: Version) -> io::Result<Self> {
Ok(Self(storable_vec::StorableVec::import(path, version)?))
}
fn flush_vec_if_needed(&mut self) -> io::Result<()> {
if self.pushed_len() == FLUSH_EVERY {
self.flush()
} else {
Ok(())
}
}
pub fn compute_inverse_more_to_less(&mut self, other: &storable_vec::StorableVec<T, I>) -> storable_vec::Result<()>
where
F: Fn(&Indexer, &Computer) -> storable_vec::Result<Vec<(I, T)>> + 'static,
I: StorableVecType,
T: StorableVecIndex,
{
let vec = bindex::StorableVec::import(path, version)?;
Ok(Self { vec, f: Box::new(f) })
other.iter_from(self.last()?.map(|v| *v).unwrap_or_default(), |(v, i)| {
self.push_if_needed(*i, v)
})
}
pub fn compute(&mut self, indexer: &Indexer, computer: &Computer) -> storable_vec::Result<()> {
(self.f)(indexer, computer)?
.into_iter()
.try_for_each(|(i, v)| self.push_if_needed(i, v))
pub fn compute_inverse_less_to_more(
&mut self,
first_indexes: &storable_vec::StorableVec<T, I>,
last_indexes: &storable_vec::StorableVec<T, I>,
) -> color_eyre::Result<()>
where
I: StorableVecType,
T: StorableVecIndex,
{
let (mut file_last, mut buf_last) = last_indexes.prepare_to_read_at_(self.len())?;
first_indexes.iter_from(T::from(self.len()), |(value, first_index)| {
let first_index: usize = (*first_index)
.try_into()
.map_err(|_| storable_vec::Error::FailedKeyTryIntoUsize)?;
let last_index = last_indexes.read_exact(&mut file_last, &mut buf_last)?;
let last_index: usize = (*last_index)
.try_into()
.map_err(|_| storable_vec::Error::FailedKeyTryIntoUsize)?;
(first_index..last_index).try_for_each(|index| self.push_if_needed(I::from(index), value))?;
Ok(())
})?;
self.flush()?;
Ok(())
}
// pub fn fill(&mut self) {
// self
// .vecs()
// .height_to_timestamp
// .read_iter(move |(_height, timestamp)| {
// let height = Height::from(_height);
// let date = Date::from(timestamp);
// self.vecs.date_to_first_height.push_if_needed(date, height)?;
// Ok(())
// })?;
// }
}
pub fn compute_transform<A, F>(&mut self, other: &storable_vec::StorableVec<I, A>, t: F) -> storable_vec::Result<()>
where
A: StorableVecType,
F: Fn(&A) -> T,
{
other.iter_from(I::from(self.len()), |(i, a)| self.push_if_needed(i, t(a)))
}
impl<I, T> Deref for StorableVec<I, T> {
type Target = bindex::StorableVec<I, T>;
fn deref(&self) -> &Self::Target {
&self.vec
}
}
impl<I, T> DerefMut for StorableVec<I, T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.vec
}
}
pub trait AnyComputedStorableVec {
fn compute(&mut self, indexer: &Indexer, computer: &Computer) -> storable_vec::Result<()>;
}
impl<I, T> AnyComputedStorableVec for StorableVec<I, T>
where
I: TryInto<usize>,
T: Sized + Debug + Clone,
{
fn compute(&mut self, indexer: &Indexer, computer: &Computer) -> storable_vec::Result<()> {
self.compute(indexer, computer)
pub fn compute_is_first_ordered<A>(
&mut self,
self_to_other: &storable_vec::StorableVec<I, A>,
other_to_self: &storable_vec::StorableVec<A, I>,
) -> storable_vec::Result<()>
where
A: StorableVecIndex + StorableVecType,
{
// let mut prev_a_opt = None;
// self_to_other.iter_from(I::from(self.len()), |(i, a)| {
// if prev_a_opt.is_none() {
// prev_a_opt.replace(a);
// self.push_if_needed(i, other_to_self.read_at(a) == i);
// } else {
// let prev_a = prev_a_opt.unwrap();
// if a != prev_a
// }
// other_to_self.seek_read(a);
// self.push_if_needed(i, t(a));
// Ok(())
// })
Ok(())
}
pub fn compute_last_index_from_first(
&mut self,
first_index_vec: &storable_vec::StorableVec<I, T>,
final_len: usize,
) -> color_eyre::Result<()>
where
T: Copy + From<usize> + Sub<T, Output = T> + StorableVecIndex,
{
let mut prev_index: Option<I> = None;
first_index_vec.iter_from(I::from(self.len()), |(i, v)| {
if let Some(prev_index) = prev_index {
self.push_if_needed(prev_index, *v - T::from(1))?;
}
prev_index.replace(i);
self.flush_vec_if_needed().map_err(storable_vec::Error::IO)
})?;
if let Some(prev_index) = prev_index {
self.push_if_needed(prev_index, T::from(final_len) - T::from(1))?;
}
self.flush()?;
Ok(())
}
pub fn compute_count_from_indexes<T2>(
&mut self,
first_indexes: &storable_vec::StorableVec<I, T2>,
last_indexes: &storable_vec::StorableVec<I, T2>,
) -> color_eyre::Result<()>
where
T: From<T2>,
T2: StorableVecType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
<T2 as TryInto<T>>::Error: error::Error + Send + Sync + 'static,
{
let (mut file_last, mut buf_last) = last_indexes.prepare_to_read_at_(self.len())?;
first_indexes.iter_from(I::from(self.len()), |(i, first_index)| {
let last_index = last_indexes.read_exact(&mut file_last, &mut buf_last)?;
let count = *last_index + 1_usize - *first_index;
self.push_if_needed(i, count.into())?;
self.flush_vec_if_needed().map_err(storable_vec::Error::IO)
})?;
self.flush()?;
Ok(())
}
}

15
computer/src/storage/storable_vecs/mod.rs
View File

@@ -1,12 +1,13 @@
use std::{fs, path::Path};
use bindex::{Addressindex, Amount, Height, StorableVec, Timestamp, Txindex, Txinindex, Txoutindex, Version};
use bindex::{Addressindex, Amount, Height, Timestamp, Txindex, Txinindex, Txoutindex};
use storable_vec::Version;
use crate::structs::{Date, Feerate};
// mod base;
mod base;
// use base::*;
use base::*;
pub struct StorableVecs {
pub date_to_first_height: StorableVec<Date, Height>,
@@ -15,7 +16,7 @@ pub struct StorableVecs {
// pub height_to_fee: StorableVec<Txindex, Amount>,
// pub height_to_inputcount: StorableVec<Txindex, u32>,
// pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
// pub height_to_last_txindex: StorableVec<Height, Txindex>,
pub height_to_last_txindex: StorableVec<Height, Txindex>,
// pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
// pub height_to_maxfeerate: StorableVec<Txindex, Feerate>,
// pub height_to_medianfeerate: StorableVec<Txindex, Feerate>,
@@ -25,6 +26,8 @@ pub struct StorableVecs {
// pub height_to_totalfees: StorableVec<Height, Amount>,
// pub height_to_txcount: StorableVec<Txindex, u32>,
pub txindex_to_fee: StorableVec<Txindex, Amount>,
pub txindex_to_height: StorableVec<Txindex, Height>,
pub txindex_to_is_coinbase: StorableVec<Txindex, bool>,
// pub txindex_to_feerate: StorableVec<Txindex, Feerate>,
pub txindex_to_inputcount: StorableVec<Txindex, u32>,
pub txindex_to_last_txinindex: StorableVec<Txindex, Txinindex>,
@@ -46,7 +49,7 @@ impl StorableVecs {
// &path.join("height_to_last_addressindex"),
// Version::from(1),
// )?,
// height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"), Version::from(1))?,
height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"), Version::from(1))?,
// height_to_last_txoutindex: StorableVec::import(&path.join("height_to_last_txoutindex"), Version::from(1))?,
// height_to_maxfeerate: StorableVec::import(&path.join("height_to_maxfeerate"), Version::from(1))?,
// height_to_medianfeerate: StorableVec::import(&path.join("height_to_medianfeerate"), Version::from(1))?,
@@ -56,6 +59,8 @@ impl StorableVecs {
// height_to_totalfees: StorableVec::import(&path.join("height_to_totalfees"), Version::from(1))?,
// height_to_txcount: StorableVec::import(&path.join("height_to_txcount"), Version::from(1))?,
txindex_to_fee: StorableVec::import(&path.join("txindex_to_fee"), Version::from(1))?,
txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?,
txindex_to_is_coinbase: StorableVec::import(&path.join("txindex_to_is_coinbase"), Version::from(1))?,
// txindex_to_feerate: StorableVec::import(&path.join("txindex_to_feerate"), Version::from(1))?,
txindex_to_inputcount: StorableVec::import(&path.join("txindex_to_inputcount"), Version::from(1))?,
txindex_to_last_txinindex: StorableVec::import(&path.join("txindex_to_last_txinindex"), Version::from(1))?,

19
computer/src/structs/date.rs
View File

@@ -1,9 +1,11 @@
use std::ops::Add;
use bindex::Timestamp;
use color_eyre::eyre::eyre;
use derive_deref::Deref;
use jiff::{civil::Date as _Date, tz::TimeZone};
use jiff::{civil::Date as _Date, tz::TimeZone, Span};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)]
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deref)]
pub struct Date(_Date);
impl Date {
@@ -39,3 +41,16 @@ impl TryFrom<Date> for usize {
}
}
}
impl From<usize> for Date {
fn from(value: usize) -> Self {
Self(Self::INDEX_ZERO.checked_add(Span::new().days(value as i64)).unwrap())
}
}
impl Add<usize> for Date {
type Output = Self;
fn add(self, rhs: usize) -> Self::Output {
Self(self.0.checked_add(Span::new().days(rhs as i64)).unwrap())
}
}