Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-18 14:38:10 -07:00
Code Issues Packages Projects Releases Wiki Activity

computer + kibo: part 9

Browse Source
This commit is contained in:
nym21
2025-04-11 12:00:26 +02:00
parent 9788b01f35
commit 3736d6ba5e
14 changed files with 220 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_computer/src/storage/vecs/base.rs
+71 -9
View File
@@ -249,11 +249,12 @@ where
let index = max_from.min(I::from(self.len()));
let one = T::from(1);
let mut prev_index: Option<I> = None;
first_indexes.iter_from(index, |(i, v, ..)| {
first_indexes.iter_from(index, |(index, v, ..)| {
if let Some(prev_index) = prev_index.take() {
self.forced_push_at(prev_index, v.checked_sub(one).unwrap(), exit)?;
let value = v.checked_sub(one).unwrap();
self.forced_push_at(prev_index, value, exit)?;
}
prev_index.replace(i);
prev_index.replace(index);
Ok(())
})?;
if let Some(prev_index) = prev_index {
@@ -276,7 +277,65 @@ where
) -> Result<()>
where
T: From<T2>,
T2: StoredType + Copy + Add<usize, Output = T2> + CheckedSub<T2> + TryInto<T> + Default,
T2: StoredType
+ StoredIndex
+ Copy
+ Add<usize, Output = T2>
+ CheckedSub<T2>
+ TryInto<T>
+ Default,
<T2 as TryInto<T>>::Error: error::Error + 'static,
{
let opt: Option<Box<dyn FnMut(T2) -> bool>> = None;
self.compute_filtered_count_from_indexes_(max_from, first_indexes, last_indexes, opt, exit)
}
pub fn compute_filtered_count_from_indexes<T2, F>(
&mut self,
max_from: I,
first_indexes: &mut StoredVec<I, T2>,
last_indexes: &mut StoredVec<I, T2>,
filter: F,
exit: &Exit,
) -> Result<()>
where
T: From<T2>,
T2: StoredType
+ StoredIndex
+ Copy
+ Add<usize, Output = T2>
+ CheckedSub<T2>
+ TryInto<T>
+ Default,
<T2 as TryInto<T>>::Error: error::Error + 'static,
F: FnMut(T2) -> bool,
{
self.compute_filtered_count_from_indexes_(
max_from,
first_indexes,
last_indexes,
Some(Box::new(filter)),
exit,
)
}
fn compute_filtered_count_from_indexes_<T2>(
&mut self,
max_from: I,
first_indexes: &mut StoredVec<I, T2>,
last_indexes: &mut StoredVec<I, T2>,
mut filter: Option<Box<dyn FnMut(T2) -> bool + '_>>,
exit: &Exit,
) -> Result<()>
where
T: From<T2>,
T2: StoredType
+ StoredIndex
+ Copy
+ Add<usize, Output = T2>
+ CheckedSub<T2>
+ TryInto<T>
+ Default,
<T2 as TryInto<T>>::Error: error::Error + 'static,
{
self.validate_computed_version_or_reset_file(
@@ -285,11 +344,14 @@ where
let index = max_from.min(I::from(self.len()));
first_indexes.iter_from(index, |(i, first_index, ..)| {
let last_index = last_indexes.cached_get(i)?.unwrap();
let count = (*last_index + 1_usize)
.checked_sub(first_index)
.unwrap_or_default();
self.forced_push_at(i, count.into(), exit)
let last_index = last_indexes.cached_get(i)?.unwrap().into_inner();
let range = first_index.to_usize().unwrap()..=last_index.to_usize().unwrap();
let count = if let Some(filter) = filter.as_mut() {
range.into_iter().filter(|i| filter(T2::from(*i))).count()
} else {
range.count()
};
self.forced_push_at(i, T::from(T2::from(count)), exit)
})?;
self.safe_flush(exit)
crates/brk_computer/src/storage/vecs/grouped/builder.rs
+24 -3
View File
@@ -4,6 +4,7 @@ use brk_exit::Exit;
use brk_vec::{
Compressed, DynamicVec, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version,
};
use color_eyre::eyre::ContextCompat;
use crate::storage::vecs::base::ComputedVec;
@@ -162,8 +163,8 @@ where
{
let index = self.starting_index(max_from);
first_indexes.iter_from(index, |(i, first_index, ..)| {
let last_index = *last_indexes.cached_get(i)?.unwrap();
first_indexes.iter_from(index, |(i, first_index, first_indexes)| {
let last_index = last_indexes.cached_get(i)?.unwrap().into_inner();
if let Some(first) = self.first.as_mut() {
let v = source.cached_get(first_index)?.unwrap().into_inner();
@@ -199,7 +200,27 @@ where
values.sort_unstable();
if let Some(max) = self.max.as_mut() {
max.forced_push_at(i, values.last().unwrap().clone(), exit)?;
max.forced_push_at(
i,
values
.last()
.context("expect some")
.inspect_err(|_| {
dbg!(
&values,
max.path(),
first_indexes.path(),
first_index,
last_indexes.path(),
last_index,
source.len(),
source.path()
);
})
.unwrap()
.clone(),
exit,
)?;
}
if let Some(_90p) = self._90p.as_mut() {
crates/brk_computer/src/storage/vecs/indexes.rs
+6 -4
View File
@@ -363,9 +363,11 @@ impl Vecs {
exit,
)?;
let decremented_starting_height = starting_indexes.height.decremented().unwrap_or_default();
let starting_dateindex = self
.height_to_dateindex
.cached_get(starting_indexes.height.decremented().unwrap_or_default())?
.cached_get(decremented_starting_height)?
.map_or_else(Default::default, |v| v.into_inner());
self.height_to_dateindex.compute_transform(
@@ -377,7 +379,7 @@ impl Vecs {
let starting_dateindex = if let Some(dateindex) = self
.height_to_dateindex
.cached_get(starting_indexes.height.decremented().unwrap_or_default())?
.cached_get(decremented_starting_height)?
.map(|v| v.into_inner())
{
starting_dateindex.min(dateindex)
@@ -696,7 +698,7 @@ impl Vecs {
let starting_difficultyepoch = self
.height_to_difficultyepoch
.cached_get(starting_indexes.height)?
.cached_get(decremented_starting_height)?
.map_or_else(Default::default, |v| v.into_inner());
self.height_to_difficultyepoch.compute_transform(
@@ -748,7 +750,7 @@ impl Vecs {
let starting_halvingepoch = self
.height_to_halvingepoch
.cached_get(starting_indexes.height)?
.cached_get(decremented_starting_height)?
.map_or_else(Default::default, |v| v.into_inner());
self.height_to_halvingepoch.compute_transform(
crates/brk_computer/src/storage/vecs/transactions.rs
+32 -67
View File
@@ -1,6 +1,6 @@
use std::{fs, path::Path};
use brk_core::{Sats, StoredU8, StoredU32, StoredU64, Txindex, Txinindex, Txoutindex};
use brk_core::{Sats, StoredU32, StoredU64, TxVersion, Txindex, Txinindex, Txoutindex};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{Compressed, DynamicVec, Version};
@@ -26,9 +26,6 @@ pub struct Vecs {
// pub txindex_to_input_sum: ComputedVec<Txindex, Sats>,
// pub txindex_to_output_sum: ComputedVec<Txindex, Sats>,
// pub txindex_to_output_value: ComputedVecsFromTxindex<Sats>,
pub txindex_to_v1: ComputedVec<Txindex, StoredU8>,
pub txindex_to_v2: ComputedVec<Txindex, StoredU8>,
pub txindex_to_v3: ComputedVec<Txindex, StoredU8>,
pub indexes_to_tx_v1: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_tx_v2: ComputedVecsFromHeight<StoredU32>,
pub indexes_to_tx_v3: ComputedVecsFromHeight<StoredU32>,
@@ -107,21 +104,6 @@ impl Vecs {
Version::ZERO,
compressed,
)?,
txindex_to_v1: ComputedVec::forced_import(
&path.join("txindex_to_v1"),
Version::ZERO,
compressed,
)?,
txindex_to_v2: ComputedVec::forced_import(
&path.join("txindex_to_v2"),
Version::ZERO,
compressed,
)?,
txindex_to_v3: ComputedVec::forced_import(
&path.join("txindex_to_v3"),
Version::ZERO,
compressed,
)?,
indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
path,
"tx_v1",
@@ -209,6 +191,37 @@ impl Vecs {
// Some(indexer.mut_vecs().txoutindex_to_value.mut_vec()),
// )?;
let mut compute_indexes_to_tx_vany =
|indexes_to_tx_vany: &mut ComputedVecsFromHeight<StoredU32>, txversion| {
indexes_to_tx_vany.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, indexer, indexes, starting_indexes, exit| {
let indexer_vecs = indexer.mut_vecs();
vec.compute_filtered_count_from_indexes(
starting_indexes.height,
indexer_vecs.height_to_first_txindex.mut_vec(),
indexes.height_to_last_txindex.mut_vec(),
|txindex| {
let v = indexer_vecs
.txindex_to_txversion
.cached_get(txindex)
.unwrap()
.unwrap()
.into_inner();
v == txversion
},
exit,
)
},
)
};
compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v1, TxVersion::ONE)?;
compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v2, TxVersion::TWO)?;
compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v3, TxVersion::THREE)?;
let indexer_vecs = indexer.mut_vecs();
self.txindex_to_is_coinbase.compute_is_first_ordered(
@@ -218,51 +231,6 @@ impl Vecs {
exit,
)?;
self.txindex_to_v1.compute_transform(
starting_indexes.txindex,
indexer_vecs.txindex_to_txversion.mut_vec(),
|(i, v, ..)| (i, StoredU8::from(v)),
exit,
)?;
// self.indexes_to_tx_v1.compute_all(
// indexer,
// indexes,
// starting_indexes,
// exit,
// |vec, indexer, indexes, indexes, exit| {
// vec.compute_transform(
// starting_indexes.height,
// indexer.mut_vecs().txindex_to_txversion.mut_vec(),
// || {},
// exit,
// )?;
// },
// )?;
self.txindex_to_v2.compute_transform(
starting_indexes.txindex,
indexer_vecs.txindex_to_txversion.mut_vec(),
|(i, v, ..)| (i, StoredU8::from(v)),
exit,
)?;
// self.indexes_to_tx_v1.compute_rest(
// starting_indexes.txindex,
// indexer_vecs.txindex_to_txversion.mut_vec(),
// |(i, v, ..)| (i, StoredU8::from(v)),
// exit,
// )?;
self.txindex_to_v3.compute_transform(
starting_indexes.txindex,
indexer_vecs.txindex_to_txversion.mut_vec(),
|(i, v, ..)| (i, StoredU8::from(v)),
exit,
)?;
// self.indexes_to_tx_v1.compute_rest(
// starting_indexes.txindex,
// indexer_vecs.txindex_to_txversion.mut_vec(),
// |(i, v, ..)| (i, StoredU8::from(v)),
// exit,
// )?;
self.txinindex_to_value.compute_transform(
starting_indexes.txinindex,
indexer_vecs.txinindex_to_txoutindex.mut_vec(),
@@ -297,9 +265,6 @@ impl Vecs {
vec![
self.txindex_to_is_coinbase.any_vec(),
self.txinindex_to_value.any_vec(),
self.txindex_to_v1.any_vec(),
self.txindex_to_v2.any_vec(),
self.txindex_to_v3.any_vec(),
],
self.height_to_tx_count.any_vecs(),
self.txindex_to_output_count.any_vecs(),
crates/brk_core/src/structs/txindex.rs
+8
View File
@@ -7,6 +7,8 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{CheckedSub, Error};
use super::StoredU32;
#[derive(
Debug,
PartialEq,
@@ -97,3 +99,9 @@ impl From<Txindex> for ByteView {
Self::new(value.as_bytes())
}
}
impl From<Txindex> for StoredU32 {
fn from(value: Txindex) -> Self {
Self::from(value.0)
}
}
crates/brk_core/src/structs/txversion.rs
+21 -1
View File
@@ -4,9 +4,29 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::StoredU8;
#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
#[derive(
Debug,
Deref,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Immutable,
IntoBytes,
KnownLayout,
FromBytes,
Serialize,
)]
pub struct TxVersion(i32);
impl TxVersion {
pub const ONE: Self = Self(1);
pub const TWO: Self = Self(2);
pub const THREE: Self = Self(3);
}
impl From<bitcoin::transaction::Version> for TxVersion {
fn from(value: bitcoin::transaction::Version) -> Self {
Self(value.0)
crates/brk_vec/src/lib.rs
+8 -8
View File
@@ -122,6 +122,14 @@ where
StoredVec::Compressed(v) => v.mut_pushed(),
}
}
#[inline]
fn path(&self) -> &Path {
match self {
StoredVec::Raw(v) => v.path(),
StoredVec::Compressed(v) => v.path(),
}
}
}
impl<I, T> GenericVec<I, T> for StoredVec<I, T>
@@ -160,14 +168,6 @@ where
}
}
#[inline]
fn path(&self) -> &Path {
match self {
StoredVec::Raw(v) => v.path(),
StoredVec::Compressed(v) => v.path(),
}
}
#[inline]
fn version(&self) -> Version {
match self {
crates/brk_vec/src/traits/dynamic.rs
+3 -1
View File
@@ -1,4 +1,4 @@
use std::sync::Arc;
use std::{path::Path, sync::Arc};
use arc_swap::{ArcSwap, Guard};
use memmap2::Mmap;
@@ -113,4 +113,6 @@ pub trait DynamicVec: Send + Sync {
Err(Error::IndexTooLow)
}
}
fn path(&self) -> &Path;
}
crates/brk_vec/src/traits/generic.rs
+1 -4
View File
@@ -61,8 +61,7 @@ where
#[inline]
fn reset(&mut self) -> Result<()> {
self.file_write_all(&[])?;
Ok(())
self.file_truncate_and_write_all(0, &[])
}
fn new_mmap(file: File) -> Result<Arc<Mmap>> {
@@ -174,8 +173,6 @@ where
Ok(self.collect_range_axum_json(from, to)?.into_response())
}
fn path(&self) -> &Path;
#[inline]
fn path_vec(&self) -> PathBuf {
Self::path_vec_(self.path())
crates/brk_vec/src/variants/compressed.rs
+16 -7
View File
@@ -278,6 +278,11 @@ where
fn mut_pushed(&mut self) -> &mut Vec<T> {
self.inner.mut_pushed()
}
#[inline]
fn path(&self) -> &Path {
self.inner.path()
}
}
impl<I, T> GenericVec<I, T> for CompressedVec<I, T>
@@ -441,6 +446,15 @@ where
Ok(())
}
fn reset(&mut self) -> Result<()> {
let mut pages_meta = (**self.pages_meta.load()).clone();
pages_meta.truncate(0);
pages_meta.write()?;
self.pages_meta.store(Arc::new(pages_meta));
self.reset_caches();
self.file_truncate_and_write_all(0, &[])
}
fn truncate_if_needed(&mut self, index: I) -> Result<()> {
let index = index.to_usize()?;
@@ -453,12 +467,12 @@ where
return Ok(());
}
let mut pages_meta = (**self.pages_meta.load()).clone();
let page_index = Self::index_to_page_index(index);
let guard = self.guard().as_ref().unwrap();
let mut pages_meta = (**self.pages_meta.load()).clone();
let values = self.decode_page(page_index, guard)?;
let mut buf = vec![];
@@ -490,11 +504,6 @@ where
Ok(())
}
#[inline]
fn path(&self) -> &Path {
self.inner.path()
}
#[inline]
fn version(&self) -> Version {
self.inner.version()
crates/brk_vec/src/variants/raw.rs
+5 -5
View File
@@ -114,6 +114,11 @@ where
fn mut_pushed(&mut self) -> &mut Vec<T> {
&mut self.pushed
}
#[inline]
fn path(&self) -> &Path {
self.pathbuf.as_path()
}
}
impl<I, T> GenericVec<I, T> for RawVec<I, T>
@@ -211,11 +216,6 @@ where
Ok(())
}
#[inline]
fn path(&self) -> &Path {
self.pathbuf.as_path()
}
#[inline]
fn version(&self) -> Version {
self.version
websites/kibo.money/index.html
+2 -1
View File
@@ -877,9 +877,10 @@
background-image: linear-gradient(
to bottom,
transparent,
var(--background-color) 90%,
var(--background-color)
);
height: 24rem;
height: 21rem;
bottom: 0;
left: 0;
right: 0;
websites/kibo.money/scripts/options.js
+23
View File
@@ -17,6 +17,7 @@
* "Seconds" |
* "Transactions" |
* "US Dollars" |
* "Version" |
* "Virtual Bytes" |
* "Weight Units"
* } Unit
@@ -868,6 +869,26 @@ function createPartialOptions(colors) {
title: "Transaction Count",
key: "tx-count",
}),
{
name: "Version",
tree: [
createBaseSumTotal({
name: "1",
title: "Transaction V1 Count",
key: "tx-v1",
}),
createBaseSumTotal({
name: "2",
title: "Transaction V2 Count",
key: "tx-v2",
}),
createBaseSumTotal({
name: "3",
title: "Transaction V3 Count",
key: "tx-v3",
}),
],
},
],
},
{
@@ -1254,6 +1275,8 @@ export function initOptions({
anyPartial.unit = "Weight Units";
} else if (key.includes("-vbytes")) {
anyPartial.unit = "Virtual Bytes";
} else if (key.match(/v[0-9]/g)) {
anyPartial.unit = "Version";
} else {
console.log(anyPartial);
throw Error("Unit not set");
websites/kibo.money/scripts/vecid-to-indexes.js
-3
View File
@@ -167,9 +167,6 @@ export function createVecIdToIndexes() {
txid: [Txindex],
txoutindex: [Txinindex],
txversion: [Txindex],
v1: [Txindex],
v2: [Txindex],
v3: [Txindex],
value: [Txinindex, Txoutindex],
weekindex: [Dateindex, Weekindex],
yearindex: [Monthindex, Yearindex],