Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-07-08 17:48:14 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-11-06 15:17:24 +01:00
parent df577ca7f5
commit 11b323ef00
9 changed files with 119 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brk_computer/src/blks.rs
+4 -10
View File
@@ -94,22 +94,16 @@ impl Vecs {
.try_for_each(|block| -> Result<()> {
let height = block.height();
self.height_to_position.forced_push_at(
height,
block.metadata().position(),
exit,
)?;
self.height_to_position
.forced_push(height, block.metadata().position(), exit)?;
let txindex = height_to_first_txindex_iter.get_unwrap(height);
block.tx_metadata().iter().enumerate().try_for_each(
|(index, metadata)| -> Result<()> {
let txindex = txindex + index;
self.txindex_to_position.forced_push_at(
txindex,
metadata.position(),
exit,
)?;
self.txindex_to_position
.forced_push(txindex, metadata.position(), exit)?;
Ok(())
},
)?;
crates/brk_computer/src/fetched.rs
+1 -1
View File
@@ -133,7 +133,7 @@ impl Vecs {
prev.replace(ohlc.clone());
self.dateindex_to_price_ohlc_in_cents
.forced_push_at_(i, ohlc, exit)?;
.forced_push_at(i, ohlc, exit)?;
Ok(())
})?;
crates/brk_computer/src/indexes.rs
+24 -26
View File
@@ -14,7 +14,7 @@ use brk_types::{
};
use vecdb::{
AnyCloneableIterableVec, Database, EagerVec, Exit, LazyVecFrom1, LazyVecFrom2, PAGE_SIZE,
StoredIndex, VecIterator,
StoredIndex, VecIteratorExtended,
};
const VERSION: Version = Version::ZERO;
@@ -117,15 +117,13 @@ impl Vecs {
indexer.vecs.txindex_to_first_txoutindex.boxed_clone(),
|index: TxInIndex, txinindex_to_outpoint_iter, txindex_to_first_txoutindex_iter| {
txinindex_to_outpoint_iter
.next_at(index.to_usize())
.map(|(_, outpoint)| {
.get_at(index.to_usize())
.map(|outpoint| {
if outpoint.is_coinbase() {
return TxOutIndex::COINBASE;
}
txindex_to_first_txoutindex_iter
.next_at(outpoint.txindex().to_usize())
.unwrap()
.1
.get_unwrap_at(outpoint.txindex().to_usize())
+ outpoint.vout()
})
},
@@ -160,12 +158,12 @@ impl Vecs {
|index: TxIndex, txindex_to_first_txinindex_iter, txinindex_to_txoutindex_iter| {
let txindex = index.to_usize();
txindex_to_first_txinindex_iter
.next_at(txindex)
.map(|(_, start)| {
.get_at(txindex)
.map(|start| {
let start = usize::from(start);
let end = txindex_to_first_txinindex_iter
.next_at(txindex + 1)
.map(|(_, v)| usize::from(v))
.get_at(txindex + 1)
.map(|v| usize::from(v))
.unwrap_or_else(|| txinindex_to_txoutindex_iter.len());
StoredU64::from((start..end).count())
})
@@ -180,12 +178,12 @@ impl Vecs {
|index: TxIndex, txindex_to_first_txoutindex_iter, txoutindex_to_value_iter| {
let txindex = index.to_usize();
txindex_to_first_txoutindex_iter
.next_at(txindex)
.map(|(_, start)| {
.get_at(txindex)
.map(|start| {
let start = usize::from(start);
let end = txindex_to_first_txoutindex_iter
.next_at(txindex + 1)
.map(|(_, v)| usize::from(v))
.get_at(txindex + 1)
.map(|v| usize::from(v))
.unwrap_or_else(|| txoutindex_to_value_iter.len());
StoredU64::from((start..end).count())
})
@@ -558,7 +556,7 @@ impl Vecs {
prev_timestamp_fixed.replace(
height_to_timestamp_fixed_iter
.into_iter()
.unsafe_get(prev_h),
.get_unwrap(prev_h),
);
}
let timestamp_fixed =
@@ -585,7 +583,7 @@ impl Vecs {
let starting_dateindex = self
.height_to_dateindex
.into_iter()
.get_inner(decremented_starting_height)
.get(decremented_starting_height)
.unwrap_or_default();
self.height_to_dateindex.compute_transform(
@@ -598,7 +596,7 @@ impl Vecs {
let starting_dateindex = if let Some(dateindex) = self
.height_to_dateindex
.into_iter()
.get_inner(decremented_starting_height)
.get(decremented_starting_height)
{
starting_dateindex.min(dateindex)
} else {
@@ -637,7 +635,7 @@ impl Vecs {
let starting_weekindex = self
.dateindex_to_weekindex
.into_iter()
.get_inner(starting_dateindex)
.get(starting_dateindex)
.unwrap_or_default();
self.dateindex_to_weekindex.compute_range(
@@ -674,7 +672,7 @@ impl Vecs {
let starting_difficultyepoch = self
.height_to_difficultyepoch
.into_iter()
.get_inner(decremented_starting_height)
.get(decremented_starting_height)
.unwrap_or_default();
self.height_to_difficultyepoch.compute_from_index(
@@ -710,7 +708,7 @@ impl Vecs {
let starting_monthindex = self
.dateindex_to_monthindex
.into_iter()
.get_inner(starting_dateindex)
.get(starting_dateindex)
.unwrap_or_default();
self.dateindex_to_monthindex.compute_range(
@@ -747,7 +745,7 @@ impl Vecs {
let starting_quarterindex = self
.monthindex_to_quarterindex
.into_iter()
.get_inner(starting_monthindex)
.get(starting_monthindex)
.unwrap_or_default();
self.monthindex_to_quarterindex.compute_from_index(
@@ -785,7 +783,7 @@ impl Vecs {
let starting_semesterindex = self
.monthindex_to_semesterindex
.into_iter()
.get_inner(starting_monthindex)
.get(starting_monthindex)
.unwrap_or_default();
self.monthindex_to_semesterindex.compute_from_index(
@@ -823,7 +821,7 @@ impl Vecs {
let starting_yearindex = self
.monthindex_to_yearindex
.into_iter()
.get_inner(starting_monthindex)
.get(starting_monthindex)
.unwrap_or_default();
self.monthindex_to_yearindex.compute_from_index(
@@ -858,7 +856,7 @@ impl Vecs {
let starting_halvingepoch = self
.height_to_halvingepoch
.into_iter()
.get_inner(decremented_starting_height)
.get(decremented_starting_height)
.unwrap_or_default();
self.height_to_halvingepoch.compute_from_index(
@@ -886,7 +884,7 @@ impl Vecs {
let starting_decadeindex = self
.yearindex_to_decadeindex
.into_iter()
.get_inner(starting_yearindex)
.get(starting_yearindex)
.unwrap_or_default();
self.yearindex_to_decadeindex.compute_from_index(
@@ -947,7 +945,7 @@ impl Indexes {
pub fn update_from_height(&mut self, height: Height, indexes: &Vecs) {
self.indexes.height = height;
self.dateindex =
DateIndex::try_from(indexes.height_to_date_fixed.into_iter().unsafe_get(height))
DateIndex::try_from(indexes.height_to_date_fixed.into_iter().get_unwrap(height))
.unwrap();
self.weekindex = WeekIndex::from(self.dateindex);
self.monthindex = MonthIndex::from(self.dateindex);
crates/brk_computer/src/traits.rs
+1 -1
View File
@@ -197,7 +197,7 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
stacks
.iter()
.skip(index)
.skip(index.to_usize())
.enumerate()
.try_for_each(|(i, stack)| {
let mut avg_price = Dollars::from(f64::NAN);
crates/brk_indexer/src/lib.rs
+5 -5
View File
@@ -18,20 +18,20 @@ use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use vecdb::{AnyVec, Exit, GenericStoredVec, Reader, VecIteratorExtended};
mod indexes;
mod stores_v2;
// mod stores_v3;
// mod stores_v2;
mod stores_v3;
mod vecs;
pub use indexes::*;
pub use stores_v2::*;
// pub use stores_v3::*;
// pub use stores_v2::*;
pub use stores_v3::*;
pub use vecs::*;
// One version for all data sources
// Increment on **change _OR_ addition**
const VERSION: Version = Version::new(23);
const SNAPSHOT_BLOCK_RANGE: usize = 1_000;
const COLLISIONS_CHECKED_UP_TO: Height = Height::new(920_000);
const COLLISIONS_CHECKED_UP_TO: Height = Height::new(0);
#[derive(Clone)]
pub struct Indexer {
crates/brk_logger/src/lib.rs
+2 -2
View File
@@ -23,8 +23,8 @@ pub fn init(path: Option<&Path>) -> io::Result<()> {
});
Builder::from_env(Env::default().default_filter_or(
"info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
// "debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
// "info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
"debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
))
.format(move |buf, record| {
let date_time = Timestamp::now()
crates/brk_store/src/fjall_v3/mod.rs
+48 -30
View File
@@ -4,7 +4,10 @@ use brk_error::Result;
use brk_types::{Height, Version};
use byteview8::ByteView;
use fjall3::{
Database, Keyspace, KeyspaceCreateOptions, ValueType,
Database,
Keyspace,
KeyspaceCreateOptions,
// ValueType,
config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry, PinningPolicy},
};
@@ -198,13 +201,28 @@ where
)
.collect::<Vec<_>>();
items.sort_unstable();
batch.ingest(
items
.into_iter()
.map(|i| i.fjalled(&self.keyspace))
.collect::<Vec<_>>(),
);
batch.commit_keyspace(&self.keyspace)?;
items.into_iter().for_each(|item| match item {
Item::Value { key, value } => {
batch.insert(&self.keyspace, ByteView::from(key), ByteView::from(value))
}
Item::Tomb(key) => batch.remove(&self.keyspace, ByteView::from(key)),
});
batch.commit()?;
// batch.ingest(
// items
// .into_iter()
// .map(|i| i.fjalled(&self.keyspace))
// .collect::<Vec<_>>(),
// );
// batch.commit_keyspace(&self.keyspace)?;
// batch.ingest(
// items
// .into_iter()
// .map(|i| i.fjalled(&self.keyspace))
// .collect::<Vec<_>>(),
// );
// batch.commit_keyspace(&self.keyspace)?;
// batch.commit_keyspace(self.keyspace.inner())?;
Ok(())
@@ -263,26 +281,26 @@ impl<K, V> Item<K, V> {
}
}
pub fn fjalled(self, keyspace: &Keyspace) -> fjall3::Item
where
K: Into<ByteView>,
V: Into<ByteView>,
{
let keyspace_id = keyspace.id;
// let keyspace_id = keyspace.inner().id;
match self {
Item::Value { key, value } => fjall3::Item {
keyspace_id,
key: key.into().into(),
value: value.into().into(),
value_type: ValueType::Value,
},
Item::Tomb(key) => fjall3::Item {
keyspace_id,
key: key.into().into(),
value: [].into(),
value_type: ValueType::WeakTombstone,
},
}
}
// pub fn fjalled(self, keyspace: &Keyspace) -> fjall3::Item
// where
// K: Into<ByteView>,
// V: Into<ByteView>,
// {
// let keyspace_id = keyspace.id;
// // let keyspace_id = keyspace.inner().id;
// match self {
// Item::Value { key, value } => fjall3::Item {
// keyspace_id,
// key: key.into().into(),
// value: value.into().into(),
// value_type: ValueType::Value,
// },
// Item::Tomb(key) => fjall3::Item {
// keyspace_id,
// key: key.into().into(),
// value: [].into(),
// value_type: ValueType::WeakTombstone,
// },
// }
// }
}