Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-08 14:11:56 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: move addressindex_to_outputindex stores from computer to indexer

Browse Source
This commit is contained in:
nym21
2025-07-04 17:30:43 +02:00
parent 6d35c26b3f
commit 7f07b0daa7
47 changed files with 515 additions and 1391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
-3
View File
@@ -512,7 +512,6 @@ dependencies = [
"brk_vec",
"color-eyre",
"derive_deref",
"fjall",
"jiff",
"log",
"rayon",
@@ -576,7 +575,6 @@ dependencies = [
"brk_store",
"brk_vec",
"color-eyre",
"fjall",
"log",
"rayon",
]
@@ -617,7 +615,6 @@ dependencies = [
"brk_interface",
"brk_rmcp",
"log",
"tracing",
]
[[package]]
crates/brk_computer/Cargo.toml
+3 -1
View File
@@ -21,10 +21,12 @@ brk_store = { workspace = true }
brk_vec = { workspace = true }
color-eyre = { workspace = true }
derive_deref = { workspace = true }
fjall = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
zerocopy = { workspace = true }
zerocopy-derive = { workspace = true }
[package.metadata.cargo-machete]
ignored = ["zerocopy"]
crates/brk_computer/src/lib.rs
-3
View File
@@ -12,13 +12,11 @@ use brk_indexer::Indexer;
use brk_vec::{Computation, Format};
use log::info;
mod groups;
mod states;
mod stores;
mod utils;
mod vecs;
use groups::*;
use states::*;
use stores::Stores;
use vecs::Vecs;
@@ -55,7 +53,6 @@ impl Computer {
// TODO: Give self.path, join inside import
&outputs_dir.join("stores"),
VERSION + Version::ZERO,
&indexer.stores.keyspace,
)?,
fetcher,
})
crates/brk_computer/src/states/transacted.rs
+1 -3
View File
@@ -1,8 +1,6 @@
use std::ops::{Add, AddAssign};
use brk_core::{OutputType, Sats};
use crate::{GroupedBySizeRange, GroupedByType};
use brk_core::{GroupedBySizeRange, GroupedByType, OutputType, Sats};
use super::SupplyState;
crates/brk_computer/src/stores.rs
+17 -481
View File
@@ -1,131 +1,52 @@
use std::{path::Path, thread};
use brk_core::{
AddressData, EmptyAddressData, Height, OutputIndex, OutputType, P2AAddressIndex,
P2AAddressIndexOutputindex, P2PK33AddressIndex, P2PK33AddressIndexOutputindex,
P2PK65AddressIndex, P2PK65AddressIndexOutputindex, P2PKHAddressIndex,
P2PKHAddressIndexOutputindex, P2SHAddressIndex, P2SHAddressIndexOutputindex, P2TRAddressIndex,
P2TRAddressIndexOutputindex, P2WPKHAddressIndex, P2WPKHAddressIndexOutputindex,
P2WSHAddressIndex, P2WSHAddressIndexOutputindex, Result, TypeIndex, Unit, Version,
AddressData, EmptyAddressData, GroupedByAddressType, Height, OutputType, P2AAddressIndex,
P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex,
P2WPKHAddressIndex, P2WSHAddressIndex, Result, TypeIndex, Version,
};
use brk_store::{AnyStore, Store};
use fjall::{PersistMode, TransactionalKeyspace};
use log::info;
use crate::{
GroupedByAddressType,
vecs::stateful::{
AddressTypeToTypeIndexTree, AddressTypeToTypeIndexVec, WithAddressDataSource,
},
};
use crate::vecs::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Stores {
keyspace: TransactionalKeyspace,
pub p2aaddressindex_to_addressdata: Store<P2AAddressIndex, AddressData>,
pub p2aaddressindex_to_emptyaddressdata: Store<P2AAddressIndex, EmptyAddressData>,
pub p2aaddressindex_to_outputs_received: Store<P2AAddressIndexOutputindex, Unit>,
pub p2aaddressindex_to_outputs_sent: Store<P2AAddressIndexOutputindex, Unit>,
pub p2pk33addressindex_to_addressdata: Store<P2PK33AddressIndex, AddressData>,
pub p2pk33addressindex_to_emptyaddressdata: Store<P2PK33AddressIndex, EmptyAddressData>,
pub p2pk33addressindex_to_outputs_received: Store<P2PK33AddressIndexOutputindex, Unit>,
pub p2pk33addressindex_to_outputs_sent: Store<P2PK33AddressIndexOutputindex, Unit>,
pub p2pk65addressindex_to_addressdata: Store<P2PK65AddressIndex, AddressData>,
pub p2pk65addressindex_to_emptyaddressdata: Store<P2PK65AddressIndex, EmptyAddressData>,
pub p2pk65addressindex_to_outputs_received: Store<P2PK65AddressIndexOutputindex, Unit>,
pub p2pk65addressindex_to_outputs_sent: Store<P2PK65AddressIndexOutputindex, Unit>,
pub p2pkhaddressindex_to_addressdata: Store<P2PKHAddressIndex, AddressData>,
pub p2pkhaddressindex_to_emptyaddressdata: Store<P2PKHAddressIndex, EmptyAddressData>,
pub p2pkhaddressindex_to_outputs_received: Store<P2PKHAddressIndexOutputindex, Unit>,
pub p2pkhaddressindex_to_outputs_sent: Store<P2PKHAddressIndexOutputindex, Unit>,
pub p2shaddressindex_to_addressdata: Store<P2SHAddressIndex, AddressData>,
pub p2shaddressindex_to_emptyaddressdata: Store<P2SHAddressIndex, EmptyAddressData>,
pub p2shaddressindex_to_outputs_received: Store<P2SHAddressIndexOutputindex, Unit>,
pub p2shaddressindex_to_outputs_sent: Store<P2SHAddressIndexOutputindex, Unit>,
pub p2traddressindex_to_addressdata: Store<P2TRAddressIndex, AddressData>,
pub p2traddressindex_to_emptyaddressdata: Store<P2TRAddressIndex, EmptyAddressData>,
pub p2traddressindex_to_outputs_received: Store<P2TRAddressIndexOutputindex, Unit>,
pub p2traddressindex_to_outputs_sent: Store<P2TRAddressIndexOutputindex, Unit>,
pub p2wpkhaddressindex_to_addressdata: Store<P2WPKHAddressIndex, AddressData>,
pub p2wpkhaddressindex_to_emptyaddressdata: Store<P2WPKHAddressIndex, EmptyAddressData>,
pub p2wpkhaddressindex_to_outputs_received: Store<P2WPKHAddressIndexOutputindex, Unit>,
pub p2wpkhaddressindex_to_outputs_sent: Store<P2WPKHAddressIndexOutputindex, Unit>,
pub p2wshaddressindex_to_addressdata: Store<P2WSHAddressIndex, AddressData>,
pub p2wshaddressindex_to_emptyaddressdata: Store<P2WSHAddressIndex, EmptyAddressData>,
pub p2wshaddressindex_to_outputs_received: Store<P2WSHAddressIndexOutputindex, Unit>,
pub p2wshaddressindex_to_outputs_sent: Store<P2WSHAddressIndexOutputindex, Unit>,
}
impl Stores {
pub fn import(
path: &Path,
version: Version,
keyspace: &TransactionalKeyspace,
) -> color_eyre::Result<Self> {
pub fn import(path: &Path, version: Version) -> color_eyre::Result<Self> {
let (
(
p2aaddressindex_to_addressdata,
p2aaddressindex_to_emptyaddressdata,
p2aaddressindex_to_outputs_received,
p2aaddressindex_to_outputs_sent,
),
(
p2pk33addressindex_to_addressdata,
p2pk33addressindex_to_emptyaddressdata,
p2pk33addressindex_to_outputs_received,
p2pk33addressindex_to_outputs_sent,
),
(
p2pk65addressindex_to_addressdata,
p2pk65addressindex_to_emptyaddressdata,
p2pk65addressindex_to_outputs_received,
p2pk65addressindex_to_outputs_sent,
),
(
p2pkhaddressindex_to_addressdata,
p2pkhaddressindex_to_emptyaddressdata,
p2pkhaddressindex_to_outputs_received,
p2pkhaddressindex_to_outputs_sent,
),
(
p2shaddressindex_to_addressdata,
p2shaddressindex_to_emptyaddressdata,
p2shaddressindex_to_outputs_received,
p2shaddressindex_to_outputs_sent,
),
(
p2traddressindex_to_addressdata,
p2traddressindex_to_emptyaddressdata,
p2traddressindex_to_outputs_received,
p2traddressindex_to_outputs_sent,
),
(
p2wpkhaddressindex_to_addressdata,
p2wpkhaddressindex_to_emptyaddressdata,
p2wpkhaddressindex_to_outputs_received,
p2wpkhaddressindex_to_outputs_sent,
),
(
p2wshaddressindex_to_addressdata,
p2wshaddressindex_to_emptyaddressdata,
p2wshaddressindex_to_outputs_received,
p2wshaddressindex_to_outputs_sent,
),
(p2aaddressindex_to_addressdata, p2aaddressindex_to_emptyaddressdata),
(p2pk33addressindex_to_addressdata, p2pk33addressindex_to_emptyaddressdata),
(p2pk65addressindex_to_addressdata, p2pk65addressindex_to_emptyaddressdata),
(p2pkhaddressindex_to_addressdata, p2pkhaddressindex_to_emptyaddressdata),
(p2shaddressindex_to_addressdata, p2shaddressindex_to_emptyaddressdata),
(p2traddressindex_to_addressdata, p2traddressindex_to_emptyaddressdata),
(p2wpkhaddressindex_to_addressdata, p2wpkhaddressindex_to_emptyaddressdata),
(p2wshaddressindex_to_addressdata, p2wshaddressindex_to_emptyaddressdata),
) = thread::scope(|scope| {
let p2a = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2aaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
@@ -133,36 +54,18 @@ impl Stores {
)
.unwrap(),
Store::import(
keyspace,
path,
"p2aaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2aaddressindex_to_outputs_received",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2aaddressindex_to_outputs_sent",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2pk33 = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2pk33addressindex_to_addressdata",
version + VERSION + Version::ZERO,
@@ -170,36 +73,18 @@ impl Stores {
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pk33addressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pk33addressindex_to_outputs_received",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pk33addressindex_to_outputs_sent",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2pk65 = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2pk65addressindex_to_addressdata",
version + VERSION + Version::ZERO,
@@ -207,36 +92,18 @@ impl Stores {
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pk65addressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pk65addressindex_to_outputs_received",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pk65addressindex_to_outputs_sent",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2pkh = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2pkhaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
@@ -244,36 +111,18 @@ impl Stores {
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pkhaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pkhaddressindex_to_outputs_received",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2pkhaddressindex_to_outputs_sent",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2sh = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2shaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
@@ -281,36 +130,18 @@ impl Stores {
)
.unwrap(),
Store::import(
keyspace,
path,
"p2shaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2shaddressindex_to_outputs_received",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2shaddressindex_to_outputs_sent",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2tr = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2traddressindex_to_addressdata",
version + VERSION + Version::ZERO,
@@ -318,36 +149,18 @@ impl Stores {
)
.unwrap(),
Store::import(
keyspace,
path,
"p2traddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2traddressindex_to_outputs_received",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2traddressindex_to_outputs_sent",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2wpkh = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2wpkhaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
@@ -355,36 +168,18 @@ impl Stores {
)
.unwrap(),
Store::import(
keyspace,
path,
"p2wpkhaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2wpkhaddressindex_to_outputs_received",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2wpkhaddressindex_to_outputs_sent",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
let p2wsh = scope.spawn(|| {
(
Store::import(
keyspace,
path,
"p2wshaddressindex_to_addressdata",
version + VERSION + Version::ZERO,
@@ -392,29 +187,12 @@ impl Stores {
)
.unwrap(),
Store::import(
keyspace,
path,
"p2wshaddressindex_to_emptyaddressdata",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2wshaddressindex_to_outputs_received",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
Store::import(
keyspace,
path,
"p2wshaddressindex_to_outputs_sent",
version + VERSION + Version::ZERO,
None,
)
.unwrap(),
)
});
@@ -431,47 +209,29 @@ impl Stores {
});
Ok(Self {
keyspace: keyspace.clone(),
p2aaddressindex_to_addressdata,
p2aaddressindex_to_emptyaddressdata,
p2aaddressindex_to_outputs_received,
p2aaddressindex_to_outputs_sent,
p2pk33addressindex_to_addressdata,
p2pk33addressindex_to_emptyaddressdata,
p2pk33addressindex_to_outputs_received,
p2pk33addressindex_to_outputs_sent,
p2pk65addressindex_to_addressdata,
p2pk65addressindex_to_emptyaddressdata,
p2pk65addressindex_to_outputs_received,
p2pk65addressindex_to_outputs_sent,
p2pkhaddressindex_to_addressdata,
p2pkhaddressindex_to_emptyaddressdata,
p2pkhaddressindex_to_outputs_received,
p2pkhaddressindex_to_outputs_sent,
p2shaddressindex_to_addressdata,
p2shaddressindex_to_emptyaddressdata,
p2shaddressindex_to_outputs_received,
p2shaddressindex_to_outputs_sent,
p2traddressindex_to_addressdata,
p2traddressindex_to_emptyaddressdata,
p2traddressindex_to_outputs_received,
p2traddressindex_to_outputs_sent,
p2wpkhaddressindex_to_addressdata,
p2wpkhaddressindex_to_emptyaddressdata,
p2wpkhaddressindex_to_outputs_received,
p2wpkhaddressindex_to_outputs_sent,
p2wshaddressindex_to_addressdata,
p2wshaddressindex_to_emptyaddressdata,
p2wshaddressindex_to_outputs_received,
p2wshaddressindex_to_outputs_sent,
})
}
@@ -489,13 +249,7 @@ impl Stores {
self.as_mut_slice()
.into_iter()
.try_for_each(|store| store.reset())?;
info!("Persisting stores...");
self.keyspace
.persist(PersistMode::SyncAll)
.map_err(|e| e.into())
.try_for_each(|store| store.reset())
}
pub fn get_addressdata(
@@ -585,8 +339,6 @@ impl Stores {
pub fn commit(
&mut self,
height: Height,
sent: AddressTypeToTypeIndexVec<OutputIndex>,
received: AddressTypeToTypeIndexVec<OutputIndex>,
addresstype_to_typeindex_to_addressdata: AddressTypeToTypeIndexTree<
WithAddressDataSource<AddressData>,
>,
@@ -811,191 +563,7 @@ impl Stores {
});
});
thread::scope(|s| {
let GroupedByAddressType {
p2pk65,
p2pk33,
p2pkh,
p2sh,
p2wpkh,
p2wsh,
p2tr,
p2a,
} = received.unwrap();
s.spawn(|| {
self.p2aaddressindex_to_outputs_received.commit_(
height,
[].into_iter(),
p2a.into_iter()
.map(P2AAddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2pk33addressindex_to_outputs_received.commit_(
height,
[].into_iter(),
p2pk33
.into_iter()
.map(P2PK33AddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2pk65addressindex_to_outputs_received.commit_(
height,
[].into_iter(),
p2pk65
.into_iter()
.map(P2PK65AddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2pkhaddressindex_to_outputs_received.commit_(
height,
[].into_iter(),
p2pkh
.into_iter()
.map(P2PKHAddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2shaddressindex_to_outputs_received.commit_(
height,
[].into_iter(),
p2sh.into_iter()
.map(P2SHAddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2traddressindex_to_outputs_received.commit_(
height,
[].into_iter(),
p2tr.into_iter()
.map(P2TRAddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2wpkhaddressindex_to_outputs_received.commit_(
height,
[].into_iter(),
p2wpkh
.into_iter()
.map(P2WPKHAddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2wshaddressindex_to_outputs_received.commit_(
height,
[].into_iter(),
p2wsh
.into_iter()
.map(P2WSHAddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
});
thread::scope(|s| {
let GroupedByAddressType {
p2pk65,
p2pk33,
p2pkh,
p2sh,
p2wpkh,
p2wsh,
p2tr,
p2a,
} = sent.unwrap();
s.spawn(|| {
self.p2aaddressindex_to_outputs_sent.commit_(
height,
[].into_iter(),
p2a.into_iter()
.map(P2AAddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2pk33addressindex_to_outputs_sent.commit_(
height,
[].into_iter(),
p2pk33
.into_iter()
.map(P2PK33AddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2pk65addressindex_to_outputs_sent.commit_(
height,
[].into_iter(),
p2pk65
.into_iter()
.map(P2PK65AddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2pkhaddressindex_to_outputs_sent.commit_(
height,
[].into_iter(),
p2pkh
.into_iter()
.map(P2PKHAddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2shaddressindex_to_outputs_sent.commit_(
height,
[].into_iter(),
p2sh.into_iter()
.map(P2SHAddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2traddressindex_to_outputs_sent.commit_(
height,
[].into_iter(),
p2tr.into_iter()
.map(P2TRAddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2wpkhaddressindex_to_outputs_sent.commit_(
height,
[].into_iter(),
p2wpkh
.into_iter()
.map(P2WPKHAddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
s.spawn(|| {
self.p2wshaddressindex_to_outputs_sent.commit_(
height,
[].into_iter(),
p2wsh
.into_iter()
.map(P2WSHAddressIndexOutputindex::from)
.map(|i| (i, Unit)),
)
});
});
self.keyspace
.persist(PersistMode::SyncAll)
.map_err(|e| e.into())
Ok(())
}
pub fn rotate_memtables(&self) {
@@ -1004,77 +572,45 @@ impl Stores {
.for_each(|store| store.rotate_memtable());
}
pub fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 32] {
pub fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 16] {
[
&self.p2aaddressindex_to_addressdata,
&self.p2aaddressindex_to_emptyaddressdata,
&self.p2aaddressindex_to_outputs_received,
&self.p2aaddressindex_to_outputs_sent,
&self.p2pk33addressindex_to_addressdata,
&self.p2pk33addressindex_to_emptyaddressdata,
&self.p2pk33addressindex_to_outputs_received,
&self.p2pk33addressindex_to_outputs_sent,
&self.p2pk65addressindex_to_addressdata,
&self.p2pk65addressindex_to_emptyaddressdata,
&self.p2pk65addressindex_to_outputs_received,
&self.p2pk65addressindex_to_outputs_sent,
&self.p2pkhaddressindex_to_addressdata,
&self.p2pkhaddressindex_to_emptyaddressdata,
&self.p2pkhaddressindex_to_outputs_received,
&self.p2pkhaddressindex_to_outputs_sent,
&self.p2shaddressindex_to_addressdata,
&self.p2shaddressindex_to_emptyaddressdata,
&self.p2shaddressindex_to_outputs_received,
&self.p2shaddressindex_to_outputs_sent,
&self.p2traddressindex_to_addressdata,
&self.p2traddressindex_to_emptyaddressdata,
&self.p2traddressindex_to_outputs_received,
&self.p2traddressindex_to_outputs_sent,
&self.p2wpkhaddressindex_to_addressdata,
&self.p2wpkhaddressindex_to_emptyaddressdata,
&self.p2wpkhaddressindex_to_outputs_received,
&self.p2wpkhaddressindex_to_outputs_sent,
&self.p2wshaddressindex_to_addressdata,
&self.p2wshaddressindex_to_emptyaddressdata,
&self.p2wshaddressindex_to_outputs_received,
&self.p2wshaddressindex_to_outputs_sent,
]
}
fn as_mut_slice(&mut self) -> [&mut (dyn AnyStore + Send + Sync); 32] {
fn as_mut_slice(&mut self) -> [&mut (dyn AnyStore + Send + Sync); 16] {
[
&mut self.p2aaddressindex_to_addressdata,
&mut self.p2aaddressindex_to_emptyaddressdata,
&mut self.p2aaddressindex_to_outputs_received,
&mut self.p2aaddressindex_to_outputs_sent,
&mut self.p2pk33addressindex_to_addressdata,
&mut self.p2pk33addressindex_to_emptyaddressdata,
&mut self.p2pk33addressindex_to_outputs_received,
&mut self.p2pk33addressindex_to_outputs_sent,
&mut self.p2pk65addressindex_to_addressdata,
&mut self.p2pk65addressindex_to_emptyaddressdata,
&mut self.p2pk65addressindex_to_outputs_received,
&mut self.p2pk65addressindex_to_outputs_sent,
&mut self.p2pkhaddressindex_to_addressdata,
&mut self.p2pkhaddressindex_to_emptyaddressdata,
&mut self.p2pkhaddressindex_to_outputs_received,
&mut self.p2pkhaddressindex_to_outputs_sent,
&mut self.p2shaddressindex_to_addressdata,
&mut self.p2shaddressindex_to_emptyaddressdata,
&mut self.p2shaddressindex_to_outputs_received,
&mut self.p2shaddressindex_to_outputs_sent,
&mut self.p2traddressindex_to_addressdata,
&mut self.p2traddressindex_to_emptyaddressdata,
&mut self.p2traddressindex_to_outputs_received,
&mut self.p2traddressindex_to_outputs_sent,
&mut self.p2wpkhaddressindex_to_addressdata,
&mut self.p2wpkhaddressindex_to_emptyaddressdata,
&mut self.p2wpkhaddressindex_to_outputs_received,
&mut self.p2wpkhaddressindex_to_outputs_sent,
&mut self.p2wshaddressindex_to_addressdata,
&mut self.p2wshaddressindex_to_emptyaddressdata,
&mut self.p2wshaddressindex_to_outputs_received,
&mut self.p2wshaddressindex_to_outputs_sent,
]
}
}
crates/brk_computer/src/vecs/stateful/address_cohorts.rs
+7 -7
View File
@@ -1,17 +1,17 @@
use std::path::Path;
use brk_core::{Height, Result, Version};
use brk_core::{
AddressGroups, GroupFilter, GroupedByFromSize, GroupedBySizeRange, GroupedByUpToSize, Height,
Result, Version,
};
use brk_exit::Exit;
use brk_vec::{Computation, Format};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use crate::{
AddressGroups, GroupFilter, GroupedByFromSize, GroupedBySizeRange, GroupedByUpToSize,
vecs::{
Indexes, fetched,
stateful::{address_cohort, r#trait::CohortVecs},
},
use crate::vecs::{
Indexes, fetched,
stateful::{address_cohort, r#trait::CohortVecs},
};
const VERSION: Version = Version::new(0);
crates/brk_computer/src/vecs/stateful/addresstype_to_addresscount.rs
+47
View File
@@ -0,0 +1,47 @@
use brk_core::{GroupedByAddressType, Height};
use brk_vec::VecIterator;
use derive_deref::{Deref, DerefMut};
use crate::vecs::stateful::addresstype_to_addresscount_vec::AddressTypeToAddressCountVec;
#[derive(Debug, Default, Deref, DerefMut)]
pub struct AddressTypeToAddressCount(GroupedByAddressType<usize>);
impl From<(&AddressTypeToAddressCountVec, Height)> for AddressTypeToAddressCount {
fn from((groups, starting_height): (&AddressTypeToAddressCountVec, Height)) -> Self {
if let Some(prev_height) = starting_height.decremented() {
Self(GroupedByAddressType {
p2pk65: groups
.p2pk65
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2pk33: groups
.p2pk33
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2pkh: groups
.p2pkh
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2sh: groups.p2sh.into_iter().unwrap_get_inner(prev_height).into(),
p2wpkh: groups
.p2wpkh
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2wsh: groups
.p2wsh
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2tr: groups.p2tr.into_iter().unwrap_get_inner(prev_height).into(),
p2a: groups.p2a.into_iter().unwrap_get_inner(prev_height).into(),
})
} else {
Default::default()
}
}
}
crates/brk_computer/src/vecs/stateful/addresstype_to_addresscount_vec.rs
+43
View File
@@ -0,0 +1,43 @@
use brk_core::{GroupedByAddressType, Height, Result, StoredUsize};
use brk_exit::Exit;
use brk_vec::EagerVec;
use derive_deref::{Deref, DerefMut};
use crate::vecs::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;
#[derive(Debug, Clone, Deref, DerefMut)]
pub struct AddressTypeToAddressCountVec(GroupedByAddressType<EagerVec<Height, StoredUsize>>);
impl From<GroupedByAddressType<EagerVec<Height, StoredUsize>>> for AddressTypeToAddressCountVec {
fn from(value: GroupedByAddressType<EagerVec<Height, StoredUsize>>) -> Self {
Self(value)
}
}
impl AddressTypeToAddressCountVec {
pub fn forced_push_at(
&mut self,
height: Height,
addresstype_to_usize: &AddressTypeToAddressCount,
exit: &Exit,
) -> Result<()> {
self.p2pk65
.forced_push_at(height, addresstype_to_usize.p2pk65.into(), exit)?;
self.p2pk33
.forced_push_at(height, addresstype_to_usize.p2pk33.into(), exit)?;
self.p2pkh
.forced_push_at(height, addresstype_to_usize.p2pkh.into(), exit)?;
self.p2sh
.forced_push_at(height, addresstype_to_usize.p2sh.into(), exit)?;
self.p2wpkh
.forced_push_at(height, addresstype_to_usize.p2wpkh.into(), exit)?;
self.p2wsh
.forced_push_at(height, addresstype_to_usize.p2wsh.into(), exit)?;
self.p2tr
.forced_push_at(height, addresstype_to_usize.p2tr.into(), exit)?;
self.p2a
.forced_push_at(height, addresstype_to_usize.p2a.into(), exit)?;
Ok(())
}
}
crates/brk_computer/src/vecs/stateful/addresstype_to_typeindex_vec.rs
+1 -90
View File
@@ -1,10 +1,8 @@
use std::mem;
use brk_core::{AddressData, Dollars, OutputIndex, Sats, TypeIndex};
use brk_core::TypeIndex;
use derive_deref::{Deref, DerefMut};
use crate::vecs::stateful::WithAddressDataSource;
use super::GroupedByAddressType;
#[derive(Debug, Default, Deref, DerefMut)]
@@ -30,91 +28,4 @@ impl<T> AddressTypeToTypeIndexVec<T> {
mem::swap(own, other);
}
}
pub fn unwrap(self) -> GroupedByAddressType<Vec<(TypeIndex, T)>> {
self.0
}
}
impl AddressTypeToTypeIndexVec<OutputIndex> {
#[allow(clippy::type_complexity)]
pub fn extend_from_sent(
&mut self,
other: &AddressTypeToTypeIndexVec<(
OutputIndex,
Sats,
Option<WithAddressDataSource<AddressData>>,
Option<Dollars>,
usize,
f64,
bool,
)>,
) {
Self::extend_from_sent_(&mut self.p2pk33, &other.p2pk33);
Self::extend_from_sent_(&mut self.p2pkh, &other.p2pkh);
Self::extend_from_sent_(&mut self.p2sh, &other.p2sh);
Self::extend_from_sent_(&mut self.p2wpkh, &other.p2wpkh);
Self::extend_from_sent_(&mut self.p2wsh, &other.p2wsh);
Self::extend_from_sent_(&mut self.p2tr, &other.p2tr);
Self::extend_from_sent_(&mut self.p2a, &other.p2a);
}
#[allow(clippy::type_complexity)]
fn extend_from_sent_(
own: &mut Vec<(TypeIndex, OutputIndex)>,
other: &[(
TypeIndex,
(
OutputIndex,
Sats,
Option<WithAddressDataSource<AddressData>>,
Option<Dollars>,
usize,
f64,
bool,
),
)],
) {
own.extend(
other
.iter()
.map(|(type_index, (output_index, ..))| (*type_index, *output_index)),
);
}
pub fn extend_from_received(
&mut self,
other: &AddressTypeToTypeIndexVec<(
OutputIndex,
Sats,
Option<WithAddressDataSource<AddressData>>,
)>,
) {
Self::extend_from_received_(&mut self.p2pk33, &other.p2pk33);
Self::extend_from_received_(&mut self.p2pkh, &other.p2pkh);
Self::extend_from_received_(&mut self.p2sh, &other.p2sh);
Self::extend_from_received_(&mut self.p2wpkh, &other.p2wpkh);
Self::extend_from_received_(&mut self.p2wsh, &other.p2wsh);
Self::extend_from_received_(&mut self.p2tr, &other.p2tr);
Self::extend_from_received_(&mut self.p2a, &other.p2a);
}
#[allow(clippy::type_complexity)]
fn extend_from_received_(
own: &mut Vec<(TypeIndex, OutputIndex)>,
other: &[(
TypeIndex,
(
OutputIndex,
Sats,
Option<WithAddressDataSource<AddressData>>,
),
)],
) {
own.extend(
other
.iter()
.map(|(type_index, (output_index, ..))| (*type_index, *output_index)),
);
}
}
crates/brk_computer/src/vecs/stateful/mod.rs
+132 -151
View File
@@ -1,8 +1,8 @@
use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread};
use brk_core::{
AddressData, CheckedSub, DateIndex, Dollars, EmptyAddressData, Height, InputIndex, OutputIndex,
OutputType, Result, Sats, StoredUsize, Version,
AddressData, CheckedSub, DateIndex, Dollars, EmptyAddressData, GroupedByAddressType, Height,
InputIndex, OutputIndex, OutputType, Result, Sats, Version,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
@@ -14,7 +14,15 @@ use log::info;
use rayon::prelude::*;
use crate::{
BlockState, GroupedByAddressType, SupplyState, Transacted, stores::Stores, vecs::market,
BlockState, SupplyState, Transacted,
stores::Stores,
vecs::{
market,
stateful::{
addresstype_to_addresscount::AddressTypeToAddressCount,
addresstype_to_addresscount_vec::AddressTypeToAddressCountVec,
},
},
};
use super::{
@@ -25,6 +33,8 @@ use super::{
mod address_cohort;
mod address_cohorts;
mod addresstype_to_addresscount;
mod addresstype_to_addresscount_vec;
mod addresstype_to_typeindex_tree;
mod addresstype_to_typeindex_vec;
mod common;
@@ -50,9 +60,8 @@ pub struct Vecs {
pub indexes_to_opreturn_supply: ComputedValueVecsFromHeight,
// pub height_to_address_count: EagerVec<Height, StoredUsize>,
// pub height_to_empty_address_count: EagerVec<Height, StoredUsize>,
pub addresstype_to_height_to_address_count: GroupedByAddressType<EagerVec<Height, StoredUsize>>,
pub addresstype_to_height_to_empty_address_count:
GroupedByAddressType<EagerVec<Height, StoredUsize>>,
pub addresstype_to_height_to_address_count: AddressTypeToAddressCountVec,
pub addresstype_to_height_to_empty_address_count: AddressTypeToAddressCountVec,
pub utxo_vecs: utxo_cohorts::Vecs,
pub address_vecs: address_cohorts::Vecs,
}
@@ -122,106 +131,110 @@ impl Vecs {
// version + VERSION + Version::ZERO,
// format,
// )?,
addresstype_to_height_to_address_count: GroupedByAddressType {
p2pk65: EagerVec::forced_import(
path,
"p2pk65_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
path,
"p2pk33_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
path,
"p2pkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
path,
"p2sh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
path,
"p2wpkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
path,
"p2wsh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
path,
"p2tr_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
path,
"p2a_address_count",
version + VERSION + Version::ZERO,
format,
)?,
},
addresstype_to_height_to_empty_address_count: GroupedByAddressType {
p2pk65: EagerVec::forced_import(
path,
"p2pk65_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
path,
"p2pk33_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
path,
"p2pkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
path,
"p2sh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
path,
"p2wpkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
path,
"p2wsh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
path,
"p2tr_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
path,
"p2a_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
},
addresstype_to_height_to_address_count: AddressTypeToAddressCountVec::from(
GroupedByAddressType {
p2pk65: EagerVec::forced_import(
path,
"p2pk65_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
path,
"p2pk33_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
path,
"p2pkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
path,
"p2sh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
path,
"p2wpkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
path,
"p2wsh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
path,
"p2tr_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
path,
"p2a_address_count",
version + VERSION + Version::ZERO,
format,
)?,
},
),
addresstype_to_height_to_empty_address_count: AddressTypeToAddressCountVec::from(
GroupedByAddressType {
p2pk65: EagerVec::forced_import(
path,
"p2pk65_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
path,
"p2pk33_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
path,
"p2pkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
path,
"p2sh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
path,
"p2wpkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
path,
"p2wsh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
path,
"p2tr_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
path,
"p2a_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
},
),
utxo_vecs: utxo_cohorts::Vecs::forced_import(
path,
version,
@@ -439,21 +452,17 @@ impl Vecs {
} else {
Sats::ZERO
};
let mut addresstype_to_address_count = GroupedByAddressType::<usize>::from((
let mut addresstype_to_address_count = AddressTypeToAddressCount::from((
&self.addresstype_to_height_to_address_count,
starting_height,
));
let mut addresstype_to_empty_address_count = GroupedByAddressType::<usize>::from((
let mut addresstype_to_empty_address_count = AddressTypeToAddressCount::from((
&self.addresstype_to_height_to_empty_address_count,
starting_height,
));
let mut height = starting_height;
let mut addresstype_to_typeindex_to_sent_outputindex =
AddressTypeToTypeIndexVec::<OutputIndex>::default();
let mut addresstype_to_typeindex_to_received_outputindex =
AddressTypeToTypeIndexVec::<OutputIndex>::default();
let mut addresstype_to_typeindex_to_addressdata =
AddressTypeToTypeIndexTree::<WithAddressDataSource<AddressData>>::default();
let mut addresstype_to_typeindex_to_emptyaddressdata =
@@ -490,8 +499,8 @@ impl Vecs {
let input_count = height_to_input_count_iter.unwrap_get_inner(height);
let (
(mut height_to_sent, addresstype_to_typedindex_to_sent_outputindex, addresstype_to_typedindex_to_sent_data),
(mut received, addresstype_to_typedindex_to_received_outputindex, addresstype_to_typedindex_to_received_data),
(mut height_to_sent, addresstype_to_typedindex_to_sent_data),
(mut received, addresstype_to_typedindex_to_received_data),
) = thread::scope(|s| {
if chain_state_starting_height <= height {
s.spawn(|| {
@@ -592,7 +601,6 @@ impl Vecs {
value,
input_type,
typeindex,
outputindex,
addressdata_opt,
prev_price,
blocks_old,
@@ -604,7 +612,6 @@ impl Vecs {
|| {
(
BTreeMap::<Height, Transacted>::default(),
AddressTypeToTypeIndexVec::<OutputIndex>::default(),
AddressTypeToTypeIndexVec::<(
Sats,
Option<WithAddressDataSource<AddressData>>,
@@ -616,13 +623,12 @@ impl Vecs {
),
)
},
|(mut tree, mut vecs, mut vecs2),
|(mut tree, mut vecs),
(
height,
value,
input_type,
typeindex,
outputindex,
addressdata_opt,
prev_price,
blocks_old,
@@ -631,9 +637,6 @@ impl Vecs {
)| {
tree.entry(height).or_default().iterate(value, input_type);
if let Some(vec) = vecs.get_mut(input_type) {
vec.push((typeindex, outputindex));
}
if let Some(vec) = vecs2.get_mut(input_type) {
vec.push((
typeindex,
(
@@ -646,14 +649,13 @@ impl Vecs {
),
));
}
(tree, vecs, vecs2)
(tree, vecs)
},
)
.reduce(
|| {
(
BTreeMap::<Height, Transacted>::default(),
AddressTypeToTypeIndexVec::<OutputIndex>::default(),
AddressTypeToTypeIndexVec::<(
Sats,
Option<WithAddressDataSource<AddressData>>,
@@ -665,7 +667,7 @@ impl Vecs {
),
)
},
|(first_tree, mut source_vecs, mut source_vecs2), (second_tree, other_vecs, other_vecs2)| {
|(first_tree, mut source_vecs), (second_tree, other_vecs)| {
let (mut tree_source, tree_to_consume) =
if first_tree.len() > second_tree.len() {
(first_tree, second_tree)
@@ -676,8 +678,7 @@ impl Vecs {
*tree_source.entry(k).or_default() += v;
});
source_vecs.merge(other_vecs);
source_vecs2.merge(other_vecs2);
(tree_source, source_vecs, source_vecs2)
(tree_source, source_vecs)
},
)
});
@@ -738,13 +739,12 @@ impl Vecs {
None
};
(value, output_type, typeindex, outputindex, addressdata_opt)
(value, output_type, typeindex, addressdata_opt)
})
.fold(
|| {
(
Transacted::default(),
AddressTypeToTypeIndexVec::<OutputIndex>::default(),
AddressTypeToTypeIndexVec::<(
Sats,
Option<WithAddressDataSource<AddressData>>,
@@ -752,35 +752,27 @@ impl Vecs {
),
)
},
|(mut transacted, mut vecs, mut vecs2),
|(mut transacted, mut vecs),
(
value,
output_type,
typeindex,
outputindex,
addressdata_opt,
)| {
transacted.iterate(value, output_type);
if let Some(vec) = vecs.get_mut(output_type) {
vec.push((
typeindex,
outputindex,
));
}
if let Some(vec) = vecs2.get_mut(output_type) {
vec.push((
typeindex,
(value, addressdata_opt),
));
}
(transacted, vecs, vecs2)
(transacted, vecs)
},
)
.reduce(
|| {
(
Transacted::default(),
AddressTypeToTypeIndexVec::<OutputIndex>::default(),
AddressTypeToTypeIndexVec::<(
Sats,
Option<WithAddressDataSource<AddressData>>,
@@ -788,10 +780,9 @@ impl Vecs {
),
)
},
|(transacted, mut vecs, mut vecs2), (other_transacted, other_vecs, other_vecs2)| {
|(transacted, mut vecs), (other_transacted, other_vecs)| {
vecs.merge(other_vecs);
vecs2.merge(other_vecs2);
(transacted + other_transacted, vecs, vecs2)
(transacted + other_transacted, vecs)
},
)
});
@@ -829,12 +820,6 @@ impl Vecs {
};
thread::scope(|scope| -> Result<()> {
scope.spawn(|| addresstype_to_typeindex_to_sent_outputindex
.merge(addresstype_to_typedindex_to_sent_outputindex));
scope.spawn(|| addresstype_to_typeindex_to_received_outputindex
.merge(addresstype_to_typedindex_to_received_outputindex));
scope.spawn(|| {
// Push current block state before processing sends and receives
chain_state.push(BlockState {
@@ -956,8 +941,6 @@ impl Vecs {
self.flush_states(height, &chain_state, exit)?;
stores.commit(
height,
mem::take(&mut addresstype_to_typeindex_to_sent_outputindex),
mem::take(&mut addresstype_to_typeindex_to_received_outputindex),
mem::take(&mut addresstype_to_typeindex_to_addressdata),
mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata),
)?;
@@ -974,8 +957,6 @@ impl Vecs {
self.flush_states(height, &chain_state, exit)?;
stores.commit(
height,
mem::take(&mut addresstype_to_typeindex_to_sent_outputindex),
mem::take(&mut addresstype_to_typeindex_to_received_outputindex),
mem::take(&mut addresstype_to_typeindex_to_addressdata),
mem::take(&mut addresstype_to_typeindex_to_emptyaddressdata),
)?;
crates/brk_computer/src/vecs/stateful/utxo_cohorts.rs
+6 -4
View File
@@ -1,15 +1,17 @@
use std::{collections::BTreeMap, ops::ControlFlow, path::Path};
use brk_core::{CheckedSub, Dollars, HalvingEpoch, Height, Result, Timestamp, Version};
use brk_core::{
CheckedSub, Dollars, GroupFilter, GroupedByDateRange, GroupedByEpoch, GroupedByFromDate,
GroupedByFromSize, GroupedBySizeRange, GroupedBySpendableType, GroupedByTerm,
GroupedByUpToDate, GroupedByUpToSize, HalvingEpoch, Height, Result, Timestamp, UTXOGroups,
Version,
};
use brk_exit::Exit;
use brk_vec::{Computation, Format, StoredIndex};
use derive_deref::{Deref, DerefMut};
use rayon::prelude::*;
use crate::{
GroupFilter, GroupedByDateRange, GroupedByEpoch, GroupedByFromDate, GroupedByFromSize,
GroupedBySizeRange, GroupedBySpendableType, GroupedByTerm, GroupedByUpToDate,
GroupedByUpToSize, UTXOGroups,
states::{BlockState, Transacted},
vecs::{Indexes, fetched},
};
crates/brk_computer/src/groups/address.rs → crates/brk_core/src/groups/address.rs
View File
crates/brk_computer/src/groups/by_address_type.rs → crates/brk_core/src/groups/by_address_type.rs
+1 -75
View File
@@ -3,11 +3,8 @@ use std::{
ops::{Add, AddAssign},
};
use brk_core::{Height, OutputType, Result, StoredUsize};
use brk_exit::Exit;
use brk_vec::{EagerVec, VecIterator};
use super::GroupFilter;
use crate::OutputType;
#[derive(Default, Clone, Debug)]
pub struct GroupedByAddressType<T> {
@@ -170,74 +167,3 @@ where
self.p2a += rhs.p2a;
}
}
impl From<(&GroupedByAddressType<EagerVec<Height, StoredUsize>>, Height)>
for GroupedByAddressType<usize>
{
fn from(
(groups, starting_height): (&GroupedByAddressType<EagerVec<Height, StoredUsize>>, Height),
) -> Self {
if let Some(prev_height) = starting_height.decremented() {
Self {
p2pk65: groups
.p2pk65
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2pk33: groups
.p2pk33
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2pkh: groups
.p2pkh
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2sh: groups.p2sh.into_iter().unwrap_get_inner(prev_height).into(),
p2wpkh: groups
.p2wpkh
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2wsh: groups
.p2wsh
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2tr: groups.p2tr.into_iter().unwrap_get_inner(prev_height).into(),
p2a: groups.p2a.into_iter().unwrap_get_inner(prev_height).into(),
}
} else {
Default::default()
}
}
}
impl GroupedByAddressType<EagerVec<Height, StoredUsize>> {
pub fn forced_push_at(
&mut self,
height: Height,
addresstype_to_usize: &GroupedByAddressType<usize>,
exit: &Exit,
) -> Result<()> {
self.p2pk65
.forced_push_at(height, addresstype_to_usize.p2pk65.into(), exit)?;
self.p2pk33
.forced_push_at(height, addresstype_to_usize.p2pk33.into(), exit)?;
self.p2pkh
.forced_push_at(height, addresstype_to_usize.p2pkh.into(), exit)?;
self.p2sh
.forced_push_at(height, addresstype_to_usize.p2sh.into(), exit)?;
self.p2wpkh
.forced_push_at(height, addresstype_to_usize.p2wpkh.into(), exit)?;
self.p2wsh
.forced_push_at(height, addresstype_to_usize.p2wsh.into(), exit)?;
self.p2tr
.forced_push_at(height, addresstype_to_usize.p2tr.into(), exit)?;
self.p2a
.forced_push_at(height, addresstype_to_usize.p2a.into(), exit)?;
Ok(())
}
}
crates/brk_computer/src/groups/by_date_range.rs → crates/brk_core/src/groups/by_date_range.rs
View File
crates/brk_computer/src/groups/by_epoch.rs → crates/brk_core/src/groups/by_epoch.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use brk_core::{HalvingEpoch, Height};
use crate::{HalvingEpoch, Height};
use super::GroupFilter;
crates/brk_computer/src/groups/by_from_date.rs → crates/brk_core/src/groups/by_from_date.rs
View File
crates/brk_computer/src/groups/by_from_size.rs → crates/brk_core/src/groups/by_from_size.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use brk_core::Sats;
use crate::Sats;
use super::GroupFilter;
crates/brk_computer/src/groups/by_size_range.rs → crates/brk_core/src/groups/by_size_range.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign};
use brk_core::Sats;
use crate::Sats;
use super::GroupFilter;
crates/brk_computer/src/groups/by_spendable_type.rs → crates/brk_core/src/groups/by_spendable_type.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign};
use brk_core::OutputType;
use crate::OutputType;
use super::GroupFilter;
crates/brk_computer/src/groups/by_term.rs → crates/brk_core/src/groups/by_term.rs
View File
crates/brk_computer/src/groups/by_type.rs → crates/brk_core/src/groups/by_type.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign};
use brk_core::OutputType;
use crate::OutputType;
use super::{GroupedBySpendableType, GroupedByUnspendableType};
crates/brk_computer/src/groups/by_unspendable_type.rs → crates/brk_core/src/groups/by_unspendable_type.rs
View File
crates/brk_computer/src/groups/by_up_to_date.rs → crates/brk_core/src/groups/by_up_to_date.rs
View File
crates/brk_computer/src/groups/by_up_to_size.rs → crates/brk_core/src/groups/by_up_to_size.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use brk_core::Sats;
use crate::Sats;
use super::GroupFilter;
crates/brk_computer/src/groups/by_value.rs → crates/brk_core/src/groups/by_value.rs
View File
crates/brk_computer/src/groups/filter.rs → crates/brk_core/src/groups/filter.rs
+1 -1
View File
@@ -1,6 +1,6 @@
use std::ops::Range;
use brk_core::{HalvingEpoch, OutputType};
use crate::{HalvingEpoch, OutputType};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum GroupFilter {
crates/brk_computer/src/groups/mod.rs → crates/brk_core/src/groups/mod.rs
View File
crates/brk_computer/src/groups/utxo.rs → crates/brk_core/src/groups/utxo.rs
View File
crates/brk_core/src/lib.rs
+2
View File
@@ -1,11 +1,13 @@
#![doc = include_str!("../README.md")]
mod error;
mod groups;
mod structs;
mod traits;
mod utils;
pub use error::*;
pub use groups::*;
pub use structs::*;
pub use traits::*;
pub use utils::*;
crates/brk_core/src/structs/mod.rs
+2 -16
View File
@@ -22,22 +22,14 @@ mod opreturnindex;
mod outputindex;
mod outputtype;
mod p2aaddressindex;
mod p2aaddressindex_outputindex;
mod p2msoutputindex;
mod p2pk33addressindex;
mod p2pk33addressindex_outputindex;
mod p2pk65addressindex;
mod p2pk65addressindex_outputindex;
mod p2pkhaddressindex;
mod p2pkhaddressindex_outputindex;
mod p2shaddressindex;
mod p2shaddressindex_outputindex;
mod p2traddressindex;
mod p2traddressindex_outputindex;
mod p2wpkhaddressindex;
mod p2wpkhaddressindex_outputindex;
mod p2wshaddressindex;
mod p2wshaddressindex_outputindex;
mod quarterindex;
mod rawlocktime;
mod sats;
@@ -53,6 +45,7 @@ mod txidprefix;
mod txindex;
mod txversion;
mod typeindex;
mod typeindex_with_outputindex;
mod unit;
mod unknownoutputindex;
mod version;
@@ -86,22 +79,14 @@ pub use opreturnindex::*;
pub use outputindex::*;
pub use outputtype::*;
pub use p2aaddressindex::*;
pub use p2aaddressindex_outputindex::*;
pub use p2msoutputindex::*;
pub use p2pk33addressindex::*;
pub use p2pk33addressindex_outputindex::*;
pub use p2pk65addressindex::*;
pub use p2pk65addressindex_outputindex::*;
pub use p2pkhaddressindex::*;
pub use p2pkhaddressindex_outputindex::*;
pub use p2shaddressindex::*;
pub use p2shaddressindex_outputindex::*;
pub use p2traddressindex::*;
pub use p2traddressindex_outputindex::*;
pub use p2wpkhaddressindex::*;
pub use p2wpkhaddressindex_outputindex::*;
pub use p2wshaddressindex::*;
pub use p2wshaddressindex_outputindex::*;
pub use quarterindex::*;
pub use rawlocktime::*;
pub use sats::*;
@@ -117,6 +102,7 @@ pub use txidprefix::*;
pub use txindex::*;
pub use txversion::*;
pub use typeindex::*;
pub use typeindex_with_outputindex::*;
pub use unit::*;
pub use unknownoutputindex::*;
pub use version::*;
crates/brk_core/src/structs/p2aaddressindex_outputindex.rs
-55
View File
@@ -1,55 +0,0 @@
use byteview::ByteView;
use serde::Serialize;
use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes};
use super::{OutputIndex, P2AAddressIndex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize)]
pub struct P2AAddressIndexOutputindex {
addressindex: P2AAddressIndex,
outputindex: OutputIndex,
}
impl From<(TypeIndex, OutputIndex)> for P2AAddressIndexOutputindex {
fn from(value: (TypeIndex, OutputIndex)) -> Self {
Self::from((P2AAddressIndex::from(value.0), value.1))
}
}
impl From<(P2AAddressIndex, OutputIndex)> for P2AAddressIndexOutputindex {
fn from(value: (P2AAddressIndex, OutputIndex)) -> Self {
Self {
addressindex: value.0,
outputindex: value.1,
}
}
}
impl From<ByteView> for P2AAddressIndexOutputindex {
fn from(value: ByteView) -> Self {
let addressindex =
P2AAddressIndex::from(u32::from_be_bytes(copy_first_4bytes(&value).unwrap()));
let outputindex = OutputIndex::from(u64::from_be_bytes(copy_first_8bytes(&value).unwrap()));
Self {
addressindex,
outputindex,
}
}
}
impl From<P2AAddressIndexOutputindex> for ByteView {
fn from(value: P2AAddressIndexOutputindex) -> Self {
ByteView::from(&value)
}
}
impl From<&P2AAddressIndexOutputindex> for ByteView {
fn from(value: &P2AAddressIndexOutputindex) -> Self {
ByteView::from(
[
u32::from(value.addressindex).to_be_bytes().as_slice(),
u64::from(value.outputindex).to_be_bytes().as_slice(),
]
.concat(),
)
}
}
crates/brk_core/src/structs/p2pk33addressindex_outputindex.rs
-55
View File
@@ -1,55 +0,0 @@
use byteview::ByteView;
use serde::Serialize;
use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes};
use super::{OutputIndex, P2PK33AddressIndex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize)]
pub struct P2PK33AddressIndexOutputindex {
addressindex: P2PK33AddressIndex,
outputindex: OutputIndex,
}
impl From<(TypeIndex, OutputIndex)> for P2PK33AddressIndexOutputindex {
fn from(value: (TypeIndex, OutputIndex)) -> Self {
Self::from((P2PK33AddressIndex::from(value.0), value.1))
}
}
impl From<(P2PK33AddressIndex, OutputIndex)> for P2PK33AddressIndexOutputindex {
fn from(value: (P2PK33AddressIndex, OutputIndex)) -> Self {
Self {
addressindex: value.0,
outputindex: value.1,
}
}
}
impl From<ByteView> for P2PK33AddressIndexOutputindex {
fn from(value: ByteView) -> Self {
let addressindex =
P2PK33AddressIndex::from(u32::from_be_bytes(copy_first_4bytes(&value).unwrap()));
let outputindex = OutputIndex::from(u64::from_be_bytes(copy_first_8bytes(&value).unwrap()));
Self {
addressindex,
outputindex,
}
}
}
impl From<P2PK33AddressIndexOutputindex> for ByteView {
fn from(value: P2PK33AddressIndexOutputindex) -> Self {
ByteView::from(&value)
}
}
impl From<&P2PK33AddressIndexOutputindex> for ByteView {
fn from(value: &P2PK33AddressIndexOutputindex) -> Self {
ByteView::from(
[
u32::from(value.addressindex).to_be_bytes().as_slice(),
u64::from(value.outputindex).to_be_bytes().as_slice(),
]
.concat(),
)
}
}
crates/brk_core/src/structs/p2pk65addressindex_outputindex.rs
-55
View File
@@ -1,55 +0,0 @@
use byteview::ByteView;
use serde::Serialize;
use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes};
use super::{OutputIndex, P2PK65AddressIndex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize)]
pub struct P2PK65AddressIndexOutputindex {
addressindex: P2PK65AddressIndex,
outputindex: OutputIndex,
}
impl From<(TypeIndex, OutputIndex)> for P2PK65AddressIndexOutputindex {
fn from(value: (TypeIndex, OutputIndex)) -> Self {
Self::from((P2PK65AddressIndex::from(value.0), value.1))
}
}
impl From<(P2PK65AddressIndex, OutputIndex)> for P2PK65AddressIndexOutputindex {
fn from(value: (P2PK65AddressIndex, OutputIndex)) -> Self {
Self {
addressindex: value.0,
outputindex: value.1,
}
}
}
impl From<ByteView> for P2PK65AddressIndexOutputindex {
fn from(value: ByteView) -> Self {
let addressindex =
P2PK65AddressIndex::from(u32::from_be_bytes(copy_first_4bytes(&value).unwrap()));
let outputindex = OutputIndex::from(u64::from_be_bytes(copy_first_8bytes(&value).unwrap()));
Self {
addressindex,
outputindex,
}
}
}
impl From<P2PK65AddressIndexOutputindex> for ByteView {
fn from(value: P2PK65AddressIndexOutputindex) -> Self {
ByteView::from(&value)
}
}
impl From<&P2PK65AddressIndexOutputindex> for ByteView {
fn from(value: &P2PK65AddressIndexOutputindex) -> Self {
ByteView::from(
[
u32::from(value.addressindex).to_be_bytes().as_slice(),
u64::from(value.outputindex).to_be_bytes().as_slice(),
]
.concat(),
)
}
}
crates/brk_core/src/structs/p2pkhaddressindex_outputindex.rs
-55
View File
@@ -1,55 +0,0 @@
use byteview::ByteView;
use serde::Serialize;
use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes};
use super::{OutputIndex, P2PKHAddressIndex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize)]
pub struct P2PKHAddressIndexOutputindex {
addressindex: P2PKHAddressIndex,
outputindex: OutputIndex,
}
impl From<(TypeIndex, OutputIndex)> for P2PKHAddressIndexOutputindex {
fn from(value: (TypeIndex, OutputIndex)) -> Self {
Self::from((P2PKHAddressIndex::from(value.0), value.1))
}
}
impl From<(P2PKHAddressIndex, OutputIndex)> for P2PKHAddressIndexOutputindex {
fn from(value: (P2PKHAddressIndex, OutputIndex)) -> Self {
Self {
addressindex: value.0,
outputindex: value.1,
}
}
}
impl From<ByteView> for P2PKHAddressIndexOutputindex {
fn from(value: ByteView) -> Self {
let addressindex =
P2PKHAddressIndex::from(u32::from_be_bytes(copy_first_4bytes(&value).unwrap()));
let outputindex = OutputIndex::from(u64::from_be_bytes(copy_first_8bytes(&value).unwrap()));
Self {
addressindex,
outputindex,
}
}
}
impl From<P2PKHAddressIndexOutputindex> for ByteView {
fn from(value: P2PKHAddressIndexOutputindex) -> Self {
ByteView::from(&value)
}
}
impl From<&P2PKHAddressIndexOutputindex> for ByteView {
fn from(value: &P2PKHAddressIndexOutputindex) -> Self {
ByteView::from(
[
u32::from(value.addressindex).to_be_bytes().as_slice(),
u64::from(value.outputindex).to_be_bytes().as_slice(),
]
.concat(),
)
}
}
crates/brk_core/src/structs/p2shaddressindex_outputindex.rs
-55
View File
@@ -1,55 +0,0 @@
use byteview::ByteView;
use serde::Serialize;
use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes};
use super::{OutputIndex, P2SHAddressIndex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize)]
pub struct P2SHAddressIndexOutputindex {
addressindex: P2SHAddressIndex,
outputindex: OutputIndex,
}
impl From<(TypeIndex, OutputIndex)> for P2SHAddressIndexOutputindex {
fn from(value: (TypeIndex, OutputIndex)) -> Self {
Self::from((P2SHAddressIndex::from(value.0), value.1))
}
}
impl From<(P2SHAddressIndex, OutputIndex)> for P2SHAddressIndexOutputindex {
fn from(value: (P2SHAddressIndex, OutputIndex)) -> Self {
Self {
addressindex: value.0,
outputindex: value.1,
}
}
}
impl From<ByteView> for P2SHAddressIndexOutputindex {
fn from(value: ByteView) -> Self {
let addressindex =
P2SHAddressIndex::from(u32::from_be_bytes(copy_first_4bytes(&value).unwrap()));
let outputindex = OutputIndex::from(u64::from_be_bytes(copy_first_8bytes(&value).unwrap()));
Self {
addressindex,
outputindex,
}
}
}
impl From<P2SHAddressIndexOutputindex> for ByteView {
fn from(value: P2SHAddressIndexOutputindex) -> Self {
ByteView::from(&value)
}
}
impl From<&P2SHAddressIndexOutputindex> for ByteView {
fn from(value: &P2SHAddressIndexOutputindex) -> Self {
ByteView::from(
[
u32::from(value.addressindex).to_be_bytes().as_slice(),
u64::from(value.outputindex).to_be_bytes().as_slice(),
]
.concat(),
)
}
}
crates/brk_core/src/structs/p2traddressindex_outputindex.rs
-55
View File
@@ -1,55 +0,0 @@
use byteview::ByteView;
use serde::Serialize;
use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes};
use super::{OutputIndex, P2TRAddressIndex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize)]
pub struct P2TRAddressIndexOutputindex {
addressindex: P2TRAddressIndex,
outputindex: OutputIndex,
}
impl From<(TypeIndex, OutputIndex)> for P2TRAddressIndexOutputindex {
fn from(value: (TypeIndex, OutputIndex)) -> Self {
Self::from((P2TRAddressIndex::from(value.0), value.1))
}
}
impl From<(P2TRAddressIndex, OutputIndex)> for P2TRAddressIndexOutputindex {
fn from(value: (P2TRAddressIndex, OutputIndex)) -> Self {
Self {
addressindex: value.0,
outputindex: value.1,
}
}
}
impl From<ByteView> for P2TRAddressIndexOutputindex {
fn from(value: ByteView) -> Self {
let addressindex =
P2TRAddressIndex::from(u32::from_be_bytes(copy_first_4bytes(&value).unwrap()));
let outputindex = OutputIndex::from(u64::from_be_bytes(copy_first_8bytes(&value).unwrap()));
Self {
addressindex,
outputindex,
}
}
}
impl From<P2TRAddressIndexOutputindex> for ByteView {
fn from(value: P2TRAddressIndexOutputindex) -> Self {
ByteView::from(&value)
}
}
impl From<&P2TRAddressIndexOutputindex> for ByteView {
fn from(value: &P2TRAddressIndexOutputindex) -> Self {
ByteView::from(
[
u32::from(value.addressindex).to_be_bytes().as_slice(),
u64::from(value.outputindex).to_be_bytes().as_slice(),
]
.concat(),
)
}
}
crates/brk_core/src/structs/p2wpkhaddressindex_outputindex.rs
-55
View File
@@ -1,55 +0,0 @@
use byteview::ByteView;
use serde::Serialize;
use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes};
use super::{OutputIndex, P2WPKHAddressIndex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize)]
pub struct P2WPKHAddressIndexOutputindex {
addressindex: P2WPKHAddressIndex,
outputindex: OutputIndex,
}
impl From<(TypeIndex, OutputIndex)> for P2WPKHAddressIndexOutputindex {
fn from(value: (TypeIndex, OutputIndex)) -> Self {
Self::from((P2WPKHAddressIndex::from(value.0), value.1))
}
}
impl From<(P2WPKHAddressIndex, OutputIndex)> for P2WPKHAddressIndexOutputindex {
fn from(value: (P2WPKHAddressIndex, OutputIndex)) -> Self {
Self {
addressindex: value.0,
outputindex: value.1,
}
}
}
impl From<ByteView> for P2WPKHAddressIndexOutputindex {
fn from(value: ByteView) -> Self {
let addressindex =
P2WPKHAddressIndex::from(u32::from_be_bytes(copy_first_4bytes(&value).unwrap()));
let outputindex = OutputIndex::from(u64::from_be_bytes(copy_first_8bytes(&value).unwrap()));
Self {
addressindex,
outputindex,
}
}
}
impl From<P2WPKHAddressIndexOutputindex> for ByteView {
fn from(value: P2WPKHAddressIndexOutputindex) -> Self {
ByteView::from(&value)
}
}
impl From<&P2WPKHAddressIndexOutputindex> for ByteView {
fn from(value: &P2WPKHAddressIndexOutputindex) -> Self {
ByteView::from(
[
u32::from(value.addressindex).to_be_bytes().as_slice(),
u64::from(value.outputindex).to_be_bytes().as_slice(),
]
.concat(),
)
}
}
crates/brk_core/src/structs/p2wshaddressindex_outputindex.rs
-55
View File
@@ -1,55 +0,0 @@
use byteview::ByteView;
use serde::Serialize;
use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes};
use super::{OutputIndex, P2WSHAddressIndex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize)]
pub struct P2WSHAddressIndexOutputindex {
addressindex: P2WSHAddressIndex,
outputindex: OutputIndex,
}
impl From<(TypeIndex, OutputIndex)> for P2WSHAddressIndexOutputindex {
fn from(value: (TypeIndex, OutputIndex)) -> Self {
Self::from((P2WSHAddressIndex::from(value.0), value.1))
}
}
impl From<(P2WSHAddressIndex, OutputIndex)> for P2WSHAddressIndexOutputindex {
fn from(value: (P2WSHAddressIndex, OutputIndex)) -> Self {
Self {
addressindex: value.0,
outputindex: value.1,
}
}
}
impl From<ByteView> for P2WSHAddressIndexOutputindex {
fn from(value: ByteView) -> Self {
let addressindex =
P2WSHAddressIndex::from(u32::from_be_bytes(copy_first_4bytes(&value).unwrap()));
let outputindex = OutputIndex::from(u64::from_be_bytes(copy_first_8bytes(&value).unwrap()));
Self {
addressindex,
outputindex,
}
}
}
impl From<P2WSHAddressIndexOutputindex> for ByteView {
fn from(value: P2WSHAddressIndexOutputindex) -> Self {
ByteView::from(&value)
}
}
impl From<&P2WSHAddressIndexOutputindex> for ByteView {
fn from(value: &P2WSHAddressIndexOutputindex) -> Self {
ByteView::from(
[
u32::from(value.addressindex).to_be_bytes().as_slice(),
u64::from(value.outputindex).to_be_bytes().as_slice(),
]
.concat(),
)
}
}
crates/brk_core/src/structs/typeindex_with_outputindex.rs
+48
View File
@@ -0,0 +1,48 @@
use byteview::ByteView;
use serde::Serialize;
use crate::{TypeIndex, copy_first_4bytes, copy_first_8bytes};
use super::OutputIndex;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize)]
pub struct TypeIndexWithOutputindex {
typeindex: TypeIndex,
outputindex: OutputIndex,
}
impl From<(TypeIndex, OutputIndex)> for TypeIndexWithOutputindex {
fn from(value: (TypeIndex, OutputIndex)) -> Self {
Self {
typeindex: value.0,
outputindex: value.1,
}
}
}
impl From<ByteView> for TypeIndexWithOutputindex {
fn from(value: ByteView) -> Self {
let typeindex = TypeIndex::from(u32::from_be_bytes(copy_first_4bytes(&value).unwrap()));
let outputindex = OutputIndex::from(u64::from_be_bytes(copy_first_8bytes(&value).unwrap()));
Self {
typeindex,
outputindex,
}
}
}
impl From<TypeIndexWithOutputindex> for ByteView {
fn from(value: TypeIndexWithOutputindex) -> Self {
ByteView::from(&value)
}
}
impl From<&TypeIndexWithOutputindex> for ByteView {
fn from(value: &TypeIndexWithOutputindex) -> Self {
ByteView::from(
[
u32::from(value.typeindex).to_be_bytes().as_slice(),
u64::from(value.outputindex).to_be_bytes().as_slice(),
]
.concat(),
)
}
}
crates/brk_indexer/Cargo.toml
-1
View File
@@ -17,6 +17,5 @@ brk_parser = { workspace = true }
brk_store = { workspace = true }
brk_vec = { workspace = true }
color-eyre = { workspace = true }
fjall = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }
crates/brk_indexer/src/lib.rs
+6 -2
View File
@@ -7,8 +7,8 @@ use std::{collections::BTreeMap, path::Path, str::FromStr, thread};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
OutputType, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, Version, Vin, Vout,
setrlimit,
OutputType, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, TypeIndexWithOutputindex,
Unit, Version, Vin, Vout, setrlimit,
};
use bitcoin::{Transaction, TxIn, TxOut};
@@ -547,6 +547,10 @@ impl Indexer {
new_txindexvout_to_outputindex
.insert((txindex, vout), outputindex);
if outputtype.is_address() {
stores.addresstype_to_typeindex_with_outputindex.get_mut(outputtype).unwrap().insert_if_needed(TypeIndexWithOutputindex::from((typeindex, outputindex)), Unit, height);
}
Ok(())
},
)?;
crates/brk_indexer/src/stores.rs
+148 -28
View File
@@ -1,12 +1,11 @@
use std::{borrow::Cow, fs, path::Path, thread};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, Result, TxIndex,
TxidPrefix, TypeIndex, Version,
AddressBytes, AddressBytesHash, BlockHashPrefix, GroupedByAddressType, Height, OutputType,
Result, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version,
};
use brk_store::{AnyStore, Store};
use brk_vec::AnyIterableVec;
use fjall::{PersistMode, TransactionalKeyspace};
use rayon::prelude::*;
use crate::Indexes;
@@ -15,10 +14,11 @@ use super::Vecs;
#[derive(Clone)]
pub struct Stores {
pub keyspace: TransactionalKeyspace,
pub addressbyteshash_to_typeindex: Store<AddressBytesHash, TypeIndex>,
pub blockhashprefix_to_height: Store<BlockHashPrefix, Height>,
pub txidprefix_to_txindex: Store<TxidPrefix, TxIndex>,
pub addresstype_to_typeindex_with_outputindex:
GroupedByAddressType<Store<TypeIndexWithOutputindex, Unit>>,
}
const VERSION: Version = Version::ZERO;
@@ -27,18 +27,9 @@ impl Stores {
pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
let keyspace = match brk_store::open_keyspace(path) {
Ok(keyspace) => keyspace,
Err(_) => {
fs::remove_dir_all(path)?;
return Self::forced_import(path, version);
}
};
thread::scope(|scope| {
let addressbyteshash_to_typeindex = scope.spawn(|| {
Store::import(
&keyspace,
path,
"addressbyteshash_to_typeindex",
version + VERSION + Version::ZERO,
@@ -47,7 +38,6 @@ impl Stores {
});
let blockhashprefix_to_height = scope.spawn(|| {
Store::import(
&keyspace,
path,
"blockhashprefix_to_height",
version + VERSION + Version::ZERO,
@@ -56,19 +46,107 @@ impl Stores {
});
let txidprefix_to_txindex = scope.spawn(|| {
Store::import(
&keyspace,
path,
"txidprefix_to_txindex",
version + VERSION + Version::ZERO,
None,
)
});
let p2aaddressindex_with_outputindex_to_output_status = scope.spawn(|| {
Store::import(
path,
"p2aaddressindex_with_outputindex_to_output_status",
version + VERSION + Version::ZERO,
None,
)
});
let p2pk33addressindex_with_outputindex_to_output_status = scope.spawn(|| {
Store::import(
path,
"p2pk33addressindex_with_outputindex_to_output_status",
version + VERSION + Version::ZERO,
None,
)
});
let p2pk65addressindex_with_outputindex_to_output_status = scope.spawn(|| {
Store::import(
path,
"p2pk65addressindex_with_outputindex_to_output_status",
version + VERSION + Version::ZERO,
None,
)
});
let p2pkhaddressindex_with_outputindex_to_output_status = scope.spawn(|| {
Store::import(
path,
"p2pkhaddressindex_with_outputindex_to_output_status",
version + VERSION + Version::ZERO,
None,
)
});
let p2shaddressindex_with_outputindex_to_output_status = scope.spawn(|| {
Store::import(
path,
"p2shaddressindex_with_outputindex_to_output_status",
version + VERSION + Version::ZERO,
None,
)
});
let p2traddressindex_with_outputindex_to_output_status = scope.spawn(|| {
Store::import(
path,
"p2traddressindex_with_outputindex_to_output_status",
version + VERSION + Version::ZERO,
None,
)
});
let p2wpkhaddressindex_with_outputindex_to_output_status = scope.spawn(|| {
Store::import(
path,
"p2wpkhaddressindex_with_outputindex_to_output_status",
version + VERSION + Version::ZERO,
None,
)
});
let p2wshaddressindex_with_outputindex_to_output_status = scope.spawn(|| {
Store::import(
path,
"p2wshaddressindex_with_outputindex_to_output_status",
version + VERSION + Version::ZERO,
None,
)
});
Ok(Self {
keyspace: keyspace.clone(),
addressbyteshash_to_typeindex: addressbyteshash_to_typeindex.join().unwrap()?,
blockhashprefix_to_height: blockhashprefix_to_height.join().unwrap()?,
txidprefix_to_txindex: txidprefix_to_txindex.join().unwrap()?,
addresstype_to_typeindex_with_outputindex: GroupedByAddressType {
p2pk65: p2pk65addressindex_with_outputindex_to_output_status
.join()
.unwrap()?,
p2pk33: p2pk33addressindex_with_outputindex_to_output_status
.join()
.unwrap()?,
p2pkh: p2pkhaddressindex_with_outputindex_to_output_status
.join()
.unwrap()?,
p2sh: p2shaddressindex_with_outputindex_to_output_status
.join()
.unwrap()?,
p2wpkh: p2wpkhaddressindex_with_outputindex_to_output_status
.join()
.unwrap()?,
p2wsh: p2wshaddressindex_with_outputindex_to_output_status
.join()
.unwrap()?,
p2tr: p2traddressindex_with_outputindex_to_output_status
.join()
.unwrap()?,
p2a: p2aaddressindex_with_outputindex_to_output_status
.join()
.unwrap()?,
},
})
})
}
@@ -78,9 +156,41 @@ impl Stores {
vecs: &mut Vecs,
starting_indexes: &Indexes,
) -> color_eyre::Result<()> {
if self.addressbyteshash_to_typeindex.is_empty()
&& self.blockhashprefix_to_height.is_empty()
&& self.txidprefix_to_txindex.is_empty()
if self.addressbyteshash_to_typeindex.is_empty()?
&& self.blockhashprefix_to_height.is_empty()?
&& self.txidprefix_to_txindex.is_empty()?
&& self
.addresstype_to_typeindex_with_outputindex
.p2a
.is_empty()?
&& self
.addresstype_to_typeindex_with_outputindex
.p2pk33
.is_empty()?
&& self
.addresstype_to_typeindex_with_outputindex
.p2pk65
.is_empty()?
&& self
.addresstype_to_typeindex_with_outputindex
.p2pkh
.is_empty()?
&& self
.addresstype_to_typeindex_with_outputindex
.p2sh
.is_empty()?
&& self
.addresstype_to_typeindex_with_outputindex
.p2tr
.is_empty()?
&& self
.addresstype_to_typeindex_with_outputindex
.p2wpkh
.is_empty()?
&& self
.addresstype_to_typeindex_with_outputindex
.p2wsh
.is_empty()?
{
return Ok(());
}
@@ -278,8 +388,6 @@ impl Stores {
self.txidprefix_to_txindex.reset()?;
}
self.keyspace.persist(PersistMode::SyncAll)?;
self.commit(starting_indexes.height.decremented().unwrap_or_default())?;
Ok(())
@@ -296,11 +404,7 @@ impl Stores {
pub fn commit(&mut self, height: Height) -> Result<()> {
self.as_mut_slice()
.into_par_iter()
.try_for_each(|store| store.commit(height))?;
self.keyspace
.persist(PersistMode::SyncAll)
.map_err(|e| e.into())
.try_for_each(|store| store.commit(height))
}
pub fn rotate_memtables(&self) {
@@ -309,19 +413,35 @@ impl Stores {
.for_each(|store| store.rotate_memtable());
}
fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 3] {
fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 11] {
[
&self.addressbyteshash_to_typeindex,
&self.blockhashprefix_to_height,
&self.txidprefix_to_txindex,
&self.addresstype_to_typeindex_with_outputindex.p2a,
&self.addresstype_to_typeindex_with_outputindex.p2pk33,
&self.addresstype_to_typeindex_with_outputindex.p2pk65,
&self.addresstype_to_typeindex_with_outputindex.p2pkh,
&self.addresstype_to_typeindex_with_outputindex.p2sh,
&self.addresstype_to_typeindex_with_outputindex.p2tr,
&self.addresstype_to_typeindex_with_outputindex.p2wpkh,
&self.addresstype_to_typeindex_with_outputindex.p2wsh,
]
}
fn as_mut_slice(&mut self) -> [&mut (dyn AnyStore + Send + Sync); 3] {
fn as_mut_slice(&mut self) -> [&mut (dyn AnyStore + Send + Sync); 11] {
[
&mut self.addressbyteshash_to_typeindex,
&mut self.blockhashprefix_to_height,
&mut self.txidprefix_to_txindex,
&mut self.addresstype_to_typeindex_with_outputindex.p2a,
&mut self.addresstype_to_typeindex_with_outputindex.p2pk33,
&mut self.addresstype_to_typeindex_with_outputindex.p2pk65,
&mut self.addresstype_to_typeindex_with_outputindex.p2pkh,
&mut self.addresstype_to_typeindex_with_outputindex.p2sh,
&mut self.addresstype_to_typeindex_with_outputindex.p2tr,
&mut self.addresstype_to_typeindex_with_outputindex.p2wpkh,
&mut self.addresstype_to_typeindex_with_outputindex.p2wsh,
]
}
}
crates/brk_mcp/Cargo.toml
-1
View File
@@ -12,4 +12,3 @@ axum = { workspace = true }
brk_interface = { workspace = true }
log = { workspace = true }
brk_rmcp = { workspace = true }
tracing = "0.1.41"
crates/brk_store/examples/main.rs
+1 -4
View File
@@ -6,10 +6,7 @@ use brk_store::{AnyStore, Store};
fn main() -> Result<()> {
let p = Path::new("./examples/_fjall");
let keyspace = brk_store::open_keyspace(p)?;
let mut store: Store<Dollars, Sats> =
brk_store::Store::import(&keyspace, p, "n", Version::ZERO, None)?;
let mut store: Store<Dollars, Sats> = brk_store::Store::import(p, "n", Version::ZERO, None)?;
store.insert_if_needed(Dollars::from(10.0), Sats::FIFTY_BTC, Height::ZERO);
crates/brk_store/src/lib.rs
+41 -36
View File
@@ -7,14 +7,15 @@ use std::{
borrow::Cow,
collections::{BTreeMap, BTreeSet},
fmt::Debug,
mem,
fs, mem,
path::Path,
};
use brk_core::{Height, Result, Version};
use byteview::ByteView;
use fjall::{
PartitionCreateOptions, ReadTransaction, TransactionalKeyspace, TransactionalPartitionHandle,
PartitionCreateOptions, PersistMode, ReadTransaction, TransactionalKeyspace,
TransactionalPartitionHandle,
};
mod meta;
@@ -28,6 +29,7 @@ pub struct Store<Key, Value> {
meta: StoreMeta,
name: &'static str,
keyspace: TransactionalKeyspace,
// Arc it
partition: Option<TransactionalPartitionHandle>,
rtx: ReadTransaction,
puts: BTreeMap<Key, Value>,
@@ -40,12 +42,6 @@ const DEFAULT_BLOOM_FILTER_BITS: Option<u8> = Some(5);
// const CHECK_COLLISIONS: bool = true;
const MAJOR_FJALL_VERSION: Version = Version::TWO;
pub fn open_keyspace(path: &Path) -> fjall::Result<TransactionalKeyspace> {
fjall::Config::new(path.join("fjall"))
.max_write_buffer_size(32 * 1024 * 1024)
.open_transactional()
}
impl<'a, K, V> Store<K, V>
where
K: Debug + Clone + From<ByteView> + Ord + 'a,
@@ -53,18 +49,32 @@ where
ByteView: From<K> + From<&'a K> + From<V>,
{
pub fn import(
keyspace: &TransactionalKeyspace,
path: &Path,
path_: &Path,
name: &str,
version: Version,
bloom_filter_bits: Option<Option<u8>>,
) -> Result<Self> {
let path = path_.join(name);
fs::create_dir_all(&path)?;
let keyspace = match fjall::Config::new(path.join("fjall"))
.max_write_buffer_size(32 * 1024 * 1024)
.open_transactional()
{
Ok(keyspace) => keyspace,
Err(_) => {
fs::remove_dir_all(path)?;
return Self::import(path_, name, version, bloom_filter_bits);
}
};
let (meta, partition) = StoreMeta::checked_open(
keyspace,
&path.join(format!("meta/{name}")),
&keyspace,
&path.join("meta"),
MAJOR_FJALL_VERSION + version,
|| {
Self::open_partition_handle(keyspace, name, bloom_filter_bits).inspect_err(|e| {
Self::open_partition_handle(&keyspace, bloom_filter_bits).inspect_err(|e| {
eprintln!("{e}");
eprintln!("Delete {path:?} and try again");
})
@@ -98,6 +108,12 @@ where
}
}
pub fn is_empty(&self) -> Result<bool> {
self.rtx
.is_empty(self.partition.as_ref().unwrap())
.map_err(|e| e.into())
}
// pub fn puts_first_key_value(&self) -> Option<(&K, &V)> {
// self.puts.first_key_value()
// }
@@ -138,9 +154,9 @@ where
}
pub fn remove(&mut self, key: K) {
if self.is_empty() {
return;
}
// if self.is_empty()? {
// return Ok(());
// }
if !self.puts.is_empty() {
unreachable!("Shouldn't reach this");
@@ -150,6 +166,8 @@ where
dbg!(key, &self.meta.path());
unreachable!();
}
// Ok(())
}
// pub fn retain_or_del<F>(&mut self, retain: F)
@@ -167,12 +185,11 @@ where
fn open_partition_handle(
keyspace: &TransactionalKeyspace,
name: &str,
bloom_filter_bits: Option<Option<u8>>,
) -> Result<TransactionalPartitionHandle> {
keyspace
.open_partition(
name,
"partition",
PartitionCreateOptions::default()
.bloom_filter_bits(bloom_filter_bits.unwrap_or(DEFAULT_BLOOM_FILTER_BITS))
.max_memtable_size(8 * 1024 * 1024)
@@ -191,7 +208,7 @@ where
return Ok(());
}
self.meta.export(self.len(), height)?;
self.meta.export(height)?;
let mut wtx = self.keyspace.write_tx();
@@ -217,6 +234,8 @@ where
wtx.commit()?;
self.keyspace.persist(PersistMode::SyncAll)?;
self.rtx = self.keyspace.read_tx();
Ok(())
@@ -249,11 +268,12 @@ where
self.meta.reset();
let partition =
Self::open_partition_handle(&self.keyspace, self.name, self.bloom_filter_bits)?;
let partition = Self::open_partition_handle(&self.keyspace, self.bloom_filter_bits)?;
self.partition.replace(partition);
self.keyspace.persist(PersistMode::SyncAll)?;
Ok(())
}
@@ -265,21 +285,6 @@ where
self.meta.height()
}
fn len(&self) -> usize {
let len = self.meta.len() + self.puts.len() - self.dels.len();
if len > 18440000000000000000 {
dbg!((
len,
self.meta.path(),
self.meta.len(),
self.puts.len(),
&self.dels,
));
unreachable!()
}
len
}
fn has(&self, height: Height) -> bool {
self.meta.has(height)
}
crates/brk_store/src/meta.rs
+2 -32
View File
@@ -3,7 +3,7 @@ use std::{
path::{Path, PathBuf},
};
use brk_core::{Result, Version, copy_first_8bytes};
use brk_core::{Result, Version};
use fjall::{TransactionalKeyspace, TransactionalPartitionHandle};
use super::Height;
@@ -13,7 +13,6 @@ pub struct StoreMeta {
pathbuf: PathBuf,
version: Version,
height: Option<Height>,
len: usize,
}
impl StoreMeta {
@@ -44,13 +43,10 @@ impl StoreMeta {
partition = open_partition_handle()?;
}
let len = Self::read_length_(path);
let slf = Self {
pathbuf: path.to_owned(),
version,
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
len,
};
slf.version.write(&slf.path_version())?;
@@ -58,28 +54,17 @@ impl StoreMeta {
Ok((slf, partition))
}
pub fn len(&self) -> usize {
self.len
}
// pub fn is_empty(&self) -> bool {
// self.len() == 0
// }
pub fn version(&self) -> Version {
self.version
}
pub fn export(&mut self, len: usize, height: Height) -> io::Result<()> {
self.len = len;
self.write_length()?;
pub fn export(&mut self, height: Height) -> io::Result<()> {
self.height = Some(height);
height.write(&self.path_height())
}
pub fn reset(&mut self) {
self.height.take();
self.len = 0
}
pub fn path(&self) -> &Path {
@@ -108,19 +93,4 @@ impl StoreMeta {
fn path_height_(path: &Path) -> PathBuf {
path.join("height")
}
fn read_length_(path: &Path) -> usize {
fs::read(Self::path_length(path))
.map(|v| usize::from_ne_bytes(copy_first_8bytes(v.as_slice()).unwrap()))
.unwrap_or_default()
}
fn write_length(&self) -> io::Result<()> {
Self::write_length_(&self.pathbuf, self.len)
}
fn write_length_(path: &Path, len: usize) -> io::Result<()> {
fs::write(Self::path_length(path), len.to_ne_bytes())
}
fn path_length(path: &Path) -> PathBuf {
path.join("length")
}
}
crates/brk_store/src/trait.rs
-6
View File
@@ -9,12 +9,6 @@ pub trait AnyStore {
fn height(&self) -> Option<Height>;
fn len(&self) -> usize;
fn is_empty(&self) -> bool {
self.len() == 0
}
fn has(&self, height: Height) -> bool;
fn needs(&self, height: Height) -> bool;