Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-05-27 15:19:53 +02:00
parent 34919aba05
commit 93e01902e3
84 changed files with 2445 additions and 1394 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
Cargo.lock generated
View File

@@ -393,6 +393,7 @@ dependencies = [
"brk_parser",
"brk_query",
"brk_server",
"brk_store",
"brk_vec",
]
@@ -400,6 +401,7 @@ dependencies = [
name = "brk_cli"
version = "0.0.40"
dependencies = [
"bitcoincore-rpc",
"brk_computer",
"brk_core",
"brk_exit",
@@ -416,6 +418,7 @@ dependencies = [
"log",
"serde",
"tabled",
"tokio",
"toml",
]
@@ -423,6 +426,8 @@ dependencies = [
name = "brk_computer"
version = "0.0.40"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
"brk_core",
"brk_exit",
"brk_fetcher",
@@ -435,6 +440,7 @@ dependencies = [
"color-eyre",
"derive_deref",
"fjall",
"jiff",
"log",
"rayon",
"serde",
@@ -458,6 +464,7 @@ dependencies = [
"serde",
"serde_bytes",
"serde_derive",
"serde_json",
"zerocopy",
"zerocopy-derive",
]
@@ -494,6 +501,7 @@ dependencies = [
"brk_exit",
"brk_logger",
"brk_parser",
"brk_store",
"brk_vec",
"byteview 0.7.0",
"color-eyre",
@@ -533,6 +541,7 @@ name = "brk_query"
version = "0.0.40"
dependencies = [
"brk_computer",
"brk_core",
"brk_indexer",
"brk_vec",
"clap",
@@ -550,6 +559,7 @@ name = "brk_server"
version = "0.0.40"
dependencies = [
"axum",
"bitcoincore-rpc",
"brk_computer",
"brk_core",
"brk_exit",
@@ -573,6 +583,17 @@ dependencies = [
"zip",
]
[[package]]
name = "brk_store"
version = "0.0.40"
dependencies = [
"brk_core",
"byteview 0.7.0",
"color-eyre",
"fjall",
"zerocopy",
]
[[package]]
name = "brk_vec"
version = "0.0.40"
@@ -2162,16 +2183,6 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "parking_lot"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.10"
@@ -2736,15 +2747,6 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "signal-hook-registry"
version = "1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410"
dependencies = [
"libc",
]
[[package]]
name = "simd-adler32"
version = "0.3.7"
@@ -2956,12 +2958,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779"
dependencies = [
"backtrace",
"bytes",
"libc",
"mio",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"tokio-macros",
"windows-sys 0.52.0",

2
Cargo.toml
View File

@@ -29,6 +29,7 @@ brk_logger = { version = "0", path = "crates/brk_logger" }
brk_parser = { version = "0", path = "crates/brk_parser" }
brk_query = { version = "0", path = "crates/brk_query" }
brk_server = { version = "0", path = "crates/brk_server" }
brk_store = { version = "0", path = "crates/brk_store" }
brk_vec = { version = "0", path = "crates/brk_vec" }
byteview = "0.7.0"
clap = { version = "4.5.38", features = ["string"] }
@@ -44,6 +45,7 @@ serde = { version = "1.0.219" }
serde_derive = "1.0.219"
serde_json = { version = "1.0.140", features = ["float_roundtrip"] }
tabled = "0.19.0"
tokio = { version = "1.45.1", features = ["rt-multi-thread"] }
zerocopy = { version = "0.8.25" }
zerocopy-derive = "0.8.25"

1
README.md
View File

@@ -77,6 +77,7 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
- [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust
- [`brk_query`](https://crates.io/crates/brk_query): A library that finds requested datasets.
- [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer`
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
## Acknowledgments

3
crates/brk/Cargo.toml
View File

@@ -18,6 +18,7 @@ full = [
"parser",
"query",
"server",
"store",
"vec",
]
core = ["brk_core"]
@@ -29,6 +30,7 @@ logger = ["brk_logger"]
parser = ["brk_parser"]
query = ["brk_query"]
server = ["brk_server"]
store = ["brk_store"]
vec = ["brk_vec"]
[dependencies]
@@ -42,6 +44,7 @@ brk_logger = { workspace = true, optional = true }
brk_parser = { workspace = true, optional = true }
brk_query = { workspace = true, optional = true }
brk_server = { workspace = true, optional = true }
brk_store = { workspace = true, optional = true }
brk_vec = { workspace = true, optional = true }
[package.metadata.docs.rs]

4
crates/brk/src/lib.rs
View File

@@ -36,6 +36,10 @@ pub use brk_query as query;
#[doc(inline)]
pub use brk_server as server;
#[cfg(feature = "store")]
#[doc(inline)]
pub use brk_store as store;
#[cfg(feature = "vec")]
#[doc(inline)]
pub use brk_vec as vec;

2
crates/brk_cli/Cargo.toml
View File

@@ -7,6 +7,7 @@ license.workspace = true
repository.workspace = true
[dependencies]
bitcoincore-rpc = { workspace = true }
brk_computer = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
@@ -23,6 +24,7 @@ color-eyre = { workspace = true }
log = { workspace = true }
serde = { workspace = true }
tabled = { workspace = true }
tokio = { workspace = true }
toml = "0.8.22"
[[bin]]

6
crates/brk_cli/src/run.rs
View File

@@ -5,13 +5,13 @@ use std::{
time::Duration,
};
use bitcoincore_rpc::{self, Auth, Client, RpcApi};
use brk_computer::Computer;
use brk_core::{default_bitcoin_path, default_brk_path, dot_brk_path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::rpc::{self, Auth, Client, RpcApi};
use brk_server::{Server, Website, tokio};
use brk_server::{Server, Website};
use brk_vec::Computation;
use clap_derive::{Parser, ValueEnum};
use color_eyre::eyre::eyre;
@@ -314,7 +314,7 @@ impl RunConfig {
}
pub fn rpc(&self) -> color_eyre::Result<&'static Client> {
Ok(Box::leak(Box::new(rpc::Client::new(
Ok(Box::leak(Box::new(Client::new(
&format!(
"http://{}:{}",
self.rpcconnect().unwrap_or(&"localhost".to_string()),

3
crates/brk_computer/Cargo.toml
View File

@@ -7,6 +7,8 @@ license.workspace = true
repository.workspace = true
[dependencies]
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
brk_fetcher = { workspace = true }
@@ -19,6 +21,7 @@ clap_derive = { workspace = true }
color-eyre = { workspace = true }
derive_deref = { workspace = true }
fjall = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }

6
crates/brk_computer/examples/main.rs
View File

@@ -5,7 +5,7 @@ use brk_core::{default_bitcoin_path, default_brk_path};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::{Parser, rpc};
use brk_parser::Parser;
use brk_vec::Computation;
pub fn main() -> color_eyre::Result<()> {
@@ -15,9 +15,9 @@ pub fn main() -> color_eyre::Result<()> {
let bitcoin_dir = default_bitcoin_path();
let rpc = Box::leak(Box::new(rpc::Client::new(
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();

4
crates/brk_computer/src/lib.rs
View File

@@ -5,11 +5,11 @@
use std::path::{Path, PathBuf};
use brk_core::Version;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
pub use brk_parser::rpc;
use brk_vec::{AnyCollectableVec, Compressed, Computation, Version};
use brk_vec::{AnyCollectableVec, Compressed, Computation};
mod states;
mod stores;

32
crates/brk_computer/src/states/cohort.rs
View File

@@ -2,7 +2,6 @@ use brk_core::Dollars;
use super::{RealizedState, SupplyState};
// Vecs ? probably
#[derive(Debug, Default, Clone)]
pub struct CohortState {
pub supply: SupplyState,
@@ -11,6 +10,12 @@ pub struct CohortState {
}
impl CohortState {
pub fn reset_single_iteration_values(&mut self) {
if let Some(realized) = self.realized.as_mut() {
realized.reset_single_iteration_values();
}
}
pub fn increment(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.supply += supply_state;
if let Some(realized) = self.realized.as_mut() {
@@ -25,7 +30,28 @@ impl CohortState {
}
}
pub fn send(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {}
pub fn receive(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.supply += supply_state;
if let Some(realized) = self.realized.as_mut() {
realized.receive(supply_state, price.unwrap());
}
}
pub fn receive(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {}
pub fn send(
&mut self,
supply_state: &SupplyState,
current_price: Option<Dollars>,
prev_price: Option<Dollars>,
older_than_hour: bool,
) {
self.supply -= supply_state;
if let Some(realized) = self.realized.as_mut() {
realized.send(
supply_state,
current_price.unwrap(),
prev_price.unwrap(),
older_than_hour,
);
}
}
}

26
crates/brk_computer/src/states/outputs/by_size.rs
View File

@@ -107,31 +107,31 @@ impl<T> OutputsBySize<T> {
pub fn get_mut(&mut self, group: usize) -> &mut T {
if group == 0 {
&mut self._0sat
} else if group == 10 {
} else if group == 1 {
&mut self.from_1sat_to_10sats
} else if group == 100 {
} else if group == 10 {
&mut self.from_10sats_to_100sats
} else if group == 1_000 {
} else if group == 100 {
&mut self.from_100sats_to_1_000sats
} else if group == 10_000 {
} else if group == 1_000 {
&mut self.from_1_000sats_to_10_000sats
} else if group == 100_000 {
} else if group == 10_000 {
&mut self.from_10_000sats_to_100_000sats
} else if group == 1_000_000 {
} else if group == 100_000 {
&mut self.from_100_000sats_to_1_000_000sats
} else if group == 10_000_000 {
} else if group == 1_000_000 {
&mut self.from_1_000_000sats_to_10_000_000sats
} else if group == 1_00_000_000 {
} else if group == 10_000_000 {
&mut self.from_10_000_000sats_to_1btc
} else if group == 10_00_000_000 {
} else if group == 1_00_000_000 {
&mut self.from_1btc_to_10btc
} else if group == 100_00_000_000 {
} else if group == 10_00_000_000 {
&mut self.from_10btc_to_100btc
} else if group == 1_000_00_000_000 {
} else if group == 100_00_000_000 {
&mut self.from_100btc_to_1_000btc
} else if group == 10_000_00_000_000 {
} else if group == 1_000_00_000_000 {
&mut self.from_1_000btc_to_10_000btc
} else if group == 100_000_00_000_000 {
} else if group == 10_000_00_000_000 {
&mut self.from_10_000btc_to_100_000btc
} else {
&mut self.from_100_000btc

4
crates/brk_computer/src/states/outputs/by_type.rs
View File

@@ -24,7 +24,7 @@ impl<T> OutputsByType<T> {
OutputType::P2A => &self.spendable.p2a,
OutputType::Empty => &self.spendable.empty,
OutputType::Unknown => &self.spendable.unknown,
OutputType::OpReturn => &self.unspendable.op_return,
OutputType::OpReturn => &self.unspendable.opreturn,
}
}
@@ -41,7 +41,7 @@ impl<T> OutputsByType<T> {
OutputType::P2A => &mut self.spendable.p2a,
OutputType::Unknown => &mut self.spendable.unknown,
OutputType::Empty => &mut self.spendable.empty,
OutputType::OpReturn => &mut self.unspendable.op_return,
OutputType::OpReturn => &mut self.unspendable.opreturn,
}
}

8
crates/brk_computer/src/states/outputs/by_unspendable_type.rs
View File

@@ -2,12 +2,12 @@ use std::ops::{Add, AddAssign};
#[derive(Default, Clone)]
pub struct OutputsByUnspendableType<T> {
pub op_return: T,
pub opreturn: T,
}
impl<T> OutputsByUnspendableType<T> {
pub fn as_vec(&self) -> [&T; 1] {
[&self.op_return]
[&self.opreturn]
}
}
@@ -18,7 +18,7 @@ where
type Output = Self;
fn add(self, rhs: Self) -> Self::Output {
Self {
op_return: self.op_return + rhs.op_return,
opreturn: self.opreturn + rhs.opreturn,
}
}
}
@@ -28,6 +28,6 @@ where
T: AddAssign,
{
fn add_assign(&mut self, rhs: Self) {
self.op_return += rhs.op_return;
self.opreturn += rhs.opreturn;
}
}

53
crates/brk_computer/src/states/outputs/mod.rs
View File

@@ -2,7 +2,7 @@ use brk_vec::StoredIndex;
use rayon::prelude::*;
use std::{collections::BTreeMap, ops::ControlFlow};
use brk_core::{Dollars, HalvingEpoch, Height, Timestamp};
use brk_core::{CheckedSub, Dollars, HalvingEpoch, Height, Timestamp};
mod by_epoch;
mod by_from;
@@ -133,16 +133,27 @@ impl Outputs<(OutputFilter, vecs::utxos::cohort::Vecs)> {
.collect::<Vec<_>>();
let last_timestamp = chain_state.last().unwrap().timestamp;
let current_price = chain_state.last().unwrap().price;
height_to_sent.into_iter().for_each(|(height, sent)| {
let block_state = chain_state.get(height.unwrap_to_usize()).unwrap();
let price = block_state.price;
let prev_price = block_state.price;
let days_old = block_state
.timestamp
.difference_in_days_between(last_timestamp);
self.all.1.state.decrement(&sent.spendable_supply, price);
let older_than_hour =
jiff::Timestamp::from(last_timestamp.checked_sub(block_state.timestamp).unwrap())
.as_second()
>= 60 * 60;
self.all.1.state.send(
&sent.spendable_supply,
current_price,
prev_price,
older_than_hour,
);
time_based_vecs
.iter_mut()
@@ -154,25 +165,32 @@ impl Outputs<(OutputFilter, vecs::utxos::cohort::Vecs)> {
_ => unreachable!(),
})
.for_each(|(_, vecs)| {
vecs.state.decrement(&sent.spendable_supply, price);
vecs.state.send(
&sent.spendable_supply,
current_price,
prev_price,
older_than_hour,
);
});
sent.by_type.spendable.as_typed_vec().into_iter().for_each(
|(output_type, supply_state)| {
self.by_type
.get_mut(output_type)
.1
.state
.decrement(supply_state, price)
self.by_type.get_mut(output_type).1.state.send(
supply_state,
current_price,
prev_price,
older_than_hour,
)
},
);
sent.by_size.into_iter().for_each(|(group, supply_state)| {
self.by_size
.get_mut(group)
.1
.state
.decrement(&supply_state, price);
self.by_size.get_mut(group).1.state.send(
&supply_state,
current_price,
prev_price,
older_than_hour,
);
});
});
}
@@ -189,7 +207,7 @@ impl Outputs<(OutputFilter, vecs::utxos::cohort::Vecs)> {
.into_iter()
.chain(self.by_up_to.as_mut_vec().map(|(_, v)| v))
.for_each(|v| {
v.state.increment(&supply_state, price);
v.state.receive(&supply_state, price);
});
self.by_type
@@ -200,8 +218,7 @@ impl Outputs<(OutputFilter, vecs::utxos::cohort::Vecs)> {
OutputFilter::Type(output_type) => *output_type,
_ => unreachable!(),
};
vecs.state
.increment(received.by_type.get(output_type), price)
vecs.state.receive(received.by_type.get(output_type), price)
});
received
@@ -212,7 +229,7 @@ impl Outputs<(OutputFilter, vecs::utxos::cohort::Vecs)> {
.get_mut(group)
.1
.state
.increment(&supply_state, price);
.receive(&supply_state, price);
});
}
}

111
crates/brk_computer/src/states/realized.rs
View File

@@ -1,49 +1,98 @@
use std::cmp::Ordering;
use brk_core::{Bitcoin, CheckedSub, Dollars};
use super::SupplyState;
#[derive(Debug, Default, Clone)]
pub struct RealizedState {
pub realized_cap: Dollars,
// pub realized_profit: Dollars, // sent price vs now price
// pub realized_loss: Dollars, // sent price vs now price
// pub value_created: Dollars, // supply * price now
// pub adjusted_value_created: Dollars, // supply - up to 1 hour supply * price now
// pub value_destroyed: Dollars, // supply * price then
// pub adjusted_value_destroyed: Dollars, // supply - up to 1 hour supply * price then
pub cap: Dollars,
pub profit: Dollars,
pub loss: Dollars,
pub value_created: Dollars,
pub adj_value_created: Dollars,
pub value_destroyed: Dollars,
pub adj_value_destroyed: Dollars,
}
impl RealizedState {
pub const NAN: Self = Self {
realized_cap: Dollars::NAN,
// realized_profit: Dollars::NAN,
// realized_loss: Dollars::NAN,
// value_created: Dollars::NAN,
// adjusted_value_created: Dollars::NAN,
// value_destroyed: Dollars::NAN,
// adjusted_value_destroyed: Dollars::NAN,
cap: Dollars::NAN,
profit: Dollars::NAN,
loss: Dollars::NAN,
value_created: Dollars::NAN,
adj_value_created: Dollars::NAN,
value_destroyed: Dollars::NAN,
adj_value_destroyed: Dollars::NAN,
};
pub fn increment(&mut self, supply_state: &SupplyState, price: Dollars) {
if supply_state.value.is_not_zero() {
if self.realized_cap == Dollars::NAN {
self.realized_cap = Dollars::ZERO;
// self.realized_profit = Dollars::ZERO;
// self.realized_loss = Dollars::ZERO;
// self.value_created = Dollars::ZERO;
// self.adjusted_value_created = Dollars::ZERO;
// self.value_destroyed = Dollars::ZERO;
// self.adjusted_value_destroyed = Dollars::ZERO;
}
self.realized_cap += price * Bitcoin::from(supply_state.value);
pub fn reset_single_iteration_values(&mut self) {
if self.cap != Dollars::NAN {
self.profit = Dollars::ZERO;
self.loss = Dollars::ZERO;
self.value_created = Dollars::ZERO;
self.adj_value_created = Dollars::ZERO;
self.value_destroyed = Dollars::ZERO;
self.adj_value_destroyed = Dollars::ZERO;
}
}
pub fn increment(&mut self, supply_state: &SupplyState, price: Dollars) {
if supply_state.value.is_zero() {
return;
}
if self.cap == Dollars::NAN {
self.cap = Dollars::ZERO;
self.profit = Dollars::ZERO;
self.loss = Dollars::ZERO;
self.value_created = Dollars::ZERO;
self.adj_value_created = Dollars::ZERO;
self.value_destroyed = Dollars::ZERO;
self.adj_value_destroyed = Dollars::ZERO;
}
let value = price * Bitcoin::from(supply_state.value);
self.cap += value;
}
pub fn decrement(&mut self, supply_state: &SupplyState, price: Dollars) {
self.realized_cap = self
.realized_cap
.checked_sub(price * Bitcoin::from(supply_state.value))
.unwrap();
let value = price * Bitcoin::from(supply_state.value);
self.cap = self.cap.checked_sub(value).unwrap();
}
pub fn receive(&mut self, supply_state: &SupplyState, current_price: Dollars) {
self.increment(supply_state, current_price);
}
pub fn send(
&mut self,
supply_state: &SupplyState,
current_price: Dollars,
prev_price: Dollars,
older_than_hour: bool,
) {
let current_value = current_price * Bitcoin::from(supply_state.value);
let prev_value = prev_price * Bitcoin::from(supply_state.value);
self.value_created += current_value;
self.value_destroyed += prev_value;
if older_than_hour {
self.adj_value_created += current_value;
self.adj_value_destroyed += prev_value;
}
match current_price.cmp(&prev_price) {
Ordering::Greater => {
self.profit += current_value.checked_sub(prev_value).unwrap();
}
Ordering::Less => {
self.loss += prev_value.checked_sub(current_value).unwrap();
}
Ordering::Equal => {}
}
self.decrement(supply_state, prev_price);
}
}

2
crates/brk_computer/src/stores.rs
View File

@@ -1,6 +1,6 @@
use std::path::Path;
use brk_vec::Version;
use brk_core::Version;
use fjall::TransactionalKeyspace;
const _VERSION: Version = Version::ZERO;

21
crates/brk_computer/src/vecs/blocks.rs
View File

@@ -2,12 +2,11 @@ use std::{fs, path::Path};
use brk_core::{
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize,
Timestamp, Weight,
Timestamp, Version, Weight,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, Computation, EagerVec, Version};
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, Computation, EagerVec};
use super::{
Indexes,
@@ -71,7 +70,9 @@ impl Vecs {
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
path,
@@ -79,7 +80,9 @@ impl Vecs {
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
indexes_to_block_size: ComputedVecsFromHeight::forced_import(
path,
@@ -87,7 +90,9 @@ impl Vecs {
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
height_to_vbytes: EagerVec::forced_import(
path,
@@ -101,7 +106,9 @@ impl Vecs {
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
difficultyepoch_to_timestamp: EagerVec::forced_import(
path,

4
crates/brk_computer/src/vecs/constants.rs
View File

@@ -1,9 +1,9 @@
use std::{fs, path::Path};
use brk_core::StoredU8;
use brk_core::{StoredU8, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyVec, Compressed, Computation, Version};
use brk_vec::{AnyCollectableVec, AnyVec, Compressed, Computation};
use super::{
Indexes,

4
crates/brk_computer/src/vecs/fetched.rs
View File

@@ -2,12 +2,12 @@ use std::{fs, path::Path};
use brk_core::{
Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex,
OHLCCents, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, WeekIndex, YearIndex,
OHLCCents, OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, Computation, EagerVec, Version};
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, Computation, EagerVec};
use super::{
Indexes,

109
crates/brk_computer/src/vecs/grouped/builder.rs
View File

@@ -1,11 +1,8 @@
use std::path::Path;
use brk_core::{CheckedSub, StoredUsize};
use brk_core::{CheckedSub, Result, StoredUsize, Version};
use brk_exit::Exit;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, Compressed, EagerVec, Result, StoredIndex, StoredType,
Version,
};
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, EagerVec, StoredIndex, StoredType};
use color_eyre::eyre::ContextCompat;
use crate::utils::get_percentile;
@@ -29,7 +26,7 @@ where
pub _10p: Option<Box<EagerVec<I, T>>>,
pub min: Option<Box<EagerVec<I, T>>>,
pub last: Option<Box<EagerVec<I, T>>>,
pub total: Option<Box<EagerVec<I, T>>>,
pub cumulative: Option<Box<EagerVec<I, T>>>,
}
const VERSION: Version = Version::ZERO;
@@ -141,11 +138,11 @@ where
.unwrap(),
)
}),
total: options.total.then(|| {
cumulative: options.cumulative.then(|| {
Box::new(
EagerVec::forced_import(
path,
&prefix("total"),
&prefix("cumulative"),
version + VERSION + Version::ZERO,
compressed,
)
@@ -207,20 +204,20 @@ where
source: &impl AnyIterableVec<I, T>,
exit: &Exit,
) -> Result<()> {
if self.total.is_none() {
if self.cumulative.is_none() {
return Ok(());
};
let index = self.starting_index(max_from);
let total_vec = self.total.as_mut().unwrap();
let cumulative_vec = self.cumulative.as_mut().unwrap();
let mut total = index.decremented().map_or(T::from(0_usize), |index| {
total_vec.iter().unwrap_get_inner(index)
let mut cumulative = index.decremented().map_or(T::from(0_usize), |index| {
cumulative_vec.iter().unwrap_get_inner(index)
});
source.iter_at(index).try_for_each(|(i, v)| -> Result<()> {
total = total.clone() + v.into_inner();
total_vec.forced_push_at(i, total.clone(), exit)
cumulative = cumulative.clone() + v.into_inner();
cumulative_vec.forced_push_at(i, cumulative.clone(), exit)
})?;
self.safe_flush(exit)?;
@@ -248,11 +245,11 @@ where
let mut count_indexes_iter = count_indexes.iter();
let mut source_iter = source.iter();
let total_vec = self.total.as_mut();
let cumulative_vec = self.cumulative.as_mut();
let mut total = total_vec.map(|total_vec| {
let mut cumulative = cumulative_vec.map(|cumulative_vec| {
index.decremented().map_or(T::from(0_usize), |index| {
total_vec.iter().unwrap_get_inner(index)
cumulative_vec.iter().unwrap_get_inner(index)
})
});
@@ -286,8 +283,9 @@ where
last.forced_push_at(index, v, exit)?;
}
let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
let needs_sum_or_cumulative = self.sum.is_some() || self.cumulative.is_some();
let needs_average_sum_or_cumulative =
needs_sum_or_cumulative || self.average.is_some();
let needs_sorted = self.max.is_some()
|| self._90p.is_some()
|| self._75p.is_some()
@@ -295,7 +293,7 @@ where
|| self._25p.is_some()
|| self._10p.is_some()
|| self.min.is_some();
let needs_values = needs_sorted || needs_average_sum_or_total;
let needs_values = needs_sorted || needs_average_sum_or_cumulative;
if needs_values {
source_iter.set(first_index);
@@ -356,7 +354,7 @@ where
}
}
if needs_average_sum_or_total {
if needs_average_sum_or_cumulative {
let len = values.len();
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
@@ -365,15 +363,15 @@ where
average.forced_push_at(i, avg, exit)?;
}
if needs_sum_or_total {
if needs_sum_or_cumulative {
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum.clone(), exit)?;
}
if let Some(total_vec) = self.total.as_mut() {
let t = total.as_ref().unwrap().clone() + sum;
total.replace(t.clone());
total_vec.forced_push_at(i, t, exit)?;
if let Some(cumulative_vec) = self.cumulative.as_mut() {
let t = cumulative.as_ref().unwrap().clone() + sum;
cumulative.replace(t.clone());
cumulative_vec.forced_push_at(i, t, exit)?;
}
}
}
@@ -423,9 +421,9 @@ where
let mut source_average_iter = source.average.as_ref().map(|f| f.iter());
let mut source_sum_iter = source.sum.as_ref().map(|f| f.iter());
let mut total = self.total.as_mut().map(|total_vec| {
let mut cumulative = self.cumulative.as_mut().map(|cumulative_vec| {
index.decremented().map_or(T::from(0_usize), |index| {
total_vec.iter().unwrap_get_inner(index)
cumulative_vec.iter().unwrap_get_inner(index)
})
});
@@ -457,10 +455,11 @@ where
last.forced_push_at(index, v, exit)?;
}
let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
let needs_sum_or_cumulative = self.sum.is_some() || self.cumulative.is_some();
let needs_average_sum_or_cumulative =
needs_sum_or_cumulative || self.average.is_some();
let needs_sorted = self.max.is_some() || self.min.is_some();
let needs_values = needs_sorted || needs_average_sum_or_total;
let needs_values = needs_sorted || needs_average_sum_or_cumulative;
if needs_values {
if needs_sorted {
@@ -487,7 +486,7 @@ where
}
}
if needs_average_sum_or_total {
if needs_average_sum_or_cumulative {
if let Some(average) = self.average.as_mut() {
let source_average_iter = source_average_iter.as_mut().unwrap();
source_average_iter.set(first_index);
@@ -497,14 +496,14 @@ where
.collect::<Vec<_>>();
let len = values.len();
let total = values.into_iter().fold(T::from(0), |a, b| a + b);
// TODO: Multiply by count then divide by total
let cumulative = values.into_iter().fold(T::from(0), |a, b| a + b);
// TODO: Multiply by count then divide by cumulative
// Right now it's not 100% accurate as there could be more or less elements in the lower timeframe (28 days vs 31 days in a month for example)
let avg = total / len;
let avg = cumulative / len;
average.forced_push_at(i, avg, exit)?;
}
if needs_sum_or_total {
if needs_sum_or_cumulative {
let source_sum_iter = source_sum_iter.as_mut().unwrap();
source_sum_iter.set(first_index);
let values = source_sum_iter
@@ -518,10 +517,10 @@ where
sum_vec.forced_push_at(i, sum.clone(), exit)?;
}
if let Some(total_vec) = self.total.as_mut() {
let t = total.as_ref().unwrap().clone() + sum;
total.replace(t.clone());
total_vec.forced_push_at(i, t, exit)?;
if let Some(cumulative_vec) = self.cumulative.as_mut() {
let t = cumulative.as_ref().unwrap().clone() + sum;
cumulative.replace(t.clone());
cumulative_vec.forced_push_at(i, t, exit)?;
}
}
}
@@ -581,8 +580,8 @@ where
self.last.as_ref().unwrap()
}
#[allow(unused)]
pub fn unwrap_total(&self) -> &EagerVec<I, T> {
self.total.as_ref().unwrap()
pub fn unwrap_cumulative(&self) -> &EagerVec<I, T> {
self.cumulative.as_ref().unwrap()
}
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
@@ -609,8 +608,8 @@ where
if let Some(sum) = self.sum.as_ref() {
v.push(sum.as_ref());
}
if let Some(total) = self.total.as_ref() {
v.push(total.as_ref());
if let Some(cumulative) = self.cumulative.as_ref() {
v.push(cumulative.as_ref());
}
if let Some(_90p) = self._90p.as_ref() {
v.push(_90p.as_ref());
@@ -650,8 +649,8 @@ where
if let Some(sum) = self.sum.as_mut() {
sum.safe_flush(exit)?;
}
if let Some(total) = self.total.as_mut() {
total.safe_flush(exit)?;
if let Some(cumulative) = self.cumulative.as_mut() {
cumulative.safe_flush(exit)?;
}
if let Some(_90p) = self._90p.as_mut() {
_90p.safe_flush(exit)?;
@@ -691,8 +690,8 @@ where
if let Some(sum) = self.sum.as_mut() {
sum.validate_computed_version_or_reset_file(Version::ZERO + version)?;
}
if let Some(total) = self.total.as_mut() {
total.validate_computed_version_or_reset_file(Version::ZERO + version)?;
if let Some(cumulative) = self.cumulative.as_mut() {
cumulative.validate_computed_version_or_reset_file(Version::ZERO + version)?;
}
if let Some(_90p) = self._90p.as_mut() {
_90p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
@@ -724,7 +723,7 @@ pub struct StorableVecGeneatorOptions {
min: bool,
first: bool,
last: bool,
total: bool,
cumulative: bool,
}
impl StorableVecGeneatorOptions {
@@ -788,8 +787,8 @@ impl StorableVecGeneatorOptions {
self
}
pub fn add_total(mut self) -> Self {
self.total = true;
pub fn add_cumulative(mut self) -> Self {
self.cumulative = true;
self
}
@@ -848,8 +847,8 @@ impl StorableVecGeneatorOptions {
}
#[allow(unused)]
pub fn rm_total(mut self) -> Self {
self.total = false;
pub fn rm_cumulative(mut self) -> Self {
self.cumulative = false;
self
}
@@ -890,7 +889,7 @@ impl StorableVecGeneatorOptions {
self.min,
self.first,
self.last,
self.total,
self.cumulative,
]
.iter()
.filter(|b| **b)
@@ -900,7 +899,7 @@ impl StorableVecGeneatorOptions {
pub fn copy_self_extra(&self) -> Self {
Self {
total: self.total,
cumulative: self.cumulative,
..Self::default()
}
}

6
crates/brk_computer/src/vecs/grouped/from_dateindex.rs
View File

@@ -1,9 +1,11 @@
use std::path::Path;
use brk_core::{DateIndex, DecadeIndex, MonthIndex, QuarterIndex, WeekIndex, YearIndex};
use brk_core::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, EagerVec, Result, Version};
use brk_vec::{AnyCollectableVec, Compressed, EagerVec};
use crate::vecs::{Indexes, indexes};

5
crates/brk_computer/src/vecs/grouped/from_height.rs
View File

@@ -1,11 +1,12 @@
use std::path::Path;
use brk_core::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, WeekIndex, YearIndex,
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result, Version,
WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, EagerVec, Result, Version};
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, EagerVec};
use crate::vecs::{Indexes, indexes};

4
crates/brk_computer/src/vecs/grouped/from_height_strict.rs
View File

@@ -1,9 +1,9 @@
use std::path::Path;
use brk_core::{DifficultyEpoch, Height};
use brk_core::{DifficultyEpoch, Height, Result, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, EagerVec, Result, Version};
use brk_vec::{AnyCollectableVec, Compressed, EagerVec};
use crate::vecs::{Indexes, indexes};

17
crates/brk_computer/src/vecs/grouped/from_txindex.rs
View File

@@ -2,13 +2,12 @@ use std::path::Path;
use brk_core::{
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
Sats, TxIndex, WeekIndex, YearIndex,
Result, Sats, TxIndex, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyVec, CollectableVec, Compressed, EagerVec, Result, StoredIndex,
VecIterator, Version,
AnyCollectableVec, AnyVec, CollectableVec, Compressed, EagerVec, StoredIndex, VecIterator,
};
use crate::vecs::{Indexes, fetched, indexes};
@@ -425,12 +424,12 @@ impl ComputedVecsFromTxindex<Bitcoin> {
exit,
)?;
}
if let Some(total) = self.height.total.as_mut() {
total.forced_push_at(
if let Some(cumulative) = self.height.cumulative.as_mut() {
cumulative.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_total()
.unwrap_cumulative()
.into_iter()
.unwrap_get_inner(height),
),
@@ -608,13 +607,13 @@ impl ComputedVecsFromTxindex<Dollars> {
exit,
)?;
}
if let Some(total) = self.height.total.as_mut() {
total.forced_push_at(
if let Some(cumulative) = self.height.cumulative.as_mut() {
cumulative.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_total()
.unwrap_cumulative()
.into_iter()
.unwrap_get_inner(height),
exit,

6
crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs
View File

@@ -1,11 +1,11 @@
use std::{f32, path::Path};
use brk_core::{Date, DateIndex, Dollars, StoredF32};
use brk_core::{Date, DateIndex, Dollars, Result, StoredF32, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Compressed, EagerVec, Result,
StoredIndex, VecIterator, Version,
AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Compressed, EagerVec, StoredIndex,
VecIterator,
};
use crate::{

6
crates/brk_computer/src/vecs/grouped/value_from_height.rs
View File

@@ -1,11 +1,9 @@
use std::path::Path;
use brk_core::{Bitcoin, Dollars, Height, Sats};
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, CollectableVec, Compressed, EagerVec, Result, StoredVec, Version,
};
use brk_vec::{AnyCollectableVec, CollectableVec, Compressed, EagerVec, StoredVec};
use crate::vecs::{Indexes, fetched, indexes};

4
crates/brk_computer/src/vecs/grouped/value_from_txindex.rs
View File

@@ -1,11 +1,11 @@
use std::path::Path;
use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex};
use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, CollectableVec, Compressed,
Computation, ComputedVecFrom3, LazyVecFrom1, StoredIndex, StoredVec, Version,
Computation, ComputedVecFrom3, LazyVecFrom1, StoredIndex, StoredVec,
};
use crate::vecs::{Indexes, fetched, indexes};

4
crates/brk_computer/src/vecs/indexes.rs
View File

@@ -5,13 +5,13 @@ use brk_core::{
InputIndex, MonthIndex, OpReturnIndex, OutputIndex, P2ABytes, P2AIndex, P2MSIndex, P2PK33Bytes,
P2PK33Index, P2PK65Bytes, P2PK65Index, P2PKHBytes, P2PKHIndex, P2SHBytes, P2SHIndex, P2TRBytes,
P2TRIndex, P2WPKHBytes, P2WPKHIndex, P2WSHBytes, P2WSHIndex, QuarterIndex, Sats, StoredUsize,
Timestamp, TxIndex, Txid, UnknownOutputIndex, WeekIndex, YearIndex,
Timestamp, TxIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, CloneableAnyIterableVec, Compressed, Computation, ComputedVec,
ComputedVecFrom1, ComputedVecFrom2, EagerVec, StoredIndex, VecIterator, Version,
ComputedVecFrom1, ComputedVecFrom2, EagerVec, StoredIndex, VecIterator,
};
const VERSION: Version = Version::ZERO;

10
crates/brk_computer/src/vecs/market.rs
View File

@@ -1,9 +1,9 @@
use std::{fs, path::Path, thread};
use brk_core::{Date, DateIndex, Dollars, Sats, StoredF32, StoredUsize};
use brk_core::{Date, DateIndex, Dollars, Sats, StoredF32, StoredUsize, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, Computation, StoredIndex, VecIterator, Version};
use brk_vec::{AnyCollectableVec, Compressed, Computation, StoredIndex, VecIterator};
use super::{
Indexes, fetched,
@@ -1097,17 +1097,17 @@ impl Vecs {
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
let mut total_subsidy_in_btc = transactions
let mut cumulative_subsidy_in_btc = transactions
.indexes_to_subsidy
.bitcoin
.dateindex
.unwrap_total()
.unwrap_cumulative()
.into_iter();
v.compute_transform(
starting_indexes.dateindex,
&fetched.timeindexes_to_close.dateindex,
|(i, close, ..)| {
let supply = total_subsidy_in_btc.unwrap_get_inner(i);
let supply = cumulative_subsidy_in_btc.unwrap_get_inner(i);
(i, *close * supply)
},
exit,

4
crates/brk_computer/src/vecs/mining.rs
View File

@@ -1,9 +1,9 @@
use std::{fs, path::Path};
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64};
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, Computation, VecIterator, Version};
use brk_vec::{AnyCollectableVec, Compressed, Computation, VecIterator};
use super::{
Indexes,

3
crates/brk_computer/src/vecs/mod.rs
View File

@@ -1,9 +1,10 @@
use std::{fs, path::Path};
use brk_core::Version;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, Computation, Version};
use brk_vec::{AnyCollectableVec, Compressed, Computation};
pub mod blocks;
pub mod constants;

66
crates/brk_computer/src/vecs/transactions.rs
View File

@@ -2,15 +2,13 @@ use std::{fs, path::Path};
use brk_core::{
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32,
StoredUsize, TxIndex, TxVersion, Weight,
StoredUsize, TxIndex, TxVersion, Version, Weight,
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::bitcoin;
use brk_vec::{
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Compressed, Computation,
ComputedVec, ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, StoredIndex, VecIterator,
Version,
};
use super::{
@@ -244,7 +242,7 @@ impl Vecs {
// StorableVecGeneatorOptions::default()
// .add_average()
// .add_sum()
// .add_total(),
// .add_cumulative(),
// )?;
let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3(
@@ -293,7 +291,7 @@ impl Vecs {
// StorableVecGeneatorOptions::default()
// .add_average()
// .add_sum()
// .add_total(),
// .add_cumulative(),
// )?;
let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2(
@@ -351,7 +349,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_input_count: ComputedVecsFromTxindex::forced_import(
path,
@@ -364,7 +362,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_output_count: ComputedVecsFromTxindex::forced_import(
path,
@@ -377,7 +375,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
path,
@@ -385,7 +383,9 @@ impl Vecs {
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
indexes_to_tx_v2: ComputedVecsFromHeight::forced_import(
path,
@@ -393,7 +393,9 @@ impl Vecs {
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
indexes_to_tx_v3: ComputedVecsFromHeight::forced_import(
path,
@@ -401,7 +403,9 @@ impl Vecs {
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)?,
indexes_to_fee: ComputedValueVecsFromTxindex::forced_import(
path,
@@ -414,7 +418,7 @@ impl Vecs {
fetched,
StorableVecGeneatorOptions::default()
.add_sum()
.add_total()
.add_cumulative()
.add_percentiles()
.add_minmax()
.add_average(),
@@ -461,7 +465,7 @@ impl Vecs {
StorableVecGeneatorOptions::default()
.add_percentiles()
.add_sum()
.add_total()
.add_cumulative()
.add_minmax()
.add_average(),
compute_dollars,
@@ -474,7 +478,7 @@ impl Vecs {
compressed,
StorableVecGeneatorOptions::default()
.add_sum()
.add_total()
.add_cumulative()
.add_percentiles()
.add_minmax()
.add_average(),
@@ -486,7 +490,9 @@ impl Vecs {
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum().add_total(),
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
compute_dollars,
)?,
indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
@@ -500,7 +506,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import(
path,
@@ -513,7 +519,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import(
path,
@@ -526,7 +532,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import(
path,
@@ -539,7 +545,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import(
path,
@@ -552,7 +558,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import(
path,
@@ -565,7 +571,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import(
path,
@@ -578,7 +584,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import(
path,
@@ -591,7 +597,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import(
path,
@@ -604,7 +610,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
path,
@@ -617,7 +623,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
path,
@@ -630,7 +636,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
path,
@@ -643,7 +649,7 @@ impl Vecs {
.add_minmax()
.add_percentiles()
.add_sum()
.add_total(),
.add_cumulative(),
)?,
indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import(
path,
@@ -1088,16 +1094,16 @@ impl Vecs {
let mut input_count_iter = self
.indexes_to_input_count
.height
.unwrap_total()
.unwrap_cumulative()
.into_iter();
let mut opreturn_count_iter = self
.indexes_to_opreturn_count
.height_extra
.unwrap_total()
.unwrap_cumulative()
.into_iter();
v.compute_transform(
starting_indexes.height,
self.indexes_to_output_count.height.unwrap_total(),
self.indexes_to_output_count.height.unwrap_cumulative(),
|(h, output_count, ..)| {
let input_count = input_count_iter.unwrap_get_inner(h);
let opreturn_count = opreturn_count_iter.unwrap_get_inner(h);

473
crates/brk_computer/src/vecs/utxos/cohort.rs
View File

@@ -1,11 +1,9 @@
use std::{fs, path::Path};
use brk_core::{CheckedSub, Dollars, Height, Sats, StoredUsize};
use brk_core::{CheckedSub, Dollars, Height, Result, Sats, StoredUsize, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyVec, Compressed, Computation, EagerVec, Result, VecIterator, Version,
};
use brk_vec::{AnyCollectableVec, AnyVec, Compressed, Computation, EagerVec, VecIterator};
use crate::{
states::{CohortState, RealizedState},
@@ -32,6 +30,19 @@ pub struct Vecs {
pub height_to_utxo_count: EagerVec<Height, StoredUsize>,
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
pub height_to_realized_profit: Option<EagerVec<Height, Dollars>>,
pub indexes_to_realized_profit: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_realized_loss: Option<EagerVec<Height, Dollars>>,
pub indexes_to_realized_loss: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_value_created: Option<EagerVec<Height, Dollars>>,
pub indexes_to_value_created: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_adjusted_value_created: Option<EagerVec<Height, Dollars>>,
pub indexes_to_adjusted_value_created: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_value_destroyed: Option<EagerVec<Height, Dollars>>,
pub indexes_to_value_destroyed: Option<ComputedVecsFromHeight<Dollars>>,
pub height_to_adjusted_value_destroyed: Option<EagerVec<Height, Dollars>>,
pub indexes_to_adjusted_value_destroyed: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_realized_price: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_realized_price_extra: Option<ComputedRatioVecsFromDateIndex>,
}
@@ -134,6 +145,130 @@ impl Vecs {
)
.unwrap()
}),
height_to_realized_profit: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("realized_profit"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_realized_profit: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
&suffix("realized_profit"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)
.unwrap()
}),
height_to_realized_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("realized_loss"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_realized_loss: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
&suffix("realized_loss"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default()
.add_sum()
.add_cumulative(),
)
.unwrap()
}),
height_to_value_created: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("value_created"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_value_created: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
&suffix("value_created"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum(),
)
.unwrap()
}),
height_to_adjusted_value_created: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("adjusted_value_created"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_adjusted_value_created: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
&suffix("adjusted_value_created"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum(),
)
.unwrap()
}),
height_to_value_destroyed: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("value_destroyed"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_value_destroyed: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
&suffix("value_destroyed"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum(),
)
.unwrap()
}),
height_to_adjusted_value_destroyed: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("adjusted_value_destroyed"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_adjusted_value_destroyed: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
path,
&suffix("adjusted_value_destroyed"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_sum(),
)
.unwrap()
}),
})
}
@@ -144,6 +279,24 @@ impl Vecs {
self.height_to_realized_cap
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_realized_profit
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_realized_loss
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_value_created
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_adjusted_value_created
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_value_destroyed
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_adjusted_value_destroyed
.as_ref()
.map_or(usize::MAX, |v| v.len()),
]
.into_iter()
.map(Height::from)
@@ -169,7 +322,7 @@ impl Vecs {
.unwrap_get_inner(prev_height);
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
self.state.realized.as_mut().unwrap().realized_cap = height_to_realized_cap
self.state.realized.as_mut().unwrap().cap = height_to_realized_cap
.into_iter()
.unwrap_get_inner(prev_height);
}
@@ -191,6 +344,73 @@ impl Vecs {
height_to_realized_cap.validate_computed_version_or_reset_file(
base_version + height_to_realized_cap.inner_version(),
)?;
let height_to_realized_profit_inner_version = self
.height_to_realized_profit
.as_ref()
.unwrap()
.inner_version();
self.height_to_realized_profit
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_realized_profit_inner_version,
)?;
let height_to_realized_loss_inner_version = self
.height_to_realized_loss
.as_ref()
.unwrap()
.inner_version();
self.height_to_realized_loss
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_realized_loss_inner_version,
)?;
let height_to_value_created_inner_version = self
.height_to_value_created
.as_ref()
.unwrap()
.inner_version();
self.height_to_value_created
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_value_created_inner_version,
)?;
let height_to_adjusted_value_created_inner_version = self
.height_to_adjusted_value_created
.as_ref()
.unwrap()
.inner_version();
self.height_to_adjusted_value_created
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_adjusted_value_created_inner_version,
)?;
let height_to_value_destroyed_inner_version = self
.height_to_value_destroyed
.as_ref()
.unwrap()
.inner_version();
self.height_to_value_destroyed
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_value_destroyed_inner_version,
)?;
let height_to_adjusted_value_destroyed_inner_version = self
.height_to_adjusted_value_destroyed
.as_ref()
.unwrap()
.inner_version();
self.height_to_adjusted_value_destroyed
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_adjusted_value_destroyed_inner_version,
)?;
}
Ok(())
@@ -211,18 +431,37 @@ impl Vecs {
)?;
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
height_to_realized_cap.forced_push_at(
height,
self.state
.realized
.as_ref()
.unwrap_or_else(|| {
dbg!(&self.state);
panic!();
})
.realized_cap,
exit,
)?;
let realized = self.state.realized.as_ref().unwrap_or_else(|| {
dbg!(&self.state);
panic!();
});
height_to_realized_cap.forced_push_at(height, realized.cap, exit)?;
self.height_to_realized_profit
.as_mut()
.unwrap()
.forced_push_at(height, realized.profit, exit)?;
self.height_to_realized_loss
.as_mut()
.unwrap()
.forced_push_at(height, realized.loss, exit)?;
self.height_to_value_created
.as_mut()
.unwrap()
.forced_push_at(height, realized.value_created, exit)?;
self.height_to_adjusted_value_created
.as_mut()
.unwrap()
.forced_push_at(height, realized.adj_value_created, exit)?;
self.height_to_value_destroyed
.as_mut()
.unwrap()
.forced_push_at(height, realized.value_destroyed, exit)?;
self.height_to_adjusted_value_destroyed
.as_mut()
.unwrap()
.forced_push_at(height, realized.adj_value_destroyed, exit)?;
}
Ok(())
}
@@ -234,6 +473,30 @@ impl Vecs {
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
height_to_realized_cap.safe_flush(exit)?;
self.height_to_realized_profit
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_realized_loss
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_value_created
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_adjusted_value_created
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_value_destroyed
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_adjusted_value_destroyed
.as_mut()
.unwrap()
.safe_flush(exit)?;
}
Ok(())
@@ -270,41 +533,102 @@ impl Vecs {
exit,
Some(self.height_to_realized_cap.as_ref().unwrap()),
)?;
}
if let Some(indexes_to_realized_price) = self.indexes_to_realized_price.as_mut() {
indexes_to_realized_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_divide(
starting_indexes.height,
self.height_to_realized_cap.as_ref().unwrap(),
&**self.indexes_to_supply.bitcoin.height.as_ref().unwrap(),
exit,
)
},
)?;
}
self.indexes_to_realized_price
.as_mut()
.unwrap()
.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
vec.compute_divide(
starting_indexes.height,
self.height_to_realized_cap.as_ref().unwrap(),
&**self.indexes_to_supply.bitcoin.height.as_ref().unwrap(),
exit,
)
},
)?;
if let Some(indexes_to_realized_price_extra) = self.indexes_to_realized_price_extra.as_mut()
{
indexes_to_realized_price_extra.compute_rest(
indexer,
indexes,
fetched.as_ref().unwrap(),
starting_indexes,
exit,
Some(
self.indexes_to_realized_price
.as_ref()
.unwrap()
.dateindex
.unwrap_last(),
),
)?;
self.indexes_to_realized_price_extra
.as_mut()
.unwrap()
.compute_rest(
indexer,
indexes,
fetched.as_ref().unwrap(),
starting_indexes,
exit,
Some(
self.indexes_to_realized_price
.as_ref()
.unwrap()
.dateindex
.unwrap_last(),
),
)?;
self.indexes_to_realized_profit
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.height_to_realized_profit.as_ref().unwrap()),
)?;
self.indexes_to_realized_loss
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.height_to_realized_loss.as_ref().unwrap()),
)?;
self.indexes_to_value_created
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.height_to_value_created.as_ref().unwrap()),
)?;
self.indexes_to_adjusted_value_created
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.height_to_adjusted_value_created.as_ref().unwrap()),
)?;
self.indexes_to_value_destroyed
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.height_to_value_destroyed.as_ref().unwrap()),
)?;
self.indexes_to_adjusted_value_destroyed
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.height_to_adjusted_value_destroyed.as_ref().unwrap()),
)?;
}
Ok(())
@@ -330,6 +654,42 @@ impl Vecs {
self.indexes_to_realized_price_extra
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.height_to_realized_profit
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.indexes_to_realized_profit
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.height_to_realized_loss
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.indexes_to_realized_loss
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.height_to_value_created
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.indexes_to_value_created
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.height_to_adjusted_value_created
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.indexes_to_adjusted_value_created
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.height_to_value_destroyed
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.indexes_to_value_destroyed
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.height_to_adjusted_value_destroyed
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.indexes_to_adjusted_value_destroyed
.as_ref()
.map_or(vec![], |v| v.vecs()),
]
.into_iter()
.flatten()
@@ -350,6 +710,19 @@ impl Clone for Vecs {
height_to_utxo_count: self.height_to_utxo_count.clone(),
indexes_to_utxo_count: self.indexes_to_utxo_count.clone(),
height_to_realized_profit: self.height_to_realized_profit.clone(),
indexes_to_realized_profit: self.indexes_to_realized_profit.clone(),
height_to_realized_loss: self.height_to_realized_loss.clone(),
indexes_to_realized_loss: self.indexes_to_realized_loss.clone(),
height_to_value_created: self.height_to_value_created.clone(),
indexes_to_value_created: self.indexes_to_value_created.clone(),
height_to_adjusted_value_created: self.height_to_adjusted_value_created.clone(),
indexes_to_adjusted_value_created: self.indexes_to_adjusted_value_created.clone(),
height_to_value_destroyed: self.height_to_value_destroyed.clone(),
indexes_to_value_destroyed: self.indexes_to_value_destroyed.clone(),
height_to_adjusted_value_destroyed: self.height_to_adjusted_value_destroyed.clone(),
indexes_to_adjusted_value_destroyed: self.indexes_to_adjusted_value_destroyed.clone(),
indexes_to_realized_price: self.indexes_to_realized_price.clone(),
indexes_to_realized_price_extra: self.indexes_to_realized_price_extra.clone(),
}

101
crates/brk_computer/src/vecs/utxos/mod.rs
View File

@@ -1,11 +1,11 @@
use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread};
use brk_core::{Height, InputIndex, OutputIndex, OutputType, Sats};
use brk_core::{Height, InputIndex, OutputIndex, OutputType, Sats, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyVec, BaseVecIterator, CollectableVec, Compressed, Computation, EagerVec,
GenericStoredVec, StoredIndex, StoredVec, UnsafeSlice, VecIterator, Version,
GenericStoredVec, StoredIndex, StoredVec, UnsafeSlice, VecIterator,
};
use log::info;
use rayon::prelude::*;
@@ -24,6 +24,7 @@ use super::{
pub mod cohort;
const VERSION: Version = Version::ZERO;
const BYSIZE_VERSION: Version = Version::ONE;
#[derive(Clone)]
pub struct Vecs {
@@ -32,8 +33,8 @@ pub struct Vecs {
// cointime,...
pub height_to_unspendable_supply: EagerVec<Height, Sats>,
pub indexes_to_unspendable_supply: ComputedValueVecsFromHeight,
// pub height_to_opreturn_supply: EagerVec<Height, Sats>,
// pub indexes_to_opreturn_supply: ComputedValueVecsFromHeight,
pub height_to_opreturn_supply: EagerVec<Height, Sats>,
pub indexes_to_opreturn_supply: ComputedValueVecsFromHeight,
utxos_vecs: Outputs<(OutputFilter, cohort::Vecs)>,
}
@@ -74,6 +75,21 @@ impl Vecs {
StorableVecGeneatorOptions::default().add_last(),
compute_dollars,
)?,
height_to_opreturn_supply: EagerVec::forced_import(
path,
"opreturn_supply",
version + VERSION + Version::ZERO,
compressed,
)?,
indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import(
path,
"opreturn_supply",
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
compute_dollars,
)?,
utxos_vecs: {
Outputs::<(OutputFilter, cohort::Vecs)>::from(Outputs {
all: cohort::Vecs::forced_import(
@@ -540,7 +556,7 @@ impl Vecs {
Some("0sat"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_1sat_to_10sats: cohort::Vecs::forced_import(
@@ -548,7 +564,7 @@ impl Vecs {
Some("from_1sat_to_10sats"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_10sats_to_100sats: cohort::Vecs::forced_import(
@@ -556,7 +572,7 @@ impl Vecs {
Some("from_10sats_to_100sats"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_100sats_to_1_000sats: cohort::Vecs::forced_import(
@@ -564,7 +580,7 @@ impl Vecs {
Some("from_100sats_to_1_000sats"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_1_000sats_to_10_000sats: cohort::Vecs::forced_import(
@@ -572,7 +588,7 @@ impl Vecs {
Some("from_1_000sats_to_10_000sats"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_10_000sats_to_100_000sats: cohort::Vecs::forced_import(
@@ -580,7 +596,7 @@ impl Vecs {
Some("from_10_000sats_to_100_000sats"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_100_000sats_to_1_000_000sats: cohort::Vecs::forced_import(
@@ -588,7 +604,7 @@ impl Vecs {
Some("from_100_000sats_to_1_000_000sats"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_1_000_000sats_to_10_000_000sats: cohort::Vecs::forced_import(
@@ -596,7 +612,7 @@ impl Vecs {
Some("from_1_000_000sats_to_10_000_000sats"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_10_000_000sats_to_1btc: cohort::Vecs::forced_import(
@@ -604,7 +620,7 @@ impl Vecs {
Some("from_10_000_000sats_to_1btc"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_1btc_to_10btc: cohort::Vecs::forced_import(
@@ -612,7 +628,7 @@ impl Vecs {
Some("from_1btc_to_10btc"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_10btc_to_100btc: cohort::Vecs::forced_import(
@@ -620,7 +636,7 @@ impl Vecs {
Some("from_10btc_to_100btc"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_100btc_to_1_000btc: cohort::Vecs::forced_import(
@@ -628,7 +644,7 @@ impl Vecs {
Some("from_100btc_to_1_000btc"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_1_000btc_to_10_000btc: cohort::Vecs::forced_import(
@@ -636,7 +652,7 @@ impl Vecs {
Some("from_1_000btc_to_10_000btc"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_10_000btc_to_100_000btc: cohort::Vecs::forced_import(
@@ -644,7 +660,7 @@ impl Vecs {
Some("from_10_000btc_to_100_000btc"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
from_100_000btc: cohort::Vecs::forced_import(
@@ -652,7 +668,7 @@ impl Vecs {
Some("from_100_000btc"),
_computation,
compressed,
version + VERSION + Version::ZERO,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
)?,
},
@@ -790,9 +806,9 @@ impl Vecs {
version + VERSION + Version::ZERO,
fetched,
)?,
// op_return: cohort::Vecs::forced_import(
// opreturn: cohort::Vecs::forced_import(
// path,
// Some("op_return"),
// Some("opreturn"),
// _computation,
// compressed,
// VERSION + Version::ZERO,
@@ -925,6 +941,10 @@ impl Vecs {
.validate_computed_version_or_reset_file(
base_version + self.height_to_unspendable_supply.inner_version(),
)?;
self.height_to_opreturn_supply
.validate_computed_version_or_reset_file(
base_version + self.height_to_opreturn_supply.inner_version(),
)?;
let mut chain_state: Vec<BlockState>;
let mut chain_state_starting_height = Height::from(self.chain_state.len());
@@ -972,7 +992,8 @@ impl Vecs {
let starting_height = starting_indexes
.height
.min(stateful_starting_height)
.min(Height::from(self.height_to_unspendable_supply.len()));
.min(Height::from(self.height_to_unspendable_supply.len()))
.min(Height::from(self.height_to_opreturn_supply.len()));
// ---
// INIT
@@ -990,12 +1011,25 @@ impl Vecs {
Sats::ZERO
};
let mut opreturn_supply = if let Some(prev_height) = starting_height.decremented() {
self.height_to_opreturn_supply
.into_iter()
.unwrap_get_inner(prev_height)
} else {
Sats::ZERO
};
let mut height = Height::ZERO;
(starting_height.unwrap_to_usize()..height_to_first_outputindex_iter.len())
.map(Height::from)
.try_for_each(|_height| -> color_eyre::Result<()> {
height = _height;
self.utxos_vecs
.as_mut_vec()
.iter_mut()
.for_each(|(_, v)| v.state.reset_single_iteration_values());
info!("Processing utxo set at {height}...");
let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height);
@@ -1130,6 +1164,8 @@ impl Vecs {
.sum::<Sats>()
+ height_to_unclaimed_rewards_iter.unwrap_get_inner(height);
opreturn_supply += received.by_type.unspendable.opreturn.value;
if height == Height::new(0) {
received = Transacted::default();
unspendable_supply += Sats::FIFTY_BTC;
@@ -1183,6 +1219,9 @@ impl Vecs {
exit,
)?;
self.height_to_opreturn_supply
.forced_push_at(height, opreturn_supply, exit)?;
Ok(())
})?;
@@ -1199,6 +1238,10 @@ impl Vecs {
.par_iter_mut()
.try_for_each(|(_, v)| v.safe_flush_height_vecs(exit))?;
self.height_to_unspendable_supply.safe_flush(exit)?;
flat_vecs_
.par_iter_mut()
.try_for_each(|(_, v)| v.safe_flush_height_vecs(exit))?;
self.height_to_opreturn_supply.safe_flush(exit)?;
info!("Computing rest...");
@@ -1214,6 +1257,14 @@ impl Vecs {
exit,
Some(&self.height_to_unspendable_supply),
)?;
self.indexes_to_opreturn_supply.compute_rest(
indexer,
indexes,
fetched,
starting_indexes,
exit,
Some(&self.height_to_opreturn_supply),
)?;
info!("Chain state...");
@@ -1239,7 +1290,11 @@ impl Vecs {
.flat_map(|v| v.vecs())
.collect::<Vec<_>>(),
self.indexes_to_unspendable_supply.vecs(),
vec![&self.height_to_unspendable_supply],
self.indexes_to_opreturn_supply.vecs(),
vec![
&self.height_to_unspendable_supply,
&self.height_to_opreturn_supply,
],
]
.into_iter()
.flatten()

3
crates/brk_core/Cargo.toml
View File

@@ -16,8 +16,9 @@ log = { workspace = true }
rapidhash = "1.4.0"
rlimit = "0.10.2"
serde = { workspace = true }
serde_derive = { workspace = true }
serde_bytes = "0.11.17"
serde_derive = { workspace = true }
serde_json = { workspace = true }
zerocopy = { workspace = true }
zerocopy-derive = { workspace = true }

17
crates/brk_vec/src/enums/error.rs → crates/brk_core/src/enums/error.rs
View File

@@ -26,6 +26,10 @@ pub enum Error {
DifferentCompressionMode,
SystemTimeError,
ToSerdeJsonValueError(serde_json::Error),
Jiff(jiff::Error),
WrongLength,
WrongAddressType,
UnindexableDate,
}
impl From<SystemTimeError> for Error {
@@ -40,6 +44,12 @@ impl From<io::Error> for Error {
}
}
impl From<jiff::Error> for Error {
fn from(value: jiff::Error) -> Self {
Self::Jiff(value)
}
}
impl<A, B, C> From<zerocopy::error::ConvertError<A, B, C>> for Error {
fn from(_: zerocopy::error::ConvertError<A, B, C>) -> Self {
Self::ZeroCopyError
@@ -87,6 +97,13 @@ impl fmt::Display for Error {
Error::DifferentCompressionMode => write!(f, "Different compression mode chosen"),
Error::EmptyVec => write!(f, "The Vec is empty, maybe wait for a bit"),
Error::ToSerdeJsonValueError(error) => Debug::fmt(&error, f),
Error::Jiff(error) => Debug::fmt(&error, f),
Error::WrongLength => write!(f, "Wrong length"),
Error::WrongAddressType => write!(f, "Wrong address type"),
Error::UnindexableDate => write!(
f,
"Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"
),
}
}
}

0
crates/brk_vec/src/enums/mod.rs → crates/brk_core/src/enums/mod.rs
View File

0
crates/brk_vec/src/enums/value.rs → crates/brk_core/src/enums/value.rs
View File

49
crates/brk_core/src/error.rs
View File

@@ -1,49 +0,0 @@
use std::{
fmt::{self, Debug},
io,
};
pub type Result<T, E = Error> = std::result::Result<T, E>;
#[derive(Debug)]
pub enum Error {
IO(io::Error),
Jiff(jiff::Error),
ZeroCopyError,
WrongLength,
WrongAddressType,
UnindexableDate,
}
impl From<io::Error> for Error {
fn from(value: io::Error) -> Self {
Self::IO(value)
}
}
impl From<jiff::Error> for Error {
fn from(value: jiff::Error) -> Self {
Self::Jiff(value)
}
}
impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
fn from(_: zerocopy::error::SizeError<A, B>) -> Self {
Self::ZeroCopyError
}
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::IO(error) => Debug::fmt(&error, f),
Error::Jiff(error) => Debug::fmt(&error, f),
Error::ZeroCopyError => write!(f, "Zero copy convert error"),
Error::WrongLength => write!(f, "Wrong length"),
Error::WrongAddressType => write!(f, "Wrong address type"),
Error::UnindexableDate => write!(f, "Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"),
}
}
}
impl std::error::Error for Error {}

4
crates/brk_core/src/lib.rs
View File

@@ -1,9 +1,9 @@
#![doc = include_str!("../README.md")]
mod error;
mod enums;
mod structs;
mod utils;
pub use error::*;
pub use enums::*;
pub use structs::*;
pub use utils::*;

2
crates/brk_core/src/structs/mod.rs
View File

@@ -36,6 +36,7 @@ mod txidprefix;
mod txindex;
mod txversion;
mod unit;
mod version;
mod vin;
mod vout;
mod weekindex;
@@ -80,6 +81,7 @@ pub use txidprefix::*;
pub use txindex::*;
pub use txversion::*;
pub use unit::*;
pub use version::*;
pub use vin::*;
pub use vout::*;
pub use weekindex::*;

2
crates/brk_vec/src/structs/version.rs → crates/brk_core/src/structs/version.rs
View File

@@ -58,7 +58,7 @@ impl From<u64> for Version {
impl TryFrom<&Path> for Version {
type Error = Error;
fn try_from(value: &Path) -> Result<Self, Self::Error> {
let mut buf = [0; 4];
let mut buf = [0; 8];
fs::read(value)?.as_slice().read_exact(&mut buf)?;
Ok(*(Self::ref_from_bytes(&buf)?))
}

3
crates/brk_indexer/Cargo.toml
View File

@@ -11,8 +11,9 @@ bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_core = { workspace = true }
brk_exit = { workspace = true }
brk_parser = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_store = { workspace = true }
brk_vec = { workspace = true }
byteview = { workspace = true }
color-eyre = { workspace = true }

6
crates/brk_indexer/examples/main.rs
View File

@@ -3,7 +3,7 @@ use std::{path::Path, time::Instant};
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_parser::{Parser, rpc};
use brk_parser::Parser;
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
@@ -14,9 +14,9 @@ fn main() -> color_eyre::Result<()> {
let bitcoin_dir = default_bitcoin_path();
let rpc = Box::leak(Box::new(rpc::Client::new(
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();

4
crates/brk_indexer/src/indexes.rs
View File

@@ -2,7 +2,7 @@ use bitcoincore_rpc::Client;
use brk_core::{
BlockHash, CheckedSub, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
OutputType, OutputTypeIndex, P2AIndex, P2MSIndex, P2PK33Index, P2PK65Index, P2PKHIndex,
P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, TxIndex, UnknownOutputIndex,
P2SHIndex, P2TRIndex, P2WPKHIndex, P2WSHIndex, Result, TxIndex, UnknownOutputIndex,
};
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
use brk_vec::{AnyIterableVec, AnyVec, IndexedVec, StoredIndex, StoredType};
@@ -48,7 +48,7 @@ impl Indexes {
}
}
pub fn push_if_needed(&self, vecs: &mut Vecs) -> brk_vec::Result<()> {
pub fn push_if_needed(&self, vecs: &mut Vecs) -> Result<()> {
let height = self.height;
vecs.height_to_first_txindex
.push_if_needed(height, self.txindex)?;

9
crates/brk_indexer/src/lib.rs
View File

@@ -12,13 +12,14 @@ use std::{
use brk_core::{
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
OutputType, OutputTypeIndex, Sats, Timestamp, TxIndex, Txid, TxidPrefix, Vin, Vout, setrlimit,
OutputType, OutputTypeIndex, Sats, Timestamp, TxIndex, Txid, TxidPrefix, Version, Vin, Vout,
setrlimit,
};
pub use brk_parser::*;
use bitcoin::{Transaction, TxIn, TxOut};
use brk_exit::Exit;
use brk_vec::{AnyVec, Compressed, VecIterator, Version};
use brk_parser::Parser;
use brk_vec::{AnyVec, Compressed, VecIterator};
use color_eyre::eyre::{ContextCompat, eyre};
use fjall::TransactionalKeyspace;
use log::{error, info};
@@ -82,7 +83,7 @@ impl Indexer {
pub fn index(
&mut self,
parser: &Parser,
rpc: &'static rpc::Client,
rpc: &'static bitcoincore_rpc::Client,
exit: &Exit,
) -> color_eyre::Result<Indexes> {
let starting_indexes = Indexes::try_from((

11
crates/brk_indexer/src/stores/mod.rs → crates/brk_indexer/src/stores.rs
View File

@@ -2,19 +2,14 @@ use std::{fs, path::Path, thread};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, TxIndex,
TxidPrefix,
TxidPrefix, Value, Version,
};
use brk_vec::{AnyIterableVec, Value, Version};
use brk_store::Store;
use brk_vec::AnyIterableVec;
use fjall::{PersistMode, TransactionalKeyspace};
use crate::Indexes;
mod base;
mod meta;
pub use base::*;
pub use meta::*;
use super::Vecs;
#[derive(Clone)]

11
crates/brk_indexer/src/vecs/mod.rs → crates/brk_indexer/src/vecs.rs
View File

@@ -4,10 +4,11 @@ use brk_core::{
AddressBytes, BlockHash, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
OutputType, OutputTypeIndex, P2ABytes, P2AIndex, P2MSIndex, P2PK33Bytes, P2PK33Index,
P2PK65Bytes, P2PK65Index, P2PKHBytes, P2PKHIndex, P2SHBytes, P2SHIndex, P2TRBytes, P2TRIndex,
P2WPKHBytes, P2WPKHIndex, P2WSHBytes, P2WSHIndex, RawLockTime, Sats, StoredF64, StoredU32,
StoredUsize, Timestamp, TxIndex, TxVersion, Txid, UnknownOutputIndex, Weight,
P2WPKHBytes, P2WPKHIndex, P2WSHBytes, P2WSHIndex, RawLockTime, Result, Sats, StoredF64,
StoredU32, StoredUsize, Timestamp, TxIndex, TxVersion, Txid, UnknownOutputIndex, Version,
Weight,
};
use brk_vec::{AnyCollectableVec, AnyIndexedVec, Compressed, IndexedVec, Result, Version};
use brk_vec::{AnyCollectableVec, AnyIndexedVec, Compressed, IndexedVec};
use rayon::prelude::*;
use crate::Indexes;
@@ -340,7 +341,7 @@ impl Vecs {
})
}
pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> brk_vec::Result<()> {
pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> Result<()> {
let saved_height = starting_indexes.height.decremented().unwrap_or_default();
let &Indexes {
@@ -458,7 +459,7 @@ impl Vecs {
&mut self,
index: OutputTypeIndex,
bytes: AddressBytes,
) -> brk_vec::Result<()> {
) -> Result<()> {
match bytes {
AddressBytes::P2PK65(bytes) => self
.p2pk65index_to_p2pk65bytes

3
crates/brk_parser/src/lib.rs
View File

@@ -13,9 +13,6 @@ use brk_core::Height;
use crossbeam::channel::{Receiver, bounded};
use rayon::prelude::*;
pub use bitcoin;
pub use bitcoincore_rpc as rpc;
mod blk_index_to_blk_path;
mod blk_index_to_blk_recap;
mod blk_metadata;

1
crates/brk_query/Cargo.toml
View File

@@ -7,6 +7,7 @@ version.workspace = true
repository.workspace = true
[dependencies]
brk_core = { workspace = true }
brk_computer = { workspace = true }
brk_indexer = { workspace = true }
brk_vec = { workspace = true }

5
crates/brk_query/src/lib.rs
View File

@@ -4,6 +4,7 @@
#![doc = "```"]
use brk_computer::Computer;
use brk_core::Result;
use brk_indexer::Indexer;
use brk_vec::AnyCollectableVec;
use tabled::settings::Style;
@@ -92,10 +93,10 @@ impl<'a> Query<'a> {
) -> color_eyre::Result<Output> {
let mut values = vecs
.iter()
.map(|(_, vec)| -> brk_vec::Result<Vec<serde_json::Value>> {
.map(|(_, vec)| -> Result<Vec<serde_json::Value>> {
vec.collect_range_serde_json(from, to)
})
.collect::<brk_vec::Result<Vec<_>>>()?;
.collect::<Result<Vec<_>>>()?;
if values.is_empty() {
return Ok(Output::default(format));

8
crates/brk_query/src/vec_trees.rs
View File

@@ -18,8 +18,12 @@ impl<'a> VecTrees<'a> {
let split = name.split("_to_").collect::<Vec<_>>();
if split.len() != 2
&& !(split.len() == 3
&& (split.get(1) == Some(&"up")
|| split.get(1).is_some_and(|s| s.starts_with("from"))))
&& split.get(1).is_some_and(|s| {
s == &"up"
|| s.starts_with("from")
|| s == &"cumulative_up"
|| s.starts_with("cumulative_from")
}))
{
dbg!(&name, &split);
panic!();

3
crates/brk_server/Cargo.toml
View File

@@ -8,6 +8,7 @@ repository.workspace = true
[dependencies]
axum = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_computer = { workspace = true }
brk_exit = { workspace = true }
brk_core = { workspace = true }
@@ -25,7 +26,7 @@ log = { workspace = true }
minreq = { workspace = true }
oxc = { version = "0.72.0", features = ["codegen", "minifier"] }
serde = { workspace = true }
tokio = { version = "1.45.1", features = ["full"] }
tokio = { workspace = true }
tower-http = { version = "0.6.4", features = ["compression-full", "trace"] }
zip = "4.0.0"
tracing = "0.1.41"

10
crates/brk_server/examples/main.rs
View File

@@ -1,14 +1,12 @@
use std::{path::Path, thread::sleep, time::Duration};
use bitcoincore_rpc::RpcApi;
use brk_computer::Computer;
use brk_core::default_bitcoin_path;
use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::{
Parser,
rpc::{self, RpcApi},
};
use brk_parser::Parser;
use brk_server::{Server, Website};
use brk_vec::Computation;
@@ -21,9 +19,9 @@ pub fn main() -> color_eyre::Result<()> {
let bitcoin_dir = default_bitcoin_path();
let rpc = Box::leak(Box::new(rpc::Client::new(
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
"http://localhost:8332",
rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
bitcoincore_rpc::Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?));
let exit = Exit::new();

10
crates/brk_server/src/api/query/dts.rs
View File

@@ -57,16 +57,6 @@ impl DTS for Query<'static> {
contents += "\n\nexport function createVecIdToIndexes() {\n";
// contents += &indexes
// .iter()
// .enumerate()
// .map(|(i_of_i, i)| {
// // let lowered = i.to_string().to_lowercase();
// format!(" const {i} = /** @satisfies {{{i}}} */ ({i_of_i});",)
// })
// .collect::<Vec<_>>()
// .join("\n");
contents += "\n\n return /** @type {const} */ ({\n";
self.vec_trees

1
crates/brk_server/src/lib.rs
View File

@@ -26,7 +26,6 @@ use brk_query::Query;
use color_eyre::owo_colors::OwoColorize;
use files::FilesRoutes;
use log::{error, info};
pub use tokio;
use tokio::net::TcpListener;
use tower_http::{compression::CompressionLayer, trace::TraceLayer};

16
crates/brk_store/Cargo.toml Normal file
View File

@@ -0,0 +1,16 @@
[package]
name = "brk_store"
description = "A thin wrapper around fjall"
keywords = ["store", "fjall"]
categories = ["database"]
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
brk_core = { workspace = true }
byteview = { workspace = true }
color-eyre = { workspace = true }
fjall = { workspace = true }
zerocopy = { workspace = true }

9
crates/brk_indexer/src/stores/base.rs → crates/brk_store/src/lib.rs
View File

@@ -6,8 +6,7 @@ use std::{
path::Path,
};
use brk_core::Height;
use brk_vec::{Value, Version};
use brk_core::{Height, Value, Version};
use byteview::ByteView;
use fjall::{
PartitionCreateOptions, PersistMode, ReadTransaction, Result, TransactionalKeyspace,
@@ -15,7 +14,8 @@ use fjall::{
};
use zerocopy::{Immutable, IntoBytes};
use super::StoreMeta;
mod meta;
use meta::*;
pub struct Store<Key, Value> {
meta: StoreMeta,
@@ -47,7 +47,8 @@ where
&path.join(format!("meta/{name}")),
MAJOR_FJALL_VERSION + version,
|| {
Self::open_partition_handle(&keyspace, name).inspect_err(|_| {
Self::open_partition_handle(&keyspace, name).inspect_err(|e| {
eprintln!("{e}");
eprintln!("Delete {path:?} and try again");
})
},

27
crates/brk_indexer/src/stores/meta.rs → crates/brk_store/src/meta.rs
View File

@@ -3,7 +3,7 @@ use std::{
path::{Path, PathBuf},
};
use brk_vec::Version;
use brk_core::Version;
use fjall::{TransactionalKeyspace, TransactionalPartitionHandle};
use zerocopy::{FromBytes, IntoBytes};
@@ -29,8 +29,11 @@ impl StoreMeta {
{
fs::create_dir_all(path)?;
let is_same_version = Version::try_from(Self::path_version_(path).as_path())
.is_ok_and(|prev_version| version == prev_version);
let read_version = Version::try_from(Self::path_version_(path).as_path());
let is_same_version = read_version
.as_ref()
.is_ok_and(|prev_version| &version == prev_version);
let mut partition = open_partition_handle()?;
@@ -56,13 +59,13 @@ impl StoreMeta {
pub fn len(&self) -> usize {
self.len
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
// pub fn is_empty(&self) -> bool {
// self.len() == 0
// }
pub fn version(&self) -> Version {
self.version
}
// pub fn version(&self) -> Version {
// self.version
// }
pub fn export(&mut self, len: usize, height: Height) -> io::Result<()> {
self.len = len;
@@ -71,9 +74,9 @@ impl StoreMeta {
height.write(&self.path_height())
}
pub fn reset(&self) -> io::Result<()> {
Self::reset_(self.pathbuf.as_path())
}
// pub fn reset(&self) -> io::Result<()> {
// Self::reset_(self.pathbuf.as_path())
// }
fn reset_(path: &Path) -> io::Result<()> {
fs::remove_dir_all(path)?;
fs::create_dir(path)

5
crates/brk_vec/examples/main.rs
View File

@@ -1,8 +1,7 @@
use std::{fs, path::Path};
use brk_vec::{
AnyVec, CollectableVec, Compressed, GenericStoredVec, StoredVec, VecIterator, Version,
};
use brk_core::Version;
use brk_vec::{AnyVec, CollectableVec, Compressed, GenericStoredVec, StoredVec, VecIterator};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let _ = fs::remove_dir_all("./vec");

2
crates/brk_vec/src/lib.rs
View File

@@ -3,12 +3,10 @@
#![doc = include_str!("../examples/main.rs")]
#![doc = "```"]
mod enums;
mod structs;
mod traits;
mod variants;
pub use enums::*;
pub use memmap2::Mmap;
pub use structs::*;
pub use traits::*;

2
crates/brk_vec/src/structs/compressed.rs
View File

@@ -1,6 +1,6 @@
use std::{fs, io, ops::Deref, path::Path};
use crate::{Error, Result};
use brk_core::{Error, Result};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Compressed(bool);

3
crates/brk_vec/src/structs/compressed_pages_meta.rs
View File

@@ -4,11 +4,10 @@ use std::{
path::{Path, PathBuf},
};
use brk_core::Result;
use rayon::prelude::*;
use zerocopy::{IntoBytes, TryFromBytes};
use crate::Result;
use super::{CompressedPageMetadata, UnsafeSlice};
#[derive(Debug, Clone)]

3
crates/brk_vec/src/structs/length.rs
View File

@@ -5,11 +5,10 @@ use std::{
path::Path,
};
use brk_core::{Error, Result};
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{Error, Result};
#[derive(
Debug,
Default,

2
crates/brk_vec/src/structs/mod.rs
View File

@@ -3,11 +3,9 @@ mod compressed_page_meta;
mod compressed_pages_meta;
mod length;
mod unsafe_slice;
mod version;
pub use compressed::*;
pub use compressed_page_meta::*;
pub use compressed_pages_meta::*;
pub use length::*;
pub use unsafe_slice::*;
pub use version::*;

2
crates/brk_vec/src/traits/any.rs
View File

@@ -1,6 +1,6 @@
use std::time::Duration;
use crate::{Result, Version};
use brk_core::{Result, Version};
use super::{BoxedVecIterator, StoredIndex, StoredType};

2
crates/brk_vec/src/traits/collectable.rs
View File

@@ -1,4 +1,4 @@
use crate::{Error, Result};
use brk_core::{Error, Result};
use super::{AnyIterableVec, AnyVec, StoredIndex, StoredType};

3
crates/brk_vec/src/traits/generic.rs
View File

@@ -7,10 +7,9 @@ use std::{
};
use arc_swap::ArcSwap;
use brk_core::{Result, Value};
use memmap2::Mmap;
use crate::{Result, Value};
use super::{StoredIndex, StoredType};
pub trait GenericStoredVec<I, T>: Send + Sync

2
crates/brk_vec/src/traits/index.rs
View File

@@ -4,7 +4,7 @@ use std::{
path::{Path, PathBuf},
};
use crate::{Error, Result};
use brk_core::{Error, Result};
pub trait StoredIndex
where

2
crates/brk_vec/src/traits/iterator.rs
View File

@@ -1,6 +1,6 @@
use std::iter::Skip;
use crate::Value;
use brk_core::Value;
use super::{StoredIndex, StoredType};

5
crates/brk_vec/src/variants/compressed.rs
View File

@@ -7,14 +7,15 @@ use std::{
};
use arc_swap::{ArcSwap, Guard};
use brk_core::{Error, Result, Value, Version};
use memmap2::Mmap;
use rayon::prelude::*;
use zstd::DEFAULT_COMPRESSION_LEVEL;
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec,
CompressedPageMetadata, CompressedPagesMetadata, Error, GenericStoredVec, RawVec, Result,
StoredIndex, StoredType, UnsafeSlice, Value, Version,
CompressedPageMetadata, CompressedPagesMetadata, GenericStoredVec, RawVec, StoredIndex,
StoredType, UnsafeSlice,
};
const ONE_KIB: usize = 1024;

4
crates/brk_vec/src/variants/computed.rs
View File

@@ -4,11 +4,11 @@ use brk_exit::Exit;
use clap_derive::ValueEnum;
use serde::{Deserialize, Serialize};
use brk_core::StoredPhantom;
use brk_core::{Result, StoredPhantom, Value, Version};
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
BoxedVecIterator, CollectableVec, Compressed, Result, StoredIndex, StoredType, Value, Version,
BoxedVecIterator, CollectableVec, Compressed, StoredIndex, StoredType,
};
use super::{

8
crates/brk_vec/src/variants/eager.rs
View File

@@ -10,16 +10,16 @@ use std::{
use arc_swap::ArcSwap;
use brk_core::{
Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Height, Sats, StoredUsize, TxIndex,
Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Error, Height, Result, Sats, StoredUsize,
TxIndex, Value, Version,
};
use brk_exit::Exit;
use log::info;
use memmap2::Mmap;
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Compressed, Error,
GenericStoredVec, Result, StoredIndex, StoredType, StoredVec, StoredVecIterator, Value,
VecIterator, Version,
AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Compressed,
GenericStoredVec, StoredIndex, StoredType, StoredVec, StoredVecIterator, VecIterator,
};
const ONE_KIB: usize = 1024;

6
crates/brk_vec/src/variants/indexed.rs
View File

@@ -6,11 +6,11 @@ use std::{
};
use arc_swap::ArcSwap;
use brk_core::Height;
use brk_core::{Error, Height, Result, Value, Version};
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Compressed, Error,
GenericStoredVec, Mmap, Result, StoredIndex, StoredType, StoredVec, Value, Version,
AnyCollectableVec, AnyIterableVec, AnyVec, BoxedVecIterator, CollectableVec, Compressed,
GenericStoredVec, Mmap, StoredIndex, StoredType, StoredVec,
};
use super::StoredVecIterator;

5
crates/brk_vec/src/variants/lazy1.rs
View File

@@ -1,9 +1,10 @@
use core::panic;
use std::marker::PhantomData;
use brk_core::{Result, Value, Version};
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredType, Value, Version,
BoxedVecIterator, CollectableVec, StoredIndex, StoredType,
};
pub type ComputeFrom1<I, T, S1I, S1T> =

4
crates/brk_vec/src/variants/lazy2.rs
View File

@@ -1,8 +1,10 @@
use std::marker::PhantomData;
use brk_core::{Result, Value, Version};
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredType, Value, Version,
BoxedVecIterator, CollectableVec, StoredIndex, StoredType,
};
pub type ComputeFrom2<I, T, S1I, S1T, S2I, S2T> = for<'a> fn(

4
crates/brk_vec/src/variants/lazy3.rs
View File

@@ -1,8 +1,10 @@
use std::marker::PhantomData;
use brk_core::{Result, Value, Version};
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
BoxedVecIterator, CollectableVec, Result, StoredIndex, StoredType, Value, Version,
BoxedVecIterator, CollectableVec, StoredIndex, StoredType,
};
pub type ComputeFrom3<I, T, S1I, S1T, S2I, S2T, S3I, S3T> = for<'a> fn(

3
crates/brk_vec/src/variants/raw.rs
View File

@@ -8,12 +8,13 @@ use std::{
};
use arc_swap::{ArcSwap, Guard};
use brk_core::{Error, Result, Value, Version};
use memmap2::Mmap;
use rayon::prelude::*;
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec,
Error, GenericStoredVec, Result, StoredIndex, StoredType, UnsafeSlice, Value, Version,
GenericStoredVec, StoredIndex, StoredType, UnsafeSlice,
};
#[derive(Debug)]

3
crates/brk_vec/src/variants/stored.rs
View File

@@ -1,11 +1,12 @@
use std::{path::Path, time::Duration};
use arc_swap::ArcSwap;
use brk_core::{Result, Value, Version};
use memmap2::Mmap;
use crate::{
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec,
Compressed, GenericStoredVec, Result, StoredIndex, StoredType, Value, Version,
Compressed, GenericStoredVec, StoredIndex, StoredType,
};
use super::{CompressedVec, CompressedVecIterator, RawVec, RawVecIterator};

298
websites/kibo.money/scripts/main.js
View File

@@ -45,10 +45,17 @@
* "cagr" |
* "vB" |
* "performance" |
* "zscore"
* "sd" |
* "Epoch" |
* "Height" |
* "Type" |
* "zscore" |
* "Bytes"
* } Unit
*/
const localhost = window.location.hostname === "localhost";
function initPackages() {
const imports = {
async signals() {
@@ -682,110 +689,217 @@ function createUtils() {
});
}
const thoroughUnitCheck = localhost;
/**
* @param {VecId} id
*/
function vecidToUnit(id) {
/** @type {Unit} */
/** @type {Unit | undefined} */
let unit;
if (id.endsWith("zscore")) {
unit = "zscore";
} else if (id.endsWith("cagr")) {
unit = "cagr";
} else if (id.endsWith("returns")) {
unit = "performance";
} else if (id === "drawdown" || id.endsWith("oscillator")) {
unit = "percentage";
} else if (id.endsWith("-as-price")) {
unit = "USD";
} else if (id.includes("type")) {
unit = "Type";
} else if (id.includes("days-")) {
unit = "Days";
} else if (id.includes("years-")) {
unit = "Years";
} else if (id === "rawlocktime") {
unit = "Locktime";
} else if (id.startsWith("is-")) {
unit = "Bool";
} else if (
id.includes("bytes") ||
id.includes("hash") ||
id.includes("address") ||
id.includes("txid")
if (
(!unit || thoroughUnitCheck) &&
(id.includes("in-sats") ||
id.endsWith("supply") ||
id.endsWith("stack") ||
id.endsWith("value") ||
((id.includes("coinbase") ||
id.includes("fee") ||
id.includes("subsidy") ||
id.includes("rewards")) &&
!(
id.startsWith("is-") ||
id.includes("in-btc") ||
id.includes("in-usd")
)))
) {
unit = "Hash";
} else if (id.includes("interval")) {
unit = "Seconds";
} else if (id.includes("feerate")) {
unit = "sat/vB";
} else if (id.includes("in-cents")) {
unit = "Cents";
} else if (id.includes("in-usd")) {
unit = "USD";
} else if (id.includes("ratio")) {
unit = "Ratio";
} else if (id.includes("in-btc")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Sats";
}
if ((!unit || thoroughUnitCheck) && id.includes("in-btc")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "BTC";
} else if (
id.includes("in-sats") ||
id.startsWith("sats-") ||
id.includes("input-value") ||
id.includes("output-value") ||
id.includes("fee") ||
id.includes("coinbase") ||
id.includes("subsidy") ||
id.endsWith("stack") ||
id.includes("supply") ||
id.includes("rewards") ||
id.includes("realized-cap")
) {
unit = "Sats";
} else if (
id.includes("open") ||
id.includes("high") ||
id.includes("low") ||
id.includes("close") ||
id.includes("ohlc") ||
id.includes("marketcap") ||
id.includes("ath") ||
id.includes("-sma") ||
id.endsWith("-price") ||
id.startsWith("price-") ||
id.startsWith("realized-")
}
if (
(!unit || thoroughUnitCheck) &&
(id === "high" ||
id === "ohlc" ||
id === "low" ||
id === "close" ||
id === "open" ||
id === "marketcap" ||
id.startsWith("price") ||
id.endsWith("price") ||
id.endsWith("value-created") ||
id.endsWith("value-destroyed") ||
id.endsWith("realized-cap") ||
id.endsWith("realized-loss") ||
id.endsWith("realized-loss-sum") ||
id.endsWith("realized-profit") ||
id.endsWith("realized-profit-sum") ||
id.includes("in-usd") ||
(!id.includes("ratio") && id.endsWith("sma")) ||
id.endsWith("ath"))
) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "USD";
} else if (id.includes("count") || id.match(/v[1-3]/g)) {
unit = "Count";
} else if (id.includes("date")) {
unit = "Date";
} else if (id.includes("timestamp")) {
unit = "Timestamp";
} else if (
id.includes("index") ||
id.includes("height") ||
id.includes("epoch")
}
if ((!unit || thoroughUnitCheck) && id.endsWith("cents")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Cents";
}
if (
(!unit || thoroughUnitCheck) &&
(id.endsWith("ratio") ||
(id.includes("ratio") &&
(id.endsWith("-sma") ||
id.endsWith("-1w-sma") ||
id.endsWith("-1m-sma") ||
id.endsWith("-1y-sma"))) ||
id.endsWith("1sd") ||
id.endsWith("2sd") ||
id.endsWith("3sd") ||
id.endsWith("p0-1") ||
id.endsWith("p0-5") ||
id.endsWith("p1") ||
id.endsWith("p99") ||
id.endsWith("p99-5") ||
id.endsWith("p99-9"))
) {
unit = "Index";
} else if (id === "0" || id === "1" || id === "50" || id === "100") {
unit = "constant";
} else if (id.includes("difficulty")) {
unit = "Difficulty";
} else if (id.includes("-size")) {
unit = "mb";
} else if (id.includes("weight")) {
unit = "WU";
} else if (id.includes("vbytes") || id.includes("vsize")) {
unit = "vB";
} else if (id.includes("version")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Ratio";
}
if (
(!unit || thoroughUnitCheck) &&
(id === "drawdown" || id.endsWith("oscillator"))
) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "percentage";
}
if (
(!unit || thoroughUnitCheck) &&
(id.endsWith("count") ||
id.includes("-count-") ||
id.startsWith("block-count") ||
id.includes("tx-v"))
) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Count";
}
if ((!unit || thoroughUnitCheck) && id.startsWith("is-")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Bool";
}
if ((!unit || thoroughUnitCheck) && id.endsWith("type")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Type";
}
if (
(!unit || thoroughUnitCheck) &&
(id === "interval" || id.startsWith("block-interval"))
) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Seconds";
}
if ((!unit || thoroughUnitCheck) && id.endsWith("returns")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "performance";
}
if ((!unit || thoroughUnitCheck) && id.endsWith("zscore")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "zscore";
}
if ((!unit || thoroughUnitCheck) && id.endsWith("locktime")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Locktime";
}
if ((!unit || thoroughUnitCheck) && id.endsWith("cagr")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "cagr";
}
if ((!unit || thoroughUnitCheck) && id.endsWith("version")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Version";
} else if (id === "value") {
unit = "Sats";
} else {
}
if (
(!unit || thoroughUnitCheck) &&
(id === "txid" || (id.endsWith("bytes") && !id.endsWith("vbytes")))
) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Bytes";
}
if ((!unit || thoroughUnitCheck) && id.endsWith("standard-deviation")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "sd";
}
if (
(!unit || thoroughUnitCheck) &&
(id.endsWith("-size") || id.endsWith("-size-sum"))
) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "mb";
}
if (
(!unit || thoroughUnitCheck) &&
(id.endsWith("vsize") ||
id.endsWith("vbytes") ||
id.endsWith("-vbytes-sum"))
) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "vB";
}
if ((!unit || thoroughUnitCheck) && id.includes("weight")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "WU";
}
if ((!unit || thoroughUnitCheck) && id.endsWith("index")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Index";
}
if (
(!unit || thoroughUnitCheck) &&
(id === "date" || id === "date-fixed")
) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Date";
}
if (
(!unit || thoroughUnitCheck) &&
(id === "timestamp" || id === "timestamp-fixed")
) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Timestamp";
}
if ((!unit || thoroughUnitCheck) && id.endsWith("height")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Height";
}
if ((!unit || thoroughUnitCheck) && id.endsWith("epoch")) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Epoch";
}
if ((!unit || thoroughUnitCheck) && id === "difficulty") {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Difficulty";
}
if ((!unit || thoroughUnitCheck) && id === "blockhash") {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "Hash";
}
if (
(!unit || thoroughUnitCheck) &&
(id === "0" || id === "1" || id === "50" || id === "100")
) {
if (unit) throw Error(`Unit "${unit}" already assigned "${id}"`);
unit = "constant";
}
if (!unit) {
console.log();
throw Error(`Unit not set for "${id}"`);
}
return unit;
return /** @type {Unit} */ (unit);
}
const locale = {
@@ -1476,7 +1590,7 @@ function initEnv() {
iphone,
ipad,
ios,
localhost: window.location.hostname === "localhost",
localhost,
};
}
/** @typedef {ReturnType<typeof initEnv>} Env */

1191
websites/kibo.money/scripts/options.js
View File

File diff suppressed because it is too large Load Diff

18
websites/kibo.money/scripts/simulation.js
View File

@@ -720,13 +720,13 @@ export function init({
{
title: "Dollars Converted",
type: "Line",
color: colors.dollars,
color: colors.green,
data: totalInvestedAmountData,
},
{
title: "Dollars Left",
type: "Line",
color: colors.offDollars,
color: colors.emerald,
data: dollarsLeftData,
defaultActive: false,
},
@@ -758,7 +758,7 @@ export function init({
{
title: "Bitcoin Stack",
type: "Line",
color: colors.bitcoin,
color: colors.orange,
data: bitcoinData,
},
],
@@ -788,7 +788,7 @@ export function init({
{
title: "Average Price Paid",
type: "Line",
color: colors.lightDollars,
color: colors.lime,
data: averagePricePaidData,
},
],
@@ -1074,14 +1074,14 @@ export function init({
*/
const c = (c, t) => createColoredSpan({ color: c, text: t });
const serInvestedAmount = c("dollars", fd(investedAmount));
const serInvestedAmount = c("green", fd(investedAmount));
const serDaysCount = c("sky", f(daysCount));
const serSats = c("orange", f(sats));
const serBitcoin = c("orange", `~${f(bitcoin)}`);
const serBitcoinValue = c("amber", fd(bitcoinValue));
const serAveragePricePaid = c("lightDollars", fd(averagePricePaid));
const serAveragePricePaid = c("lime", fd(averagePricePaid));
const serRoi = c("yellow", fp(roi / 100));
const serDollars = c("offDollars", fd(dollars));
const serDollars = c("emerald", fd(dollars));
const serTotalFeesPaid = c("rose", fd(totalFeesPaid));
p1.innerHTML = `After exchanging ${serInvestedAmount} in the span of ${serDaysCount} days, you would have accumulated ${serSats} Satoshis (${serBitcoin} Bitcoin) worth today ${serBitcoinValue} at an average price of ${serAveragePricePaid} per Bitcoin with a return of investment of ${serRoi}, have ${serDollars} left and paid a total of ${serTotalFeesPaid} in fees.`;
@@ -1089,8 +1089,8 @@ export function init({
const dayDiff = Math.floor(
utils.date.differenceBetween(new Date(), lastInvestDay),
);
const serDailyInvestment = c("offDollars", fd(dailyInvestment));
const setLastSatsAdded = c("bitcoin", f(lastSatsAdded));
const serDailyInvestment = c("emerald", fd(dailyInvestment));
const setLastSatsAdded = c("orange", f(lastSatsAdded));
p2.innerHTML = `You would've last bought ${c("blue", dayDiff ? `${f(dayDiff)} ${dayDiff > 1 ? "days" : "day"} ago` : "today")} and exchanged ${serDailyInvestment} for approximately ${setLastSatsAdded} Satoshis`;
const serProfitableDaysRatio = c("green", fp(profitableDaysRatio));

891
websites/kibo.money/scripts/vecid-to-indexes.js
View File

File diff suppressed because it is too large Load Diff