Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-06-10 23:13:33 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: wip

Browse Source
This commit is contained in:
nym21
2025-05-31 20:45:59 +02:00
parent cfc3081e8a
commit f976f672cf
34 changed files with 2109 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+58 -61
View File
@@ -457,8 +457,9 @@ version = "0.0.40"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
"byteview 0.7.0",
"byteview",
"derive_deref",
"fjall",
"jiff",
"log",
"rapidhash",
@@ -467,6 +468,7 @@ dependencies = [
"serde_bytes",
"serde_derive",
"serde_json",
"thiserror",
"zerocopy",
"zerocopy-derive",
]
@@ -505,7 +507,7 @@ dependencies = [
"brk_parser",
"brk_store",
"brk_vec",
"byteview 0.7.0",
"byteview",
"color-eyre",
"fjall",
"log",
@@ -592,6 +594,7 @@ dependencies = [
"brk_core",
"brk_store",
"brk_vec",
"fjall",
"rayon",
"serde",
"zerocopy",
@@ -603,7 +606,7 @@ name = "brk_store"
version = "0.0.40"
dependencies = [
"brk_core",
"byteview 0.7.0",
"byteview",
"color-eyre",
"fjall",
"zerocopy",
@@ -682,12 +685,6 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
[[package]]
name = "byteview"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f996a0c883b1ae93c53ffd21857b6eaba768998dbb1f7e2f269d53dd6553256c"
[[package]]
name = "bzip2"
version = "0.5.2"
@@ -718,9 +715,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.24"
version = "1.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7"
checksum = "d0fc897dc1e865cc67c0e05a836d9d3f1df3cbe442aa4a9473b18e12624a4951"
dependencies = [
"jobserver",
"libc",
@@ -803,9 +800,9 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
[[package]]
name = "color-eyre"
version = "0.6.4"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e1761c0e16f8883bbbb8ce5990867f4f06bf11a0253da6495a04ce4b6ef0ec"
checksum = "e5920befb47832a6d61ee3a3a846565cfa39b331331e68a3b1d1116630f2f26d"
dependencies = [
"backtrace",
"color-spantrace",
@@ -818,9 +815,9 @@ dependencies = [
[[package]]
name = "color-spantrace"
version = "0.2.2"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ddd8d5bfda1e11a501d0a7303f3bfed9aa632ebdb859be40d0fd70478ed70d5"
checksum = "b8b88ea9df13354b55bc7234ebcce36e6ef896aca2e42a15de9e10edce01b427"
dependencies = [
"once_cell",
"owo-colors",
@@ -1161,12 +1158,12 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]]
name = "fjall"
version = "2.10.0"
version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b929b3db7be7d7b4d4df67fb016fc446b8f57507b48a82e69d2a30610e460f28"
checksum = "13279146a877c2060f668bc4c477af8ef5aa42732c58dca32fcb4aff40edc5b4"
dependencies = [
"byteorder",
"byteview 0.6.1",
"byteview",
"dashmap",
"log",
"lsm-tree",
@@ -1653,9 +1650,9 @@ checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
[[package]]
name = "lock_api"
version = "0.4.12"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765"
dependencies = [
"autocfg",
"scopeguard",
@@ -1845,9 +1842,9 @@ checksum = "26995317201fa17f3656c36716aed4a7c81743a9634ac4c99c0eeda495db0cec"
[[package]]
name = "oxc"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61a4d9521b1c1c9750bef7fd133b542d48b0073097da97b0e0bbb89b6b59bc6c"
checksum = "e91ecbc3f076425da55380d3f3f873bbbd0a10f56d9f9b6c0d15c88a2188f127"
dependencies = [
"oxc_allocator",
"oxc_ast",
@@ -1888,9 +1885,9 @@ dependencies = [
[[package]]
name = "oxc_allocator"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e63bc2ee951d0991953c95412e8907457fc38c8d9366a65cfc31a0490cd6a5a2"
checksum = "94677be5d1874da150784b9d551bae0dfad7e5a6121a2c6acccda335c05504ea"
dependencies = [
"allocator-api2",
"bumpalo",
@@ -1901,9 +1898,9 @@ dependencies = [
[[package]]
name = "oxc_ast"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7def3a0a7776127c60afa410a6a4a03b27dec7c65d350478b234a7127ca53bf2"
checksum = "26f7b73c9a8e265169d98cd2395cbab52fcb849d3297b1ec2511a1d908775c73"
dependencies = [
"bitflags",
"cow-utils",
@@ -1918,9 +1915,9 @@ dependencies = [
[[package]]
name = "oxc_ast_macros"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7d2acd02146b9f5d69b4035a49f0ed91139b9168ff1559234481df9a4195630"
checksum = "3f411489fef9ace92fdea3105490f9aab07a78c9adff25b62345b1a1bd49ee0c"
dependencies = [
"phf",
"proc-macro2",
@@ -1930,9 +1927,9 @@ dependencies = [
[[package]]
name = "oxc_ast_visit"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9152368ec7ab6785568e3b53908a9c505402ee49e51cc77ebf717f7bfe60454"
checksum = "cf97a9b275654d7fc921c0c472ceab1ae167b670c9a761e5c2aec66b9076157b"
dependencies = [
"oxc_allocator",
"oxc_ast",
@@ -1942,9 +1939,9 @@ dependencies = [
[[package]]
name = "oxc_cfg"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bf5714ec4d469be32521c3c282df4d5a45abfa2bafc926b364b9869651661dd"
checksum = "2ac937d952a9990180f9090b5784713f66407a467f34e84f78bd204a7c2d6d73"
dependencies = [
"bitflags",
"itertools",
@@ -1957,9 +1954,9 @@ dependencies = [
[[package]]
name = "oxc_codegen"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e2cfa58ff8217601c4b4b525caef5a3c004bb017d3050fd73252fc91868c627"
checksum = "b09bbfd5a9faadff9578858f28dcfb46e2f9c4929ee0d8e076f928afe618010f"
dependencies = [
"bitflags",
"cow-utils",
@@ -1978,18 +1975,18 @@ dependencies = [
[[package]]
name = "oxc_data_structures"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3a91991eb494f1c3619feb27af1298bd05b0caac6198cdee528a61b7709ae9f"
checksum = "09ebee1a21c5d47b6e845357274904ec9568b001d3ff6ab04d4cc4754fd148c2"
dependencies = [
"rustversion",
]
[[package]]
name = "oxc_diagnostics"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "580a11ff9d17d5b5ca93e9fd9a3d0404f010e71b5a6ee743b4e9f4641d786ab1"
checksum = "7fa03ff098ecf135235ed3c88e013a22e9b58caf1108d32eca5261daf2d88330"
dependencies = [
"cow-utils",
"oxc-miette",
@@ -1997,9 +1994,9 @@ dependencies = [
[[package]]
name = "oxc_ecmascript"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5613f7445cef6a08b9543dda6cb1962e5829c5ff5bae036d2bd41b740602e3eb"
checksum = "e1b5dba7d2918b9a59f82ce974b85cc6de1e577d06f49d1e608505e9da6a0b02"
dependencies = [
"cow-utils",
"num-bigint",
@@ -2011,9 +2008,9 @@ dependencies = [
[[package]]
name = "oxc_estree"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fb3d8b5e6674a51a9d5a8e1f051eba423c2b78381ecd0b7847517967193cbb8"
checksum = "7a870fcc8a8a8eeb3471cb51df3628f04e9f53521608e735a34bd9b6f96ce78c"
[[package]]
name = "oxc_index"
@@ -2023,9 +2020,9 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392"
[[package]]
name = "oxc_mangler"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aaf4117f87d1d6e3adff0fd3f077cdf321a0e6292042bc1ba1b8df8be010e78b"
checksum = "1330ed79d68bae3bcc3d9ceca8b690f1d27056315ad4a6a97a8eda55e8bb8244"
dependencies = [
"fixedbitset",
"itertools",
@@ -2040,9 +2037,9 @@ dependencies = [
[[package]]
name = "oxc_minifier"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d019e9ff013696e27d6a22548fcb43c5000a8d2c8c97a9bba635c1f8aafd0402"
checksum = "3e788a77708893ba1cf82a9291ce18bdb6fbb69711aeeaf50514fd775c17a1a3"
dependencies = [
"cow-utils",
"oxc_allocator",
@@ -2062,9 +2059,9 @@ dependencies = [
[[package]]
name = "oxc_parser"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42bcc81f20b04c4d186ccf576487b93b6b437bda8626349b6ef770795e65781f"
checksum = "ca8194f22ac433d9f2575f3f5a35fdc2ec403877a2c98b8bbb7047acc73e07e6"
dependencies = [
"bitflags",
"cow-utils",
@@ -2085,9 +2082,9 @@ dependencies = [
[[package]]
name = "oxc_regular_expression"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0863519e3231996e3a6c2a86f068712dfed956d44228aa744593dde2ffbe169c"
checksum = "bac34c62476a83f00f36fb08991730db51a722082125067f9562257fd97cbb88"
dependencies = [
"bitflags",
"oxc_allocator",
@@ -2101,9 +2098,9 @@ dependencies = [
[[package]]
name = "oxc_semantic"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09a645cc47ed8dd6ace276d829cbe5cbf6a7fcb909093f533f5d29fa35545bdb"
checksum = "564256e85189bf5e6fb331f08892eeb8e13f3fcadb0dc08a85ce7db59519e174"
dependencies = [
"itertools",
"oxc_allocator",
@@ -2137,9 +2134,9 @@ dependencies = [
[[package]]
name = "oxc_span"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8580e4eedc08a6168a4a530fb407a17f70d270e8f2b2f6653914422b03f10321"
checksum = "886510fc6db2c5a7a905feeb966e613527bdbe2e544057923957ba79e3e93142"
dependencies = [
"compact_str",
"oxc-miette",
@@ -2150,9 +2147,9 @@ dependencies = [
[[package]]
name = "oxc_syntax"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d29704018fbd8b45711284729c5cde09c29391f254d7bca7e9c6899c0c44816"
checksum = "3dddefc02737686c68da8597a88ba5514e7134aba006c61f72fb1752451f95cc"
dependencies = [
"bitflags",
"cow-utils",
@@ -2171,9 +2168,9 @@ dependencies = [
[[package]]
name = "oxc_traverse"
version = "0.72.1"
version = "0.72.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c00ac4c7344ccd560bb8a2f89b2ca8b32e2efa8bdfdb305ef26be1f1fac5733"
checksum = "fa88eafbdafb3f0da21b03fb41b146602360dc2e952810e9d26688b2510e3693"
dependencies = [
"itoa",
"oxc_allocator",
@@ -2200,9 +2197,9 @@ dependencies = [
[[package]]
name = "parking_lot_core"
version = "0.9.10"
version = "0.9.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
dependencies = [
"cfg-if",
"libc",
@@ -3203,7 +3200,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"
dependencies = [
"byteorder",
"byteview 0.6.1",
"byteview",
"interval-heap",
"log",
"path-absolutize",
Cargo.toml
+3 -3
View File
@@ -32,12 +32,12 @@ brk_server = { version = "0", path = "crates/brk_server" }
brk_state = { version = "0", path = "crates/brk_state" }
brk_store = { version = "0", path = "crates/brk_store" }
brk_vec = { version = "0", path = "crates/brk_vec" }
byteview = "0.7.0"
byteview = "=0.6.1"
clap = { version = "4.5.39", features = ["string"] }
clap_derive = "4.5.32"
color-eyre = "0.6.4"
color-eyre = "0.6.5"
derive_deref = "1.1.1"
fjall = "2.10.0"
fjall = "2.11.0"
jiff = "0.2.14"
log = { version = "0.4.27" }
minreq = { version = "2.13.4", features = ["https", "serde_json"] }
crates/brk_computer/src/lib.rs
+3
View File
@@ -47,12 +47,14 @@ impl Computer {
computation: Computation,
) -> color_eyre::Result<()> {
self.vecs = Some(Vecs::import(
// TODO: Give self.path, join inside import
&self.path.join("vecs/computed"),
VERSION + Version::ZERO,
indexer,
self.fetcher.is_some(),
computation,
self.compressed,
indexer.keyspace(),
)?);
Ok(())
}
@@ -61,6 +63,7 @@ impl Computer {
/// Clone struct instead
pub fn import_stores(&mut self, indexer: &Indexer) -> color_eyre::Result<()> {
self.stores = Some(Stores::import(
// TODO: Give self.path, join inside import
&self.path.join("stores"),
VERSION + Version::ZERO,
indexer.keyspace(),
crates/brk_computer/src/vecs/blocks.rs
+2 -1
View File
@@ -49,6 +49,7 @@ impl Vecs {
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
path,
"timestamp",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_first(),
@@ -132,7 +133,7 @@ impl Vecs {
starting_indexes: &Indexes,
exit: &Exit,
) -> color_eyre::Result<()> {
self.timeindexes_to_timestamp.compute(
self.timeindexes_to_timestamp.compute_all(
indexer,
indexes,
starting_indexes,
crates/brk_computer/src/vecs/fetched.rs
+42 -16
View File
@@ -169,6 +169,7 @@ impl Vecs {
timeindexes_to_open: ComputedVecsFromDateIndex::forced_import(
path,
"open",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_first(),
@@ -176,6 +177,7 @@ impl Vecs {
timeindexes_to_high: ComputedVecsFromDateIndex::forced_import(
path,
"high",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_max(),
@@ -183,6 +185,7 @@ impl Vecs {
timeindexes_to_low: ComputedVecsFromDateIndex::forced_import(
path,
"low",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_min(),
@@ -190,6 +193,7 @@ impl Vecs {
timeindexes_to_close: ComputedVecsFromDateIndex::forced_import(
path,
"close",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
@@ -197,6 +201,7 @@ impl Vecs {
timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
"open_in_sats",
true,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_first(),
@@ -204,6 +209,7 @@ impl Vecs {
timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
"high_in_sats",
true,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_max(),
@@ -211,6 +217,7 @@ impl Vecs {
timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
"low_in_sats",
true,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_min(),
@@ -218,6 +225,7 @@ impl Vecs {
timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import(
path,
"close_in_sats",
true,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
@@ -463,7 +471,7 @@ impl Vecs {
exit,
)?;
self.timeindexes_to_close.compute(
self.timeindexes_to_close.compute_all(
indexer,
indexes,
starting_indexes,
@@ -478,7 +486,7 @@ impl Vecs {
},
)?;
self.timeindexes_to_high.compute(
self.timeindexes_to_high.compute_all(
indexer,
indexes,
starting_indexes,
@@ -493,7 +501,7 @@ impl Vecs {
},
)?;
self.timeindexes_to_low.compute(
self.timeindexes_to_low.compute_all(
indexer,
indexes,
starting_indexes,
@@ -508,7 +516,7 @@ impl Vecs {
},
)?;
self.timeindexes_to_open.compute(
self.timeindexes_to_open.compute_all(
indexer,
indexes,
starting_indexes,
@@ -776,7 +784,7 @@ impl Vecs {
},
)?;
self.timeindexes_to_open_in_sats.compute(
self.timeindexes_to_open_in_sats.compute_all(
indexer,
indexes,
starting_indexes,
@@ -784,14 +792,14 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_open.dateindex,
self.timeindexes_to_open.dateindex.as_ref().unwrap(),
|(i, open, ..)| (i, Open::new(Sats::ONE_BTC / *open)),
exit,
)
},
)?;
self.timeindexes_to_high_in_sats.compute(
self.timeindexes_to_high_in_sats.compute_all(
indexer,
indexes,
starting_indexes,
@@ -799,14 +807,14 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_low.dateindex,
self.timeindexes_to_low.dateindex.as_ref().unwrap(),
|(i, low, ..)| (i, High::new(Sats::ONE_BTC / *low)),
exit,
)
},
)?;
self.timeindexes_to_low_in_sats.compute(
self.timeindexes_to_low_in_sats.compute_all(
indexer,
indexes,
starting_indexes,
@@ -814,14 +822,14 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_high.dateindex,
self.timeindexes_to_high.dateindex.as_ref().unwrap(),
|(i, high, ..)| (i, Low::new(Sats::ONE_BTC / *high)),
exit,
)
},
)?;
self.timeindexes_to_close_in_sats.compute(
self.timeindexes_to_close_in_sats.compute_all(
indexer,
indexes,
starting_indexes,
@@ -829,7 +837,7 @@ impl Vecs {
|v, _, _, starting_indexes, exit| {
v.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_close.dateindex,
self.timeindexes_to_close.dateindex.as_ref().unwrap(),
|(i, close, ..)| (i, Close::new(Sats::ONE_BTC / *close)),
exit,
)
@@ -856,12 +864,30 @@ impl Vecs {
exit,
)?;
let mut dateindex_first_iter = self.timeindexes_to_open_in_sats.dateindex.iter();
let mut dateindex_max_iter = self.timeindexes_to_high_in_sats.dateindex.iter();
let mut dateindex_min_iter = self.timeindexes_to_low_in_sats.dateindex.iter();
let mut dateindex_first_iter = self
.timeindexes_to_open_in_sats
.dateindex
.as_ref()
.unwrap()
.iter();
let mut dateindex_max_iter = self
.timeindexes_to_high_in_sats
.dateindex
.as_ref()
.unwrap()
.iter();
let mut dateindex_min_iter = self
.timeindexes_to_low_in_sats
.dateindex
.as_ref()
.unwrap()
.iter();
self.dateindex_to_ohlc_in_sats.compute_transform(
starting_indexes.dateindex,
&self.timeindexes_to_close_in_sats.dateindex,
self.timeindexes_to_close_in_sats
.dateindex
.as_ref()
.unwrap(),
|(i, close, ..)| {
(
i,
crates/brk_computer/src/vecs/grouped/from_dateindex.rs
+56 -28
View File
@@ -5,7 +5,7 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, EagerVec};
use brk_vec::{AnyCollectableVec, AnyIterableVec, Compressed, EagerVec};
use crate::vecs::{Indexes, indexes};
@@ -16,7 +16,7 @@ pub struct ComputedVecsFromDateIndex<T>
where
T: ComputedType + PartialOrd,
{
pub dateindex: EagerVec<DateIndex, T>,
pub dateindex: Option<EagerVec<DateIndex, T>>,
pub dateindex_extra: ComputedVecBuilder<DateIndex, T>,
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
pub monthindex: ComputedVecBuilder<MonthIndex, T>,
@@ -34,10 +34,16 @@ where
pub fn forced_import(
path: &Path,
name: &str,
compute_source: bool,
version: Version,
compressed: Compressed,
options: StorableVecGeneatorOptions,
) -> color_eyre::Result<Self> {
let dateindex = compute_source.then(|| {
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, compressed)
.unwrap()
});
let dateindex_extra = ComputedVecBuilder::forced_import(
path,
name,
@@ -49,12 +55,7 @@ where
let options = options.remove_percentiles();
Ok(Self {
dateindex: EagerVec::forced_import(
path,
name,
version + VERSION + Version::ZERO,
compressed,
)?,
dateindex,
dateindex_extra,
weekindex: ComputedVecBuilder::forced_import(
path,
@@ -94,7 +95,7 @@ where
})
}
pub fn compute<F>(
pub fn compute_all<F>(
&mut self,
indexer: &Indexer,
indexes: &indexes::Vecs,
@@ -112,14 +113,15 @@ where
) -> Result<()>,
{
compute(
&mut self.dateindex,
self.dateindex.as_mut().unwrap(),
indexer,
indexes,
starting_indexes,
exit,
)?;
self.compute_rest(indexes, starting_indexes, exit)
let dateindex: Option<&EagerVec<DateIndex, T>> = None;
self.compute_rest(indexes, starting_indexes, exit, dateindex)
}
pub fn compute_rest(
@@ -127,25 +129,49 @@ where
indexes: &indexes::Vecs,
starting_indexes: &Indexes,
exit: &Exit,
dateindex: Option<&impl AnyIterableVec<DateIndex, T>>,
) -> color_eyre::Result<()> {
self.dateindex_extra
.extend(starting_indexes.dateindex, &self.dateindex, exit)?;
if let Some(dateindex) = dateindex {
self.dateindex_extra
.extend(starting_indexes.dateindex, dateindex, exit)?;
self.weekindex.compute(
starting_indexes.weekindex,
&self.dateindex,
&indexes.weekindex_to_first_dateindex,
&indexes.weekindex_to_dateindex_count,
exit,
)?;
self.weekindex.compute(
starting_indexes.weekindex,
dateindex,
&indexes.weekindex_to_first_dateindex,
&indexes.weekindex_to_dateindex_count,
exit,
)?;
self.monthindex.compute(
starting_indexes.monthindex,
&self.dateindex,
&indexes.monthindex_to_first_dateindex,
&indexes.monthindex_to_dateindex_count,
exit,
)?;
self.monthindex.compute(
starting_indexes.monthindex,
dateindex,
&indexes.monthindex_to_first_dateindex,
&indexes.monthindex_to_dateindex_count,
exit,
)?;
} else {
let dateindex = self.dateindex.as_ref().unwrap();
self.dateindex_extra
.extend(starting_indexes.dateindex, dateindex, exit)?;
self.weekindex.compute(
starting_indexes.weekindex,
dateindex,
&indexes.weekindex_to_first_dateindex,
&indexes.weekindex_to_dateindex_count,
exit,
)?;
self.monthindex.compute(
starting_indexes.monthindex,
dateindex,
&indexes.monthindex_to_first_dateindex,
&indexes.monthindex_to_dateindex_count,
exit,
)?;
}
self.quarterindex.from_aligned(
starting_indexes.quarterindex,
@@ -176,7 +202,9 @@ where
pub fn vecs(&self) -> Vec<&dyn AnyCollectableVec> {
[
vec![&self.dateindex as &dyn AnyCollectableVec],
self.dateindex
.as_ref()
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.dateindex_extra.vecs(),
self.weekindex.vecs(),
self.monthindex.vecs(),
crates/brk_computer/src/vecs/grouped/from_height.rs
+5 -7
View File
@@ -17,7 +17,7 @@ pub struct ComputedVecsFromHeight<T>
where
T: ComputedType + PartialOrd,
{
pub height: Option<Box<EagerVec<Height, T>>>,
pub height: Option<EagerVec<Height, T>>,
pub height_extra: ComputedVecBuilder<Height, T>,
pub dateindex: ComputedVecBuilder<DateIndex, T>,
pub weekindex: ComputedVecBuilder<WeekIndex, T>,
@@ -45,10 +45,8 @@ where
options: StorableVecGeneatorOptions,
) -> color_eyre::Result<Self> {
let height = compute_source.then(|| {
Box::new(
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, compressed)
.unwrap(),
)
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, compressed)
.unwrap()
});
let height_extra = ComputedVecBuilder::forced_import(
@@ -169,7 +167,7 @@ where
exit,
)?;
} else {
let height = self.height.as_ref().unwrap().as_ref();
let height = self.height.as_ref().unwrap();
self.height_extra
.extend(starting_indexes.height, height, exit)?;
@@ -238,7 +236,7 @@ where
[
self.height
.as_ref()
.map_or(vec![], |v| vec![v.as_ref() as &dyn AnyCollectableVec]),
.map_or(vec![], |v| vec![v as &dyn AnyCollectableVec]),
self.height_extra.vecs(),
self.dateindex.vecs(),
self.weekindex.vecs(),
crates/brk_computer/src/vecs/grouped/ratio_from_dateindex.rs
+285 -129
View File
@@ -69,6 +69,7 @@ impl ComputedRatioVecsFromDateIndex {
ComputedVecsFromDateIndex::forced_import(
path,
name,
true,
version + VERSION,
compressed,
options,
@@ -78,6 +79,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -85,6 +87,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_sma"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -92,6 +95,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_1w_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1w_sma"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -99,6 +103,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_1m_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1m_sma"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -106,6 +111,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_1y_sma: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1y_sma"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -113,6 +119,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_1y_sma_momentum_oscillator: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_1y_sma_momentum_oscillator"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -120,6 +127,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_standard_deviation: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_standard_deviation"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -127,6 +135,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p99_9: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_9"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -134,6 +143,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p99_5: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_5"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -141,6 +151,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p99: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -148,6 +159,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p1: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -155,6 +167,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p0_5: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_5"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -162,6 +175,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p0_1: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_1"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -169,6 +183,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p1sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1sd"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -176,6 +191,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p2sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p2sd"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -183,6 +199,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p3sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p3sd"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -190,6 +207,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_m1sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m1sd"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -197,6 +215,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_m2sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m2sd"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -204,6 +223,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_m3sd: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m3sd"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -211,6 +231,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p99_9_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_9_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -218,6 +239,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p99_5_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_5_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -225,6 +247,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p99_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p99_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -232,6 +255,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p1_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -239,6 +263,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p0_5_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_5_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -246,6 +271,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p0_1_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p0_1_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -253,6 +279,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p1sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p1sd_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -260,6 +287,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p2sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p2sd_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -267,6 +295,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_p3sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_p3sd_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -274,6 +303,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_m1sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m1sd_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -281,6 +311,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_m2sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m2sd_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -288,6 +319,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_m3sd_as_price: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_m3sd_as_price"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -295,6 +327,7 @@ impl ComputedRatioVecsFromDateIndex {
ratio_zscore: ComputedVecsFromDateIndex::forced_import(
path,
&format!("{name}_ratio_zscore"),
true,
version + VERSION + Version::ZERO,
compressed,
options,
@@ -320,10 +353,13 @@ impl ComputedRatioVecsFromDateIndex {
&Exit,
) -> Result<()>,
{
self.price
.as_mut()
.unwrap()
.compute(indexer, indexes, starting_indexes, exit, compute)?;
self.price.as_mut().unwrap().compute_all(
indexer,
indexes,
starting_indexes,
exit,
compute,
)?;
let date_to_price_opt: Option<&EagerVec<DateIndex, Dollars>> = None;
self.compute_rest(
@@ -349,9 +385,9 @@ impl ComputedRatioVecsFromDateIndex {
std::mem::transmute(&self.price.as_ref().unwrap().dateindex)
});
let closes = &fetched.timeindexes_to_close.dateindex;
let closes = fetched.timeindexes_to_close.dateindex.as_ref().unwrap();
self.ratio.compute(
self.ratio.compute_all(
indexer,
indexes,
starting_indexes,
@@ -376,7 +412,7 @@ impl ComputedRatioVecsFromDateIndex {
let min_ratio_date = DateIndex::try_from(Date::MIN_RATIO).unwrap();
self.ratio_sma.compute(
self.ratio_sma.compute_all(
indexer,
indexes,
starting_indexes,
@@ -384,7 +420,7 @@ impl ComputedRatioVecsFromDateIndex {
|v, _, _, starting_indexes, exit| {
v.compute_sma_(
starting_indexes.dateindex,
&self.ratio.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
usize::MAX,
exit,
Some(min_ratio_date),
@@ -392,7 +428,7 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_1w_sma.compute(
self.ratio_1w_sma.compute_all(
indexer,
indexes,
starting_indexes,
@@ -400,7 +436,7 @@ impl ComputedRatioVecsFromDateIndex {
|v, _, _, starting_indexes, exit| {
v.compute_sma_(
starting_indexes.dateindex,
&self.ratio.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
7,
exit,
Some(min_ratio_date),
@@ -408,7 +444,7 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_1m_sma.compute(
self.ratio_1m_sma.compute_all(
indexer,
indexes,
starting_indexes,
@@ -416,7 +452,7 @@ impl ComputedRatioVecsFromDateIndex {
|v, _, _, starting_indexes, exit| {
v.compute_sma_(
starting_indexes.dateindex,
&self.ratio.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
30,
exit,
Some(min_ratio_date),
@@ -424,7 +460,7 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_1y_sma.compute(
self.ratio_1y_sma.compute_all(
indexer,
indexes,
starting_indexes,
@@ -432,7 +468,7 @@ impl ComputedRatioVecsFromDateIndex {
|v, _, _, starting_indexes, exit| {
v.compute_sma_(
starting_indexes.dateindex,
&self.ratio.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
365,
exit,
Some(min_ratio_date),
@@ -440,16 +476,17 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_1y_sma_momentum_oscillator.compute(
self.ratio_1y_sma_momentum_oscillator.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|v, _, _, starting_indexes, exit| {
let mut ratio_1y_sma_iter = self.ratio_1y_sma.dateindex.into_iter();
let mut ratio_1y_sma_iter =
self.ratio_1y_sma.dateindex.as_ref().unwrap().into_iter();
v.compute_transform(
starting_indexes.dateindex,
&self.ratio.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
|(i, ratio, ..)| {
(
i,
@@ -461,7 +498,7 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
let ratio_version = self.ratio.dateindex.version();
let ratio_version = self.ratio.dateindex.as_ref().unwrap().version();
self.mut_ratio_vecs()
.iter_mut()
.try_for_each(|v| -> Result<()> {
@@ -478,7 +515,7 @@ impl ComputedRatioVecsFromDateIndex {
.unwrap()
.min(starting_indexes.dateindex);
let mut sorted = self.ratio.dateindex.collect_range(
let mut sorted = self.ratio.dateindex.as_ref().unwrap().collect_range(
Some(min_ratio_date.unwrap_to_usize()),
Some(starting_dateindex.unwrap_to_usize()),
)?;
@@ -489,68 +526,116 @@ impl ComputedRatioVecsFromDateIndex {
// unreachable!();
// }
let mut sma_iter = self.ratio_sma.dateindex.into_iter();
let mut sma_iter = self.ratio_sma.dateindex.as_ref().unwrap().into_iter();
let nan = StoredF32::from(f32::NAN);
self.ratio
.dateindex
.as_ref()
.unwrap()
.iter_at(starting_dateindex)
.try_for_each(|(index, ratio)| -> Result<()> {
if index < min_ratio_date {
self.ratio_p0_1.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p0_5.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p1.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p99.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p0_1
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p0_5
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p1
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p99
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p99_5
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p99_9
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_standard_deviation
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p1sd.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p2sd.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p3sd.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_m1sd.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_m2sd.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_m3sd.dateindex.forced_push_at(index, nan, exit)?;
self.ratio_p1sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p2sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_p3sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_m1sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_m2sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
self.ratio_m3sd
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, nan, exit)?;
} else {
let ratio = ratio.into_inner();
let pos = sorted.binary_search(&ratio).unwrap_or_else(|pos| pos);
sorted.insert(pos, ratio);
self.ratio_p0_1.dateindex.forced_push_at(
self.ratio_p0_1.dateindex.as_mut().unwrap().forced_push_at(
index,
get_percentile(&sorted, 0.001),
exit,
)?;
self.ratio_p0_5.dateindex.forced_push_at(
self.ratio_p0_5.dateindex.as_mut().unwrap().forced_push_at(
index,
get_percentile(&sorted, 0.005),
exit,
)?;
self.ratio_p1.dateindex.forced_push_at(
self.ratio_p1.dateindex.as_mut().unwrap().forced_push_at(
index,
get_percentile(&sorted, 0.01),
exit,
)?;
self.ratio_p99.dateindex.forced_push_at(
self.ratio_p99.dateindex.as_mut().unwrap().forced_push_at(
index,
get_percentile(&sorted, 0.99),
exit,
)?;
self.ratio_p99_5.dateindex.forced_push_at(
index,
get_percentile(&sorted, 0.995),
exit,
)?;
self.ratio_p99_9.dateindex.forced_push_at(
index,
get_percentile(&sorted, 0.999),
exit,
)?;
self.ratio_p99_5
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, get_percentile(&sorted, 0.995), exit)?;
self.ratio_p99_9
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, get_percentile(&sorted, 0.999), exit)?;
let avg = sma_iter.unwrap_get_inner(index);
@@ -562,26 +647,40 @@ impl ComputedRatioVecsFromDateIndex {
self.ratio_standard_deviation
.dateindex
.as_mut()
.unwrap()
.forced_push_at(index, sd, exit)?;
self.ratio_p1sd
.dateindex
.forced_push_at(index, avg + sd, exit)?;
self.ratio_p2sd
.dateindex
.forced_push_at(index, avg + 2 * sd, exit)?;
self.ratio_p3sd
.dateindex
.forced_push_at(index, avg + 3 * sd, exit)?;
self.ratio_m1sd
.dateindex
.forced_push_at(index, avg - sd, exit)?;
self.ratio_m2sd
.dateindex
.forced_push_at(index, avg - 2 * sd, exit)?;
self.ratio_m3sd
.dateindex
.forced_push_at(index, avg - 3 * sd, exit)?;
self.ratio_p1sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg + sd,
exit,
)?;
self.ratio_p2sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg + 2 * sd,
exit,
)?;
self.ratio_p3sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg + 3 * sd,
exit,
)?;
self.ratio_m1sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg - sd,
exit,
)?;
self.ratio_m2sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg - 2 * sd,
exit,
)?;
self.ratio_m3sd.dateindex.as_mut().unwrap().forced_push_at(
index,
avg - 3 * sd,
exit,
)?;
}
Ok(())
@@ -591,44 +690,96 @@ impl ComputedRatioVecsFromDateIndex {
.into_iter()
.try_for_each(|v| v.safe_flush(exit))?;
self.ratio_p99_9
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p99_5
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p99
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p1
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p0_5
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p0_1
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_standard_deviation
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p1sd
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p2sd
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p3sd
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_m1sd
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_m2sd
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_m3sd
.compute_rest(indexes, starting_indexes, exit)?;
self.ratio_p99_9.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p99_5.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p99.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p1.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p0_5.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p0_1.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_standard_deviation.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p1sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p2sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_p3sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_m1sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_m2sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
self.ratio_m3sd.compute_rest(
indexes,
starting_indexes,
exit,
None as Option<&EagerVec<_, _>>,
)?;
let date_to_price = date_to_price_opt.unwrap_or_else(|| unsafe {
std::mem::transmute(&self.price.as_ref().unwrap().dateindex)
});
self.ratio_p99_as_price.compute(
self.ratio_p99_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p99.dateindex.into_iter();
let mut iter = self.ratio_p99.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -641,13 +792,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p99_5_as_price.compute(
self.ratio_p99_5_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p99_5.dateindex.into_iter();
let mut iter = self.ratio_p99_5.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -660,13 +811,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p99_9_as_price.compute(
self.ratio_p99_9_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p99_9.dateindex.into_iter();
let mut iter = self.ratio_p99_9.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -679,13 +830,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p1_as_price.compute(
self.ratio_p1_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p1.dateindex.into_iter();
let mut iter = self.ratio_p1.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -698,13 +849,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p0_5_as_price.compute(
self.ratio_p0_5_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p0_5.dateindex.into_iter();
let mut iter = self.ratio_p0_5.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -717,13 +868,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p0_1_as_price.compute(
self.ratio_p0_1_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p0_1.dateindex.into_iter();
let mut iter = self.ratio_p0_1.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -736,13 +887,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p1sd_as_price.compute(
self.ratio_p1sd_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p1sd.dateindex.into_iter();
let mut iter = self.ratio_p1sd.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -755,13 +906,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p2sd_as_price.compute(
self.ratio_p2sd_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p2sd.dateindex.into_iter();
let mut iter = self.ratio_p2sd.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -774,13 +925,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_p3sd_as_price.compute(
self.ratio_p3sd_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_p3sd.dateindex.into_iter();
let mut iter = self.ratio_p3sd.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -793,13 +944,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_m1sd_as_price.compute(
self.ratio_m1sd_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_m1sd.dateindex.into_iter();
let mut iter = self.ratio_m1sd.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -812,13 +963,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_m2sd_as_price.compute(
self.ratio_m2sd_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_m2sd.dateindex.into_iter();
let mut iter = self.ratio_m2sd.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -831,13 +982,13 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_m3sd_as_price.compute(
self.ratio_m3sd_as_price.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut iter = self.ratio_m3sd.dateindex.into_iter();
let mut iter = self.ratio_m3sd.dateindex.as_ref().unwrap().into_iter();
vec.compute_transform(
starting_indexes.dateindex,
date_to_price,
@@ -850,17 +1001,22 @@ impl ComputedRatioVecsFromDateIndex {
},
)?;
self.ratio_zscore.compute(
self.ratio_zscore.compute_all(
indexer,
indexes,
starting_indexes,
exit,
|vec, _, _, starting_indexes, exit| {
let mut sma_iter = self.ratio_sma.dateindex.into_iter();
let mut sd_iter = self.ratio_standard_deviation.dateindex.into_iter();
let mut sma_iter = self.ratio_sma.dateindex.as_ref().unwrap().into_iter();
let mut sd_iter = self
.ratio_standard_deviation
.dateindex
.as_ref()
.unwrap()
.into_iter();
vec.compute_transform(
starting_indexes.dateindex,
&self.ratio.dateindex,
self.ratio.dateindex.as_ref().unwrap(),
|(i, ratio, ..)| {
let sma = sma_iter.unwrap_get_inner(i);
let sd = sd_iter.unwrap_get_inner(i);
@@ -876,19 +1032,19 @@ impl ComputedRatioVecsFromDateIndex {
fn mut_ratio_vecs(&mut self) -> Vec<&mut EagerVec<DateIndex, StoredF32>> {
vec![
&mut self.ratio_standard_deviation.dateindex,
&mut self.ratio_p99_9.dateindex,
&mut self.ratio_p99_5.dateindex,
&mut self.ratio_p99.dateindex,
&mut self.ratio_p1.dateindex,
&mut self.ratio_p0_5.dateindex,
&mut self.ratio_p0_1.dateindex,
&mut self.ratio_p1sd.dateindex,
&mut self.ratio_p2sd.dateindex,
&mut self.ratio_p3sd.dateindex,
&mut self.ratio_m1sd.dateindex,
&mut self.ratio_m2sd.dateindex,
&mut self.ratio_m3sd.dateindex,
self.ratio_standard_deviation.dateindex.as_mut().unwrap(),
self.ratio_p99_9.dateindex.as_mut().unwrap(),
self.ratio_p99_5.dateindex.as_mut().unwrap(),
self.ratio_p99.dateindex.as_mut().unwrap(),
self.ratio_p1.dateindex.as_mut().unwrap(),
self.ratio_p0_5.dateindex.as_mut().unwrap(),
self.ratio_p0_1.dateindex.as_mut().unwrap(),
self.ratio_p1sd.dateindex.as_mut().unwrap(),
self.ratio_p2sd.dateindex.as_mut().unwrap(),
self.ratio_p3sd.dateindex.as_mut().unwrap(),
self.ratio_m1sd.dateindex.as_mut().unwrap(),
self.ratio_m2sd.dateindex.as_mut().unwrap(),
self.ratio_m3sd.dateindex.as_mut().unwrap(),
]
}
crates/brk_computer/src/vecs/grouped/value_from_height.rs
+2 -2
View File
@@ -127,14 +127,14 @@ impl ComputedValueVecsFromHeight {
|v, _, _, starting_indexes, exit| {
v.compute_from_sats(
starting_indexes.height,
self.sats.height.as_ref().unwrap().as_ref(),
self.sats.height.as_ref().unwrap(),
exit,
)
},
)?;
}
let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap().as_ref();
let height_to_bitcoin = self.bitcoin.height.as_ref().unwrap();
let height_to_close = &fetched.as_ref().unwrap().chainindexes_to_close.height;
if let Some(dollars) = self.dollars.as_mut() {
crates/brk_computer/src/vecs/market.rs
+170 -37
View File
File diff suppressed because it is too large Load Diff
crates/brk_computer/src/vecs/mining.rs
+4 -2
View File
@@ -41,6 +41,7 @@ impl Vecs {
indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
path,
"difficultyepoch",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
@@ -48,6 +49,7 @@ impl Vecs {
indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
path,
"halvingepoch",
true,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
@@ -63,7 +65,7 @@ impl Vecs {
exit: &Exit,
) -> color_eyre::Result<()> {
let mut height_to_difficultyepoch_iter = indexes.height_to_difficultyepoch.into_iter();
self.indexes_to_difficultyepoch.compute(
self.indexes_to_difficultyepoch.compute_all(
indexer,
indexes,
starting_indexes,
@@ -87,7 +89,7 @@ impl Vecs {
)?;
let mut height_to_halvingepoch_iter = indexes.height_to_halvingepoch.into_iter();
self.indexes_to_halvingepoch.compute(
self.indexes_to_halvingepoch.compute_all(
indexer,
indexes,
starting_indexes,
crates/brk_computer/src/vecs/mod.rs
+3
View File
@@ -5,6 +5,7 @@ use brk_exit::Exit;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_vec::{AnyCollectableVec, Compressed, Computation};
use fjall::TransactionalKeyspace;
pub mod blocks;
pub mod constants;
@@ -40,6 +41,7 @@ impl Vecs {
fetch: bool,
computation: Computation,
compressed: Compressed,
keyspace: &TransactionalKeyspace,
) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
@@ -92,6 +94,7 @@ impl Vecs {
computation,
compressed,
fetched.as_ref(),
keyspace,
)?,
transactions: transactions::Vecs::forced_import(
path,
crates/brk_computer/src/vecs/stateful/cohort.rs
+499 -54
View File
@@ -5,20 +5,24 @@ use brk_core::{
};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_state::{CohortState, RealizedState};
use brk_vec::{AnyCollectableVec, AnyVec, Compressed, Computation, EagerVec, VecIterator};
use brk_state::CohortState;
use brk_vec::{
AnyCollectableVec, AnyVec, Compressed, Computation, EagerVec, StoredIndex, VecIterator,
};
use fjall::TransactionalKeyspace;
use crate::vecs::{
Indexes, fetched,
grouped::{
ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromHeight,
StorableVecGeneatorOptions,
ComputedRatioVecsFromDateIndex, ComputedValueVecsFromHeight, ComputedVecsFromDateIndex,
ComputedVecsFromHeight, StorableVecGeneatorOptions,
},
indexes,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
starting_height: Height,
pub state: CohortState,
@@ -34,6 +38,16 @@ pub struct Vecs {
pub height_to_adjusted_value_created: Option<EagerVec<Height, Dollars>>,
pub height_to_value_destroyed: Option<EagerVec<Height, Dollars>>,
pub height_to_adjusted_value_destroyed: Option<EagerVec<Height, Dollars>>,
pub height_to_supply_in_profit: Option<EagerVec<Height, Sats>>,
pub height_to_supply_in_loss: Option<EagerVec<Height, Sats>>,
pub height_to_supply_even: Option<EagerVec<Height, Sats>>,
pub height_to_unrealized_profit: Option<EagerVec<Height, Dollars>>,
pub height_to_unrealized_loss: Option<EagerVec<Height, Dollars>>,
pub dateindex_to_supply_in_profit: Option<EagerVec<DateIndex, Sats>>,
pub dateindex_to_supply_in_loss: Option<EagerVec<DateIndex, Sats>>,
pub dateindex_to_supply_even: Option<EagerVec<DateIndex, Sats>>,
pub dateindex_to_unrealized_profit: Option<EagerVec<DateIndex, Dollars>>,
pub dateindex_to_unrealized_loss: Option<EagerVec<DateIndex, Dollars>>,
pub dateindex_to_adjusted_spent_output_profit_ratio: Option<EagerVec<DateIndex, StoredF32>>,
pub dateindex_to_realized_cap_30d_change: Option<EagerVec<DateIndex, Dollars>>,
@@ -53,9 +67,15 @@ pub struct Vecs {
pub indexes_to_utxo_count: ComputedVecsFromHeight<StoredUsize>,
pub indexes_to_value_created: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_value_destroyed: Option<ComputedVecsFromHeight<Dollars>>,
pub indexes_to_supply_in_profit: Option<ComputedVecsFromDateIndex<Sats>>,
pub indexes_to_supply_in_loss: Option<ComputedVecsFromDateIndex<Sats>>,
pub indexes_to_supply_even: Option<ComputedVecsFromDateIndex<Sats>>,
pub indexes_to_unrealized_profit: Option<ComputedVecsFromDateIndex<Dollars>>,
pub indexes_to_unrealized_loss: Option<ComputedVecsFromDateIndex<Dollars>>,
}
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
path: &Path,
cohort_name: Option<&str>,
@@ -63,6 +83,8 @@ impl Vecs {
compressed: Compressed,
version: Version,
fetched: Option<&fetched::Vecs>,
keyspace: &TransactionalKeyspace,
stores_path: &Path,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
@@ -72,15 +94,163 @@ impl Vecs {
let suffix = |s: &str| cohort_name.map_or(s.to_string(), |name| format!("{name}_{s}"));
let mut state = CohortState::default();
if compute_dollars {
state.realized = Some(RealizedState::NAN);
}
let state = CohortState::default_and_import(
keyspace,
stores_path,
cohort_name.unwrap_or_default(),
version,
compute_dollars,
)?;
Ok(Self {
starting_height: Height::ZERO,
state,
height_to_supply_in_profit: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("supply_in_profit"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
dateindex_to_supply_in_profit: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("supply_in_profit"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_supply_in_profit: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&suffix("supply_in_profit"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_supply_even: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("supply_even"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
dateindex_to_supply_even: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("supply_even"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_supply_even: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&suffix("supply_even"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_supply_in_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("supply_in_loss"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
dateindex_to_supply_in_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("supply_in_loss"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_supply_in_loss: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&suffix("supply_in_loss"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_unrealized_profit: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("unrealized_profit"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
dateindex_to_unrealized_profit: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("unrealized_profit"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_unrealized_profit: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&suffix("unrealized_profit"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_unrealized_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("unrealized_loss"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
dateindex_to_unrealized_loss: compute_dollars.then(|| {
EagerVec::forced_import(
path,
&suffix("unrealized_loss"),
version + VERSION + Version::ZERO,
compressed,
)
.unwrap()
}),
indexes_to_unrealized_loss: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
path,
&suffix("unrealized_loss"),
false,
version + VERSION + Version::ZERO,
compressed,
StorableVecGeneatorOptions::default().add_last(),
)
.unwrap()
}),
height_to_realized_cap: compute_dollars.then(|| {
EagerVec::forced_import(
path,
@@ -346,6 +516,10 @@ impl Vecs {
pub fn starting_height(&self) -> Height {
[
self.state
.price_to_amount
.height()
.map_or(usize::MAX, |h| h.incremented().unwrap_to_usize()),
self.height_to_supply.len(),
self.height_to_utxo_count.len(),
self.height_to_realized_cap
@@ -369,6 +543,21 @@ impl Vecs {
self.height_to_adjusted_value_destroyed
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_supply_in_profit
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_supply_in_loss
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_supply_even
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_unrealized_profit
.as_ref()
.map_or(usize::MAX, |v| v.len()),
self.height_to_unrealized_loss
.as_ref()
.map_or(usize::MAX, |v| v.len()),
]
.into_iter()
.map(Height::from)
@@ -483,6 +672,113 @@ impl Vecs {
.validate_computed_version_or_reset_file(
base_version + height_to_adjusted_value_destroyed_inner_version,
)?;
let height_to_supply_in_profit_inner_version = self
.height_to_supply_in_profit
.as_ref()
.unwrap()
.inner_version();
self.height_to_supply_in_profit
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_supply_in_profit_inner_version,
)?;
let height_to_supply_in_loss_inner_version = self
.height_to_supply_in_loss
.as_ref()
.unwrap()
.inner_version();
self.height_to_supply_in_loss
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_supply_in_loss_inner_version,
)?;
let height_to_supply_even_inner_version =
self.height_to_supply_even.as_ref().unwrap().inner_version();
self.height_to_supply_even
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_supply_even_inner_version,
)?;
let height_to_unrealized_profit_inner_version = self
.height_to_unrealized_profit
.as_ref()
.unwrap()
.inner_version();
self.height_to_unrealized_profit
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_unrealized_profit_inner_version,
)?;
let height_to_unrealized_loss_inner_version = self
.height_to_unrealized_loss
.as_ref()
.unwrap()
.inner_version();
self.height_to_unrealized_loss
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + height_to_unrealized_loss_inner_version,
)?;
let dateindex_to_supply_in_profit_inner_version = self
.dateindex_to_supply_in_profit
.as_ref()
.unwrap()
.inner_version();
self.dateindex_to_supply_in_profit
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + dateindex_to_supply_in_profit_inner_version,
)?;
let dateindex_to_supply_in_loss_inner_version = self
.dateindex_to_supply_in_loss
.as_ref()
.unwrap()
.inner_version();
self.dateindex_to_supply_in_loss
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + dateindex_to_supply_in_loss_inner_version,
)?;
let dateindex_to_supply_even_inner_version = self
.dateindex_to_supply_even
.as_ref()
.unwrap()
.inner_version();
self.dateindex_to_supply_even
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + dateindex_to_supply_even_inner_version,
)?;
let dateindex_to_unrealized_profit_inner_version = self
.dateindex_to_unrealized_profit
.as_ref()
.unwrap()
.inner_version();
self.dateindex_to_unrealized_profit
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + dateindex_to_unrealized_profit_inner_version,
)?;
let dateindex_to_unrealized_loss_inner_version = self
.dateindex_to_unrealized_loss
.as_ref()
.unwrap()
.inner_version();
self.dateindex_to_unrealized_loss
.as_mut()
.unwrap()
.validate_computed_version_or_reset_file(
base_version + dateindex_to_unrealized_loss_inner_version,
)?;
}
Ok(())
@@ -538,7 +834,70 @@ impl Vecs {
Ok(())
}
pub fn safe_flush_height_vecs(&mut self, exit: &Exit) -> Result<()> {
pub fn compute_then_force_push_unrealized_states(
&mut self,
height: Height,
height_price: Option<Dollars>,
dateindex: Option<DateIndex>,
date_price: Option<Option<Dollars>>,
exit: &Exit,
) -> Result<()> {
if let Some(height_price) = height_price {
let (height_unrealized_state, date_unrealized_state) = self
.state
.compute_unrealized_states(height_price, date_price.unwrap());
self.height_to_supply_even
.as_mut()
.unwrap()
.forced_push_at(height, height_unrealized_state.supply_even, exit)?;
self.height_to_supply_in_profit
.as_mut()
.unwrap()
.forced_push_at(height, height_unrealized_state.supply_in_profit, exit)?;
self.height_to_supply_in_loss
.as_mut()
.unwrap()
.forced_push_at(height, height_unrealized_state.supply_in_loss, exit)?;
self.height_to_unrealized_profit
.as_mut()
.unwrap()
.forced_push_at(height, height_unrealized_state.unrealized_profit, exit)?;
self.height_to_unrealized_loss
.as_mut()
.unwrap()
.forced_push_at(height, height_unrealized_state.unrealized_loss, exit)?;
if let Some(date_unrealized_state) = date_unrealized_state {
let dateindex = dateindex.unwrap();
self.dateindex_to_supply_even
.as_mut()
.unwrap()
.forced_push_at(dateindex, date_unrealized_state.supply_even, exit)?;
self.dateindex_to_supply_in_profit
.as_mut()
.unwrap()
.forced_push_at(dateindex, date_unrealized_state.supply_in_profit, exit)?;
self.dateindex_to_supply_in_loss
.as_mut()
.unwrap()
.forced_push_at(dateindex, date_unrealized_state.supply_in_loss, exit)?;
self.dateindex_to_unrealized_profit
.as_mut()
.unwrap()
.forced_push_at(dateindex, date_unrealized_state.unrealized_profit, exit)?;
self.dateindex_to_unrealized_loss
.as_mut()
.unwrap()
.forced_push_at(dateindex, date_unrealized_state.unrealized_loss, exit)?;
}
}
Ok(())
}
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.height_to_supply.safe_flush(exit)?;
self.height_to_utxo_count.safe_flush(exit)?;
@@ -569,8 +928,50 @@ impl Vecs {
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_supply_in_profit
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_supply_in_loss
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_supply_even
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_unrealized_profit
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.height_to_unrealized_loss
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.dateindex_to_supply_in_profit
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.dateindex_to_supply_in_loss
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.dateindex_to_supply_even
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.dateindex_to_unrealized_profit
.as_mut()
.unwrap()
.safe_flush(exit)?;
self.dateindex_to_unrealized_loss
.as_mut()
.unwrap()
.safe_flush(exit)?;
}
self.state.commit(height)?;
Ok(())
}
@@ -618,7 +1019,7 @@ impl Vecs {
vec.compute_divide(
starting_indexes.height,
self.height_to_realized_cap.as_ref().unwrap(),
&**self.indexes_to_supply.bitcoin.height.as_ref().unwrap(),
self.indexes_to_supply.bitcoin.height.as_ref().unwrap(),
exit,
)
},
@@ -823,6 +1224,49 @@ impl Vecs {
.unwrap_last(),
exit,
)?;
self.indexes_to_supply_in_profit
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.dateindex_to_supply_in_profit.as_ref().unwrap()),
)?;
self.indexes_to_supply_in_loss
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.dateindex_to_supply_in_loss.as_ref().unwrap()),
)?;
self.indexes_to_supply_even.as_mut().unwrap().compute_rest(
indexes,
starting_indexes,
exit,
Some(self.dateindex_to_supply_even.as_ref().unwrap()),
)?;
self.indexes_to_unrealized_profit
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.dateindex_to_unrealized_profit.as_ref().unwrap()),
)?;
self.indexes_to_unrealized_loss
.as_mut()
.unwrap()
.compute_rest(
indexes,
starting_indexes,
exit,
Some(self.dateindex_to_unrealized_loss.as_ref().unwrap()),
)?;
}
Ok(())
@@ -905,53 +1349,54 @@ impl Vecs {
self.dateindex_to_sell_side_risk_ratio
.as_ref()
.map_or(vec![], |v| vec![v]),
self.height_to_supply_in_profit
.as_ref()
.map_or(vec![], |v| vec![v]),
self.height_to_supply_in_loss
.as_ref()
.map_or(vec![], |v| vec![v]),
self.height_to_supply_even
.as_ref()
.map_or(vec![], |v| vec![v]),
self.height_to_unrealized_profit
.as_ref()
.map_or(vec![], |v| vec![v]),
self.height_to_unrealized_loss
.as_ref()
.map_or(vec![], |v| vec![v]),
self.dateindex_to_supply_in_profit
.as_ref()
.map_or(vec![], |v| vec![v]),
self.dateindex_to_supply_in_loss
.as_ref()
.map_or(vec![], |v| vec![v]),
self.dateindex_to_supply_even
.as_ref()
.map_or(vec![], |v| vec![v]),
self.dateindex_to_unrealized_profit
.as_ref()
.map_or(vec![], |v| vec![v]),
self.dateindex_to_unrealized_loss
.as_ref()
.map_or(vec![], |v| vec![v]),
self.indexes_to_supply_in_profit
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_supply_in_loss
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_supply_even
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_unrealized_profit
.as_ref()
.map_or(vec![], |v| v.vecs()),
self.indexes_to_unrealized_loss
.as_ref()
.map_or(vec![], |v| v.vecs()),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
}
}
impl Clone for Vecs {
fn clone(&self) -> Self {
Self {
starting_height: self.starting_height,
state: CohortState::default(),
height_to_realized_cap: self.height_to_realized_cap.clone(),
height_to_supply: self.height_to_supply.clone(),
height_to_utxo_count: self.height_to_utxo_count.clone(),
height_to_realized_profit: self.height_to_realized_profit.clone(),
height_to_realized_loss: self.height_to_realized_loss.clone(),
height_to_value_created: self.height_to_value_created.clone(),
height_to_adjusted_value_created: self.height_to_adjusted_value_created.clone(),
height_to_value_destroyed: self.height_to_value_destroyed.clone(),
height_to_adjusted_value_destroyed: self.height_to_adjusted_value_destroyed.clone(),
indexes_to_supply: self.indexes_to_supply.clone(),
indexes_to_utxo_count: self.indexes_to_utxo_count.clone(),
indexes_to_realized_cap: self.indexes_to_realized_cap.clone(),
indexes_to_realized_profit: self.indexes_to_realized_profit.clone(),
indexes_to_realized_loss: self.indexes_to_realized_loss.clone(),
indexes_to_negative_realized_loss: self.indexes_to_negative_realized_loss.clone(),
indexes_to_value_created: self.indexes_to_value_created.clone(),
indexes_to_adjusted_value_created: self.indexes_to_adjusted_value_created.clone(),
indexes_to_value_destroyed: self.indexes_to_value_destroyed.clone(),
indexes_to_adjusted_value_destroyed: self.indexes_to_adjusted_value_destroyed.clone(),
dateindex_to_realized_cap_30d_change: self.dateindex_to_realized_cap_30d_change.clone(),
indexes_to_realized_value: self.indexes_to_realized_value.clone(),
indexes_to_net_realized_profit_and_loss: self
.indexes_to_net_realized_profit_and_loss
.clone(),
indexes_to_realized_price: self.indexes_to_realized_price.clone(),
dateindex_to_sell_side_risk_ratio: self.dateindex_to_sell_side_risk_ratio.clone(),
indexes_to_realized_price_extra: self.indexes_to_realized_price_extra.clone(),
dateindex_to_spent_output_profit_ratio: self
.dateindex_to_spent_output_profit_ratio
.clone(),
dateindex_to_adjusted_spent_output_profit_ratio: self
.dateindex_to_adjusted_spent_output_profit_ratio
.clone(),
}
}
}
crates/brk_computer/src/vecs/stateful/mod.rs
+234 -25
View File
@@ -1,12 +1,13 @@
use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread};
use brk_core::{Height, InputIndex, OutputIndex, OutputType, Sats, Version};
use brk_core::{DateIndex, Height, InputIndex, OutputIndex, OutputType, Sats, Version};
use brk_exit::Exit;
use brk_indexer::Indexer;
use brk_vec::{
AnyCollectableVec, AnyVec, BaseVecIterator, CollectableVec, Compressed, Computation, EagerVec,
GenericStoredVec, StoredIndex, StoredVec, UnsafeSlice, VecIterator,
};
use fjall::TransactionalKeyspace;
use log::info;
use outputs::OutputCohorts;
use rayon::prelude::*;
@@ -47,12 +48,15 @@ impl Vecs {
_computation: Computation,
compressed: Compressed,
fetched: Option<&fetched::Vecs>,
keyspace: &TransactionalKeyspace,
) -> color_eyre::Result<Self> {
let compute_dollars = fetched.is_some();
let mut states_path = path.to_owned();
states_path.pop();
states_path = states_path.join("states");
let mut root_path = path.to_owned();
root_path.pop();
let states_path = root_path.join("states");
root_path.pop();
let stores_path = root_path.join("stores");
Ok(Self {
chain_state: StoredVec::forced_import(
@@ -101,6 +105,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
by_term: OutputsByTerm {
short: cohort::Vecs::forced_import(
@@ -110,6 +116,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
long: cohort::Vecs::forced_import(
path,
@@ -118,6 +126,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
by_up_to: OutputsByUpTo {
@@ -128,6 +138,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1w: cohort::Vecs::forced_import(
path,
@@ -136,6 +148,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1m: cohort::Vecs::forced_import(
path,
@@ -144,6 +158,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_2m: cohort::Vecs::forced_import(
path,
@@ -152,6 +168,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3m: cohort::Vecs::forced_import(
path,
@@ -160,6 +178,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_4m: cohort::Vecs::forced_import(
path,
@@ -168,6 +188,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_5m: cohort::Vecs::forced_import(
path,
@@ -176,6 +198,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_6m: cohort::Vecs::forced_import(
path,
@@ -184,6 +208,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1y: cohort::Vecs::forced_import(
path,
@@ -192,6 +218,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_2y: cohort::Vecs::forced_import(
path,
@@ -200,6 +228,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3y: cohort::Vecs::forced_import(
path,
@@ -208,6 +238,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_4y: cohort::Vecs::forced_import(
path,
@@ -216,6 +248,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_5y: cohort::Vecs::forced_import(
path,
@@ -224,6 +258,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_6y: cohort::Vecs::forced_import(
path,
@@ -232,6 +268,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_7y: cohort::Vecs::forced_import(
path,
@@ -240,6 +278,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_8y: cohort::Vecs::forced_import(
path,
@@ -248,6 +288,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_10y: cohort::Vecs::forced_import(
path,
@@ -256,6 +298,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_15y: cohort::Vecs::forced_import(
path,
@@ -264,6 +308,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
by_from: OutputsByFrom {
@@ -274,6 +320,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1w: cohort::Vecs::forced_import(
path,
@@ -282,6 +330,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1m: cohort::Vecs::forced_import(
path,
@@ -290,6 +340,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_2m: cohort::Vecs::forced_import(
path,
@@ -298,6 +350,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3m: cohort::Vecs::forced_import(
path,
@@ -306,6 +360,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_4m: cohort::Vecs::forced_import(
path,
@@ -314,6 +370,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_5m: cohort::Vecs::forced_import(
path,
@@ -322,6 +380,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_6m: cohort::Vecs::forced_import(
path,
@@ -330,6 +390,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1y: cohort::Vecs::forced_import(
path,
@@ -338,6 +400,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_2y: cohort::Vecs::forced_import(
path,
@@ -346,6 +410,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3y: cohort::Vecs::forced_import(
path,
@@ -354,6 +420,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_4y: cohort::Vecs::forced_import(
path,
@@ -362,6 +430,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_5y: cohort::Vecs::forced_import(
path,
@@ -370,6 +440,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_6y: cohort::Vecs::forced_import(
path,
@@ -378,6 +450,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_7y: cohort::Vecs::forced_import(
path,
@@ -386,6 +460,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_8y: cohort::Vecs::forced_import(
path,
@@ -394,6 +470,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_10y: cohort::Vecs::forced_import(
path,
@@ -402,6 +480,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_15y: cohort::Vecs::forced_import(
path,
@@ -410,6 +490,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
by_range: OutputsByRange {
@@ -420,6 +502,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1w_to_1m: cohort::Vecs::forced_import(
path,
@@ -428,6 +512,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1m_to_3m: cohort::Vecs::forced_import(
path,
@@ -436,6 +522,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3m_to_6m: cohort::Vecs::forced_import(
path,
@@ -444,6 +532,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_6m_to_1y: cohort::Vecs::forced_import(
path,
@@ -452,6 +542,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1y_to_2y: cohort::Vecs::forced_import(
path,
@@ -460,6 +552,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_2y_to_3y: cohort::Vecs::forced_import(
path,
@@ -468,6 +562,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3y_to_4y: cohort::Vecs::forced_import(
path,
@@ -476,6 +572,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_4y_to_5y: cohort::Vecs::forced_import(
path,
@@ -484,6 +582,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_5y_to_7y: cohort::Vecs::forced_import(
path,
@@ -492,6 +592,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_7y_to_10y: cohort::Vecs::forced_import(
path,
@@ -500,6 +602,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_10y_to_15y: cohort::Vecs::forced_import(
path,
@@ -508,6 +612,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
by_epoch: OutputsByEpoch {
@@ -518,6 +624,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_1: cohort::Vecs::forced_import(
path,
@@ -526,6 +634,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_2: cohort::Vecs::forced_import(
path,
@@ -534,6 +644,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_3: cohort::Vecs::forced_import(
path,
@@ -542,6 +654,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
_4: cohort::Vecs::forced_import(
path,
@@ -550,6 +664,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
by_size: OutputsBySize {
@@ -560,6 +676,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_1sat_to_10sats: cohort::Vecs::forced_import(
path,
@@ -568,6 +686,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_10sats_to_100sats: cohort::Vecs::forced_import(
path,
@@ -576,6 +696,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_100sats_to_1_000sats: cohort::Vecs::forced_import(
path,
@@ -584,6 +706,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_1_000sats_to_10_000sats: cohort::Vecs::forced_import(
path,
@@ -592,6 +716,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_10_000sats_to_100_000sats: cohort::Vecs::forced_import(
path,
@@ -600,6 +726,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_100_000sats_to_1_000_000sats: cohort::Vecs::forced_import(
path,
@@ -608,6 +736,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_1_000_000sats_to_10_000_000sats: cohort::Vecs::forced_import(
path,
@@ -616,6 +746,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_10_000_000sats_to_1btc: cohort::Vecs::forced_import(
path,
@@ -624,6 +756,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_1btc_to_10btc: cohort::Vecs::forced_import(
path,
@@ -632,6 +766,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_10btc_to_100btc: cohort::Vecs::forced_import(
path,
@@ -640,6 +776,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_100btc_to_1_000btc: cohort::Vecs::forced_import(
path,
@@ -648,6 +786,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_1_000btc_to_10_000btc: cohort::Vecs::forced_import(
path,
@@ -656,6 +796,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_10_000btc_to_100_000btc: cohort::Vecs::forced_import(
path,
@@ -664,6 +806,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
from_100_000btc: cohort::Vecs::forced_import(
path,
@@ -672,6 +816,8 @@ impl Vecs {
compressed,
version + BYSIZE_VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
// by_value: OutputsByValue {
@@ -775,6 +921,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2pk33: cohort::Vecs::forced_import(
path,
@@ -783,6 +931,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2pkh: cohort::Vecs::forced_import(
path,
@@ -791,6 +941,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2ms: cohort::Vecs::forced_import(
path,
@@ -799,6 +951,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2sh: cohort::Vecs::forced_import(
path,
@@ -807,6 +961,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
// opreturn: cohort::Vecs::forced_import(
// path,
@@ -815,6 +971,7 @@ impl Vecs {
// compressed,
// VERSION + Version::ZERO,
// fetched,
// keyspace
// )?,
p2wpkh: cohort::Vecs::forced_import(
path,
@@ -823,6 +980,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2wsh: cohort::Vecs::forced_import(
path,
@@ -831,6 +990,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2tr: cohort::Vecs::forced_import(
path,
@@ -839,6 +1000,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
p2a: cohort::Vecs::forced_import(
path,
@@ -847,6 +1010,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
empty: cohort::Vecs::forced_import(
path,
@@ -855,6 +1020,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
unknown: cohort::Vecs::forced_import(
path,
@@ -863,6 +1030,8 @@ impl Vecs {
compressed,
version + VERSION + Version::ZERO,
fetched,
keyspace,
&stores_path,
)?,
},
})
@@ -896,11 +1065,16 @@ impl Vecs {
.sats
.height
.as_ref()
.unwrap()
.as_ref();
let height_to_close = &fetched
.unwrap();
let height_to_close = fetched
.as_ref()
.map(|fetched| &fetched.chainindexes_to_close.height);
let dateindex_to_close = fetched
.as_ref()
.map(|fetched| fetched.timeindexes_to_close.dateindex.as_ref().unwrap());
let height_to_date_fixed = &indexes.height_to_date_fixed;
let dateindex_to_first_height = &indexes.dateindex_to_first_height;
let dateindex_to_height_count = &indexes.dateindex_to_height_count;
let inputindex_to_outputindex_mmap = inputindex_to_outputindex.mmap().load();
let outputindex_to_value_mmap = outputindex_to_value.mmap().load();
@@ -917,6 +1091,10 @@ impl Vecs {
// let mut outputindex_to_outputtype_iter_2 = outputindex_to_outputtype.into_iter();
let mut height_to_unclaimed_rewards_iter = height_to_unclaimed_rewards.into_iter();
let mut height_to_timestamp_fixed_iter = height_to_timestamp_fixed.into_iter();
let mut dateindex_to_close_iter = dateindex_to_close.as_ref().map(|v| v.into_iter());
let mut height_to_date_fixed_iter = height_to_date_fixed.into_iter();
let mut dateindex_to_first_height_iter = dateindex_to_first_height.into_iter();
let mut dateindex_to_height_count_iter = dateindex_to_height_count.into_iter();
let mut flat_vecs_ = self.utxos_vecs.as_mut_vec();
@@ -934,7 +1112,13 @@ impl Vecs {
+ height_to_unclaimed_rewards.version()
+ height_to_close
.as_ref()
.map_or(Version::ZERO, |v| v.version());
.map_or(Version::ZERO, |v| v.version())
+ dateindex_to_close
.as_ref()
.map_or(Version::ZERO, |v| v.version())
+ height_to_date_fixed.version()
+ dateindex_to_first_height.version()
+ dateindex_to_height_count.version();
flat_vecs_
.iter_mut()
@@ -1047,10 +1231,6 @@ impl Vecs {
let output_count = height_to_output_count_iter.unwrap_get_inner(height);
let input_count = height_to_input_count_iter.unwrap_get_inner(height);
// let sent_state = SentState::default();
// let received_state = ReceivedState::default();
// let realized_state = RealizedState::default();
let (mut height_to_sent, mut received) = thread::scope(|s| {
if chain_state_starting_height <= height {
s.spawn(|| {
@@ -1093,15 +1273,11 @@ impl Vecs {
.unwrap()
.into_inner();
// let input_height = *cached_txindex_to_height
// .entry(input_txindex)
// .or_insert_with(|| {
let height = txindex_to_height
.get_or_read(input_txindex, &txindex_to_height_mmap)
.unwrap()
.unwrap()
.into_inner();
// });
(height, value, input_type)
})
@@ -1204,8 +1380,9 @@ impl Vecs {
panic!("temp, just making sure")
}
self.utxos_vecs
.as_mut_vec()
let mut utxos_vecs = self.utxos_vecs.as_mut_vec();
utxos_vecs
.iter_mut()
.try_for_each(|(_, v)| v.forced_pushed_at(height, exit))?;
@@ -1218,19 +1395,48 @@ impl Vecs {
self.height_to_opreturn_supply
.forced_push_at(height, opreturn_supply, exit)?;
let date = height_to_date_fixed_iter.unwrap_get_inner(height);
let dateindex = DateIndex::try_from(date).unwrap();
let date_first_height = dateindex_to_first_height_iter.unwrap_get_inner(dateindex);
let date_height_count = dateindex_to_height_count_iter.unwrap_get_inner(dateindex);
let is_date_last_height =
date_first_height + Height::from(*date_height_count) == height;
let date_price = dateindex_to_close_iter
.as_mut()
.map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex)));
utxos_vecs.par_iter_mut().try_for_each(|(_, v)| {
v.compute_then_force_push_unrealized_states(
height,
price,
is_date_last_height.then_some(dateindex),
date_price,
exit,
)
})?;
if height != Height::ZERO && height.unwrap_to_usize() % 100_000 == 0 {
info!("Flushing...");
utxos_vecs
.par_iter_mut()
.try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?;
self.height_to_unspendable_supply.safe_flush(exit)?;
self.height_to_opreturn_supply.safe_flush(exit)?;
}
Ok(())
})?;
exit.block();
let mut flat_vecs_ = self.utxos_vecs.as_mut_vec();
info!("Flushing...");
// Flush rest of values
flat_vecs_
self.utxos_vecs
.as_mut_vec()
.par_iter_mut()
.try_for_each(|(_, v)| v.safe_flush_height_vecs(exit))?;
.try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))?;
self.height_to_unspendable_supply.safe_flush(exit)?;
self.height_to_opreturn_supply.safe_flush(exit)?;
@@ -1246,9 +1452,12 @@ impl Vecs {
info!("Computing rest...");
// Compute other vecs from height vecs
flat_vecs_.par_iter_mut().try_for_each(|(_, v)| {
v.compute_rest(indexer, indexes, fetched, starting_indexes, exit)
})?;
self.utxos_vecs
.as_mut_vec()
.par_iter_mut()
.try_for_each(|(_, v)| {
v.compute_rest(indexer, indexes, fetched, starting_indexes, exit)
})?;
self.indexes_to_unspendable_supply.compute_rest(
indexer,
indexes,
crates/brk_computer/src/vecs/transactions.rs
+2 -12
View File
@@ -864,12 +864,7 @@ impl Vecs {
self.indexes_to_fee.sats.height.unwrap_sum().iter();
vec.compute_transform(
starting_indexes.height,
self.indexes_to_coinbase
.sats
.height
.as_ref()
.unwrap()
.as_ref(),
self.indexes_to_coinbase.sats.height.as_ref().unwrap(),
|(height, coinbase, ..)| {
let fees = indexes_to_fee_sum_iter.unwrap_get_inner(height);
(height, coinbase.checked_sub(fees).unwrap())
@@ -888,12 +883,7 @@ impl Vecs {
|vec, _, _, starting_indexes, exit| {
vec.compute_transform(
starting_indexes.height,
self.indexes_to_subsidy
.sats
.height
.as_ref()
.unwrap()
.as_ref(),
self.indexes_to_subsidy.sats.height.as_ref().unwrap(),
|(height, subsidy, ..)| {
let halving = HalvingEpoch::from(height);
let expected =
crates/brk_core/Cargo.toml
+2
View File
@@ -11,6 +11,7 @@ bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
byteview = { workspace = true }
derive_deref = { workspace = true }
fjall = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
rapidhash = "1.4.0"
@@ -19,6 +20,7 @@ serde = { workspace = true }
serde_bytes = "0.11.17"
serde_derive = { workspace = true }
serde_json = { workspace = true }
thiserror = "2.0.12"
zerocopy = { workspace = true }
zerocopy-derive = { workspace = true }
crates/brk_core/src/enums/error.rs
+30 -17
View File
@@ -1,20 +1,24 @@
use std::{
fmt::{self, Debug},
io,
time::SystemTimeError,
io, result, time,
};
use crate::Version;
pub type Result<T, E = Error> = std::result::Result<T, E>;
pub type Result<T, E = Error> = result::Result<T, E>;
#[derive(Debug)]
pub enum Error {
IO(io::Error),
SerdeJson(serde_json::Error),
Jiff(jiff::Error),
Fjall(fjall::Error),
SystemTimeError(time::SystemTimeError),
ZeroCopyError,
WrongEndian,
DifferentVersion { found: Version, expected: Version },
MmapsVecIsTooSmall,
IO(io::Error),
ZeroCopyError,
IndexTooHigh,
EmptyVec,
IndexTooLow,
@@ -24,17 +28,16 @@ pub enum Error {
UnsupportedUnflushedState,
RangeFromAfterTo(usize, usize),
DifferentCompressionMode,
SystemTimeError,
ToSerdeJsonValueError(serde_json::Error),
Jiff(jiff::Error),
WrongLength,
WrongAddressType,
UnindexableDate,
String(&'static str),
}
impl From<SystemTimeError> for Error {
fn from(_: SystemTimeError) -> Self {
Self::SystemTimeError
impl From<time::SystemTimeError> for Error {
fn from(value: time::SystemTimeError) -> Self {
Self::SystemTimeError(value)
}
}
@@ -50,6 +53,12 @@ impl From<jiff::Error> for Error {
}
}
impl From<fjall::Error> for Error {
fn from(value: fjall::Error) -> Self {
Self::Fjall(value)
}
}
impl<A, B, C> From<zerocopy::error::ConvertError<A, B, C>> for Error {
fn from(_: zerocopy::error::ConvertError<A, B, C>) -> Self {
Self::ZeroCopyError
@@ -64,13 +73,20 @@ impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
impl From<serde_json::Error> for Error {
fn from(error: serde_json::Error) -> Self {
Self::ToSerdeJsonValueError(error)
Self::SerdeJson(error)
}
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::IO(error) => Debug::fmt(&error, f),
Error::SystemTimeError(error) => Debug::fmt(&error, f),
Error::SerdeJson(error) => Debug::fmt(&error, f),
Error::Jiff(error) => Debug::fmt(&error, f),
Error::Fjall(error) => Debug::fmt(&error, f),
Error::ZeroCopyError => write!(f, "ZeroCopy error"),
Error::WrongEndian => write!(f, "Wrong endian"),
Error::DifferentVersion { found, expected } => {
write!(
@@ -79,7 +95,6 @@ impl fmt::Display for Error {
)
}
Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
Error::IO(error) => Debug::fmt(&error, f),
Error::IndexTooHigh => write!(f, "Index too high"),
Error::IndexTooLow => write!(f, "Index too low"),
Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
@@ -91,19 +106,17 @@ impl fmt::Display for Error {
"Unsupported unflush state, please flush before using this function"
)
}
Error::ZeroCopyError => write!(f, "Zero copy convert error"),
Error::SystemTimeError => write!(f, "SystemTimeError"),
Error::RangeFromAfterTo(from, to) => write!(f, "Range, from {from} is after to {to}"),
Error::DifferentCompressionMode => write!(f, "Different compression mode chosen"),
Error::EmptyVec => write!(f, "The Vec is empty, maybe wait for a bit"),
Error::ToSerdeJsonValueError(error) => Debug::fmt(&error, f),
Error::Jiff(error) => Debug::fmt(&error, f),
Error::WrongLength => write!(f, "Wrong length"),
Error::WrongAddressType => write!(f, "Wrong address type"),
Error::UnindexableDate => write!(
f,
"Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"
),
Error::String(s) => write!(f, "{s}"),
}
}
}
crates/brk_core/src/structs/dollars.rs
+30 -3
View File
@@ -4,11 +4,12 @@ use std::{
ops::{Add, AddAssign, Div, Mul},
};
use byteview::ByteView;
use derive_deref::Deref;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
use crate::{CheckedSub, Error, copy_first_8bytes};
use super::{Bitcoin, Cents, Close, Sats, StoredF32, StoredF64};
@@ -133,12 +134,18 @@ impl Div<Bitcoin> for Dollars {
impl Mul<Bitcoin> for Dollars {
type Output = Self;
fn mul(self, rhs: Bitcoin) -> Self::Output {
self * Sats::from(rhs)
}
}
impl Mul<Sats> for Dollars {
type Output = Self;
fn mul(self, rhs: Sats) -> Self::Output {
if self.is_nan() {
self
} else {
Self::from(Cents::from(
u128::from(Sats::from(rhs)) * u128::from(Cents::from(self))
/ u128::from(Sats::ONE_BTC),
u128::from(rhs) * u128::from(Cents::from(self)) / u128::from(Sats::ONE_BTC),
))
}
}
@@ -248,3 +255,23 @@ impl Ord for Dollars {
}
}
}
impl TryFrom<ByteView> for Dollars {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
let bytes = copy_first_8bytes(&value)?;
Ok(Self::from(f64::from_be_bytes(bytes)))
}
}
impl From<Dollars> for ByteView {
fn from(value: Dollars) -> Self {
Self::from(&value)
}
}
impl From<&Dollars> for ByteView {
fn from(value: &Dollars) -> Self {
Self::new(&value.to_be_bytes())
}
}
crates/brk_core/src/structs/sats.rs
+22 -1
View File
@@ -4,10 +4,11 @@ use std::{
};
use bitcoin::Amount;
use byteview::ByteView;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::CheckedSub;
use crate::{CheckedSub, Error, copy_first_8bytes};
use super::{Bitcoin, Cents, Dollars, Height};
@@ -187,3 +188,23 @@ impl From<Sats> for u128 {
value.0 as u128
}
}
impl TryFrom<ByteView> for Sats {
type Error = Error;
fn try_from(value: ByteView) -> Result<Self, Self::Error> {
let bytes = copy_first_8bytes(&value)?;
Ok(Self::from(u64::from_be_bytes(bytes)))
}
}
impl From<&Sats> for ByteView {
fn from(value: &Sats) -> Self {
Self::new(&value.0.to_be_bytes())
}
}
impl From<Sats> for ByteView {
fn from(value: Sats) -> Self {
Self::from(&value)
}
}
crates/brk_core/src/utils/bytes.rs
+4 -2
View File
@@ -1,9 +1,11 @@
use crate::{Error, Result};
#[allow(clippy::result_unit_err)]
pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8], ()> {
pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8]> {
let mut buf: [u8; 8] = [0; 8];
let buf_len = buf.len();
if slice.len() < buf_len {
return Err(());
return Err(Error::String("Buffer is too small to convert to 8 bytes"));
}
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
buf[i] = *r;
crates/brk_exit/src/lib.rs
+4
View File
@@ -57,6 +57,10 @@ impl Exit {
self.blocking.store(true, Ordering::SeqCst);
}
pub fn blocked(&self) -> bool {
self.blocking.load(Ordering::SeqCst)
}
pub fn release(&self) {
self.blocking.store(false, Ordering::SeqCst);
}
crates/brk_indexer/src/stores.rs
+13 -8
View File
@@ -1,8 +1,8 @@
use std::{fs, path::Path, thread};
use brk_core::{
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, TxIndex,
TxidPrefix, Value, Version,
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutputType, OutputTypeIndex, Result,
TxIndex, TxidPrefix, Value, Version,
};
use brk_store::Store;
use brk_vec::AnyIterableVec;
@@ -37,26 +37,29 @@ impl Stores {
thread::scope(|scope| {
let addressbyteshash_to_outputtypeindex = scope.spawn(|| {
Store::import(
keyspace.clone(),
&keyspace,
path,
"addressbyteshash_to_outputtypeindex",
version + VERSION + Version::ZERO,
None,
)
});
let blockhashprefix_to_height = scope.spawn(|| {
Store::import(
keyspace.clone(),
&keyspace,
path,
"blockhashprefix_to_height",
version + VERSION + Version::ZERO,
None,
)
});
let txidprefix_to_txindex = scope.spawn(|| {
Store::import(
keyspace.clone(),
&keyspace,
path,
"txidprefix_to_txindex",
version + VERSION + Version::ZERO,
None,
)
});
@@ -285,8 +288,8 @@ impl Stores {
.unwrap()
}
pub fn commit(&mut self, height: Height) -> fjall::Result<()> {
thread::scope(|scope| -> fjall::Result<()> {
pub fn commit(&mut self, height: Height) -> Result<()> {
thread::scope(|scope| -> Result<()> {
let addressbyteshash_to_outputtypeindex_commit_handle =
scope.spawn(|| self.addressbyteshash_to_outputtypeindex.commit(height));
let blockhashprefix_to_height_commit_handle =
@@ -303,7 +306,9 @@ impl Stores {
Ok(())
})?;
self.keyspace.persist(PersistMode::SyncAll)
self.keyspace
.persist(PersistMode::SyncAll)
.map_err(|e| e.into())
}
pub fn rotate_memtables(&self) {
crates/brk_server/Cargo.toml
+1 -1
View File
@@ -24,7 +24,7 @@ color-eyre = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
minreq = { workspace = true }
oxc = { version = "0.72.1", features = ["codegen", "minifier"] }
oxc = { version = "0.72.2", features = ["codegen", "minifier"] }
serde = { workspace = true }
tokio = { workspace = true }
tower-http = { version = "0.6.4", features = ["compression-full", "trace"] }
crates/brk_state/Cargo.toml
+1
View File
@@ -10,6 +10,7 @@ repository.workspace = true
brk_core = { workspace = true }
brk_vec = { workspace = true }
brk_store = { workspace = true }
fjall = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true }
zerocopy = { workspace = true }
crates/brk_state/src/cohort.rs
+100 -12
View File
@@ -1,16 +1,47 @@
use brk_core::{Dollars, Sats};
use std::{cmp::Ordering, path::Path};
use brk_core::{CheckedSub, Dollars, Height, Result, Sats, Version};
use brk_store::Store;
use fjall::TransactionalKeyspace;
use crate::UnrealizedState;
use super::{RealizedState, SupplyState};
#[derive(Default, Clone)]
#[derive(Clone)]
pub struct CohortState {
pub supply: SupplyState,
pub realized: Option<RealizedState>,
// pub price_to_amount: Store<Dollars, Sats>,
pub price_to_amount: Store<Dollars, Sats>,
}
impl CohortState {
pub fn default_and_import(
keyspace: &TransactionalKeyspace,
path: &Path,
name: &str,
version: Version,
compute_dollars: bool,
) -> Result<Self> {
Ok(Self {
supply: SupplyState::default(),
realized: compute_dollars.then_some(RealizedState::NAN),
price_to_amount: Store::import(
keyspace,
path,
&format!("{name}_price_to_amount"),
version + Version::TWO,
Some(None),
)?,
})
}
pub fn commit(&mut self, height: Height) -> Result<()> {
self.price_to_amount
.retain_or_del(|_, sats| *sats != Sats::ZERO);
self.price_to_amount.commit(height)
}
pub fn reset_single_iteration_values(&mut self) {
if let Some(realized) = self.realized.as_mut() {
realized.reset_single_iteration_values();
@@ -20,21 +51,27 @@ impl CohortState {
pub fn increment(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.supply += supply_state;
if let Some(realized) = self.realized.as_mut() {
realized.increment(supply_state, price.unwrap());
let price = price.unwrap();
realized.increment(supply_state, price);
*self.price_to_amount.get_mut_or_default(&price) += supply_state.value;
}
}
pub fn decrement(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.supply -= supply_state;
if let Some(realized) = self.realized.as_mut() {
realized.decrement(supply_state, price.unwrap());
let price = price.unwrap();
realized.decrement(supply_state, price);
*self.price_to_amount.get_mut_or_default(&price) -= supply_state.value;
}
}
pub fn receive(&mut self, supply_state: &SupplyState, price: Option<Dollars>) {
self.supply += supply_state;
if let Some(realized) = self.realized.as_mut() {
realized.receive(supply_state, price.unwrap());
let price = price.unwrap();
realized.receive(supply_state, price);
*self.price_to_amount.get_mut_or_default(&price) += supply_state.value;
}
}
@@ -47,12 +84,63 @@ impl CohortState {
) {
self.supply -= supply_state;
if let Some(realized) = self.realized.as_mut() {
realized.send(
supply_state,
current_price.unwrap(),
prev_price.unwrap(),
older_than_hour,
);
let current_price = current_price.unwrap();
let prev_price = prev_price.unwrap();
realized.send(supply_state, current_price, prev_price, older_than_hour);
*self.price_to_amount.get_mut_or_default(&prev_price) -= supply_state.value;
}
}
pub fn compute_unrealized_states(
&self,
height_price: Dollars,
date_price: Option<Dollars>,
) -> (UnrealizedState, Option<UnrealizedState>) {
if self.price_to_amount.is_empty() {
return (
UnrealizedState::NAN,
date_price.map(|_| UnrealizedState::NAN),
);
}
let mut height_unrealized_state = UnrealizedState::ZERO;
let mut date_unrealized_state = date_price.map(|_| UnrealizedState::ZERO);
let update_state =
|price: Dollars, current_price: Dollars, sats: Sats, state: &mut UnrealizedState| {
match price.cmp(&current_price) {
Ordering::Equal => {
state.supply_even += sats;
}
Ordering::Less => {
state.supply_in_profit += sats;
// if price > Dollars::ZERO {
// state.unrealized_profit +=
// current_price.checked_sub(price).unwrap() * sats;
// }
}
Ordering::Greater => {
state.supply_in_loss += sats;
// state.unrealized_loss += price.checked_sub(current_price).unwrap() * sats;
}
}
};
self.price_to_amount
.unordered_clone_iter()
.for_each(|(price, sats)| {
update_state(price, height_price, sats, &mut height_unrealized_state);
if let Some(date_price) = date_price {
update_state(
price,
date_price,
sats,
date_unrealized_state.as_mut().unwrap(),
)
}
});
(height_unrealized_state, date_unrealized_state)
}
}
crates/brk_state/src/lib.rs
+2
View File
@@ -5,11 +5,13 @@ mod realized;
// mod hot;
mod supply;
mod transacted;
mod unrealized;
pub use block::*;
pub use cohort::*;
pub use outputs::*;
pub use realized::*;
pub use unrealized::*;
// pub use hot::*;
pub use supply::*;
pub use transacted::*;
crates/brk_state/src/realized.rs
+5 -5
View File
@@ -1,6 +1,6 @@
use std::cmp::Ordering;
use brk_core::{Bitcoin, CheckedSub, Dollars};
use brk_core::{CheckedSub, Dollars};
use super::SupplyState;
@@ -52,12 +52,12 @@ impl RealizedState {
self.adj_value_destroyed = Dollars::ZERO;
}
let value = price * Bitcoin::from(supply_state.value);
let value = price * supply_state.value;
self.cap += value;
}
pub fn decrement(&mut self, supply_state: &SupplyState, price: Dollars) {
let value = price * Bitcoin::from(supply_state.value);
let value = price * supply_state.value;
self.cap = self.cap.checked_sub(value).unwrap();
}
@@ -72,8 +72,8 @@ impl RealizedState {
prev_price: Dollars,
older_than_hour: bool,
) {
let current_value = current_price * Bitcoin::from(supply_state.value);
let prev_value = prev_price * Bitcoin::from(supply_state.value);
let current_value = current_price * supply_state.value;
let prev_value = prev_price * supply_state.value;
self.value_created += current_value;
self.value_destroyed += prev_value;
crates/brk_state/src/unrealized.rs
+28
View File
@@ -0,0 +1,28 @@
use brk_core::{Dollars, Sats};
#[derive(Debug, Default, Clone)]
pub struct UnrealizedState {
pub supply_in_profit: Sats,
pub supply_even: Sats,
pub supply_in_loss: Sats,
pub unrealized_profit: Dollars,
pub unrealized_loss: Dollars,
}
impl UnrealizedState {
pub const NAN: Self = Self {
supply_in_profit: Sats::ZERO,
supply_even: Sats::ZERO,
supply_in_loss: Sats::ZERO,
unrealized_profit: Dollars::NAN,
unrealized_loss: Dollars::NAN,
};
pub const ZERO: Self = Self {
supply_in_profit: Sats::ZERO,
supply_even: Sats::ZERO,
supply_in_loss: Sats::ZERO,
unrealized_profit: Dollars::ZERO,
unrealized_loss: Dollars::ZERO,
};
}
crates/brk_store/src/lib.rs
+63 -16
View File
@@ -6,10 +6,10 @@ use std::{
path::Path,
};
use brk_core::{Height, Value, Version};
use brk_core::{Height, Result, Value, Version};
use byteview::ByteView;
use fjall::{
PartitionCreateOptions, PersistMode, ReadTransaction, Result, TransactionalKeyspace,
PartitionCreateOptions, PersistMode, ReadTransaction, TransactionalKeyspace,
TransactionalPartitionHandle,
};
use zerocopy::{Immutable, IntoBytes};
@@ -25,29 +25,34 @@ pub struct Store<Key, Value> {
rtx: ReadTransaction,
puts: BTreeMap<Key, Value>,
dels: BTreeSet<Key>,
bloom_filter_bits: Option<Option<u8>>,
}
/// Use default if will read
const DEFAULT_BLOOM_FILTER_BITS: Option<u8> = Some(5);
const CHECK_COLLISISONS: bool = true;
const MAJOR_FJALL_VERSION: Version = Version::TWO;
impl<K, V> Store<K, V>
where
K: Debug + Clone + Into<ByteView> + Ord + Immutable + IntoBytes,
K: Debug + Clone + Into<ByteView> + TryFrom<ByteView> + Ord + Immutable + IntoBytes,
V: Debug + Clone + Into<ByteView> + TryFrom<ByteView>,
<K as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static,
<V as TryFrom<ByteView>>::Error: error::Error + Send + Sync + 'static,
{
pub fn import(
keyspace: TransactionalKeyspace,
keyspace: &TransactionalKeyspace,
path: &Path,
name: &str,
version: Version,
) -> color_eyre::Result<Self> {
bloom_filter_bits: Option<Option<u8>>,
) -> Result<Self> {
let (meta, partition) = StoreMeta::checked_open(
&keyspace,
keyspace,
&path.join(format!("meta/{name}")),
MAJOR_FJALL_VERSION + version,
|| {
Self::open_partition_handle(&keyspace, name).inspect_err(|e| {
Self::open_partition_handle(keyspace, name, bloom_filter_bits).inspect_err(|e| {
eprintln!("{e}");
eprintln!("Delete {path:?} and try again");
})
@@ -59,11 +64,12 @@ where
Ok(Self {
meta,
name: name.to_owned(),
keyspace,
keyspace: keyspace.clone(),
partition,
rtx,
puts: BTreeMap::new(),
dels: BTreeSet::new(),
bloom_filter_bits,
})
}
@@ -77,6 +83,29 @@ where
}
}
pub fn get_mut_or_default(&mut self, key: &K) -> &mut V
where
V: Default,
{
self.puts.entry(key.clone()).or_insert_with(|| {
if let Some(slice) = self.rtx.get(&self.partition, key.as_bytes()).unwrap() {
V::try_from(slice.as_bytes().into()).unwrap()
} else {
V::default()
}
})
}
pub fn unordered_clone_iter(&self) -> impl Iterator<Item = (K, V)> {
self.rtx
.iter(&self.partition)
.map(|res| res.unwrap())
.map(|(k, v)| (K::try_from(ByteView::from(k)).unwrap(), v))
.filter(|(k, _)| !self.puts.contains_key(k) && !self.dels.contains(k))
.map(|(k, v)| (k, V::try_from(ByteView::from(v)).unwrap()))
.chain(self.puts.iter().map(|(k, v)| (k.clone(), v.clone())))
}
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
if self.needs(height) {
if !self.dels.is_empty() {
@@ -102,6 +131,19 @@ where
}
}
pub fn retain_or_del<F>(&mut self, retain: F)
where
F: Fn(&K, &mut V) -> bool,
{
self.puts.retain(|k, v| {
let ret = retain(k, v);
if !ret {
self.dels.insert(k.clone());
}
ret
});
}
pub fn commit(&mut self, height: Height) -> Result<()> {
if self.has(height) && self.puts.is_empty() && self.dels.is_empty() {
return Ok(());
@@ -169,20 +211,24 @@ where
fn open_partition_handle(
keyspace: &TransactionalKeyspace,
name: &str,
bloom_filter_bits: Option<Option<u8>>,
) -> Result<TransactionalPartitionHandle> {
keyspace.open_partition(
name,
PartitionCreateOptions::default()
.bloom_filter_bits(Some(5))
.max_memtable_size(8 * 1024 * 1024)
.manual_journal_persist(true),
)
keyspace
.open_partition(
name,
PartitionCreateOptions::default()
.bloom_filter_bits(bloom_filter_bits.unwrap_or(DEFAULT_BLOOM_FILTER_BITS))
.max_memtable_size(8 * 1024 * 1024)
.manual_journal_persist(true),
)
.map_err(|e| e.into())
}
pub fn reset_partition(&mut self) -> Result<()> {
self.keyspace.delete_partition(self.partition.clone())?;
self.keyspace.persist(PersistMode::SyncAll)?;
self.partition = Self::open_partition_handle(&self.keyspace, &self.name)?;
self.partition =
Self::open_partition_handle(&self.keyspace, &self.name, self.bloom_filter_bits)?;
Ok(())
}
}
@@ -201,6 +247,7 @@ where
rtx: self.keyspace.read_tx(),
puts: self.puts.clone(),
dels: self.dels.clone(),
bloom_filter_bits: self.bloom_filter_bits,
}
}
}
crates/brk_store/src/meta.rs
+4 -4
View File
@@ -3,7 +3,7 @@ use std::{
path::{Path, PathBuf},
};
use brk_core::Version;
use brk_core::{Result, Version};
use fjall::{TransactionalKeyspace, TransactionalPartitionHandle};
use zerocopy::{FromBytes, IntoBytes};
@@ -23,9 +23,9 @@ impl StoreMeta {
path: &Path,
version: Version,
open_partition_handle: F,
) -> color_eyre::Result<(Self, TransactionalPartitionHandle)>
) -> Result<(Self, TransactionalPartitionHandle)>
where
F: Fn() -> fjall::Result<TransactionalPartitionHandle>,
F: Fn() -> Result<TransactionalPartitionHandle>,
{
fs::create_dir_all(path)?;
@@ -109,7 +109,7 @@ impl StoreMeta {
path.join("height")
}
fn read_length_(path: &Path) -> color_eyre::Result<usize> {
fn read_length_(path: &Path) -> Result<usize> {
Ok(fs::read(Self::path_length(path))
.map(|v| usize::read_from_bytes(v.as_slice()).unwrap_or_default())
.unwrap_or_default())
crates/brk_vec/src/variants/eager.rs
+14 -4
View File
@@ -58,9 +58,14 @@ where
if exit.triggered() {
return Ok(());
}
exit.block();
let blocked = exit.blocked();
if !blocked {
exit.block();
}
self.inner.truncate_if_needed(index)?;
exit.release();
if !blocked {
exit.release();
}
Ok(())
}
@@ -89,9 +94,14 @@ where
if exit.triggered() {
return Ok(());
}
exit.block();
let blocked = exit.blocked();
if !blocked {
exit.block();
}
self.inner.flush()?;
exit.release();
if !blocked {
exit.release();
}
Ok(())
}
websites/kibo.money/packages/lightweight-charts/wrapper.js
+1 -1
View File
@@ -511,7 +511,7 @@ export default import("./v5.0.6-treeshaked/script.js").then((lc) => {
lineWidth: /** @type {any} */ (1.5),
visible: defaultActive !== false,
baseValue: {
price: 0,
price: options?.createPriceLine?.value ?? 0,
},
...options,
topLineColor: options?.topLineColor ?? colors.green(),
websites/kibo.money/scripts/options.js
+7 -46
View File
@@ -1273,9 +1273,15 @@ function createPartialOptions(colors) {
color: colors.green,
}),
createBaseSeries({
key: `${fixKey(args.key)}negative-realized-loss`,
key: `${fixKey(args.key)}realized-loss`,
name: "Loss",
color: colors.red,
defaultActive: false,
}),
createBaseSeries({
key: `${fixKey(args.key)}negative-realized-loss`,
name: "Negative Loss",
color: colors.red,
}),
],
},
@@ -2818,48 +2824,3 @@ export function initOptions({
// name: "Highly Liquid",
// },
// ]);
// return {
// xTermHolders,
// upTo,
// fromX,
// fromXToY,
// epochs,
// age,
// type,
// size,
// address,
// liquidities,
// averages,
// cumulativeReturns,
// compoundReturns,
// percentiles,
// };
// }
// /**
// * @typedef {ReturnType<typeof initGroups>} Groups
// *
// * @typedef {Groups["age"][number]["id"]} AgeCohortId
// *
// * @typedef {Exclude<AgeCohortId, "">} AgeCohortIdSub
// *
// * @typedef {Groups["address"][number]["key"]} AddressCohortId
// *
// * @typedef {Groups["liquidities"][number]["id"]} LiquidityId
// *
// * @typedef {AgeCohortId | AddressCohortId} AnyCohortId
// *
// * @typedef {AddressCohortId | LiquidityId} AnyAddressCohortId
// *
// * @typedef {AnyCohortId | LiquidityId} AnyPossibleCohortId
// *
// * @typedef {'' | `${AgeCohortIdSub | AddressCohortId | LiquidityId}-`} AnyDatasetPrefix
// *
// * @typedef {Groups["averages"][number]["key"]} AverageName
// *
// * @typedef {Groups["cumulativeReturns"][number]["key"]} CumulativeReturnKey
// *
// * @typedef {Groups["compoundReturns"][number]["key"]} CompoundReturnKey
// *
// * @typedef {Groups["percentiles"][number]["id"]} PercentileId
// */
websites/kibo.money/scripts/vecid-to-indexes.js
+410
View File
File diff suppressed because it is too large Load Diff