Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-11-06 13:07:07 +01:00
parent a2ba4d89f3
commit df577ca7f5
62 changed files with 1078 additions and 2291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -14,7 +14,7 @@ bridge/
_*
# Logs
.log*
*.log*
# Environment variables/configs
.env

133
Cargo.lock generated
View File

@@ -73,7 +73,7 @@ checksum = "fe233a377643e0fc1a56421d7c90acdec45c291b30345eb9f08e8d0ddce5a4ab"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -212,7 +212,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -755,8 +755,8 @@ dependencies = [
"brk_traversable",
"brk_types",
"log",
"rand 0.9.2",
"rayon",
"redb",
"rustc-hash",
"vecdb",
]
@@ -885,7 +885,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde_json",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -1294,7 +1294,6 @@ dependencies = [
"candystore",
"log",
"parking_lot 0.12.5",
"redb",
"rustc-hash",
]
@@ -1326,7 +1325,7 @@ version = "0.0.111"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -1336,13 +1335,12 @@ dependencies = [
"allocative",
"bitcoin",
"brk_error",
"byteview 0.8.0",
"byteview 0.6.1",
"derive_deref",
"itoa",
"jiff",
"num_enum",
"rapidhash",
"redb",
"ryu",
"schemars",
"serde",
@@ -1400,7 +1398,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -1542,7 +1540,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -1812,7 +1810,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -1823,7 +1821,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81"
dependencies = [
"darling_core",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -1865,7 +1863,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf656eb071fe87d23716f933788a35a8ad6baa6fdbf66a67a261dbd3f9dc81a"
dependencies = [
"quote2",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -1885,7 +1883,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -1916,7 +1914,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
"unicode-xid",
]
@@ -1950,7 +1948,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -2004,7 +2002,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -2231,7 +2229,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -2691,7 +2689,7 @@ checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -2865,6 +2863,8 @@ dependencies = [
[[package]]
name = "lsm-tree"
version = "3.0.0-pre.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70cb98c7f7806b0ad4e9872735c8674bba8ce1277d0fc688d66e8a272b578dd7"
dependencies = [
"byteorder-lite",
"byteview 0.8.0",
@@ -2978,7 +2978,7 @@ checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
"target-features",
]
@@ -2999,7 +2999,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -3124,7 +3124,7 @@ dependencies = [
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -3238,7 +3238,7 @@ checksum = "003b4612827f6501183873fb0735da92157e3c7daa71c40921c7d2758fec2229"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -3282,7 +3282,7 @@ dependencies = [
"phf",
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -3812,7 +3812,7 @@ dependencies = [
"phf_shared",
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -3841,7 +3841,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -3864,14 +3864,14 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "pnp"
version = "0.12.4"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7adbc1ab7344e1e77be663e91cb129e989e398c319df7a9b8dbda9dd6758df38"
checksum = "2acd0b1e3a154e7c4610b9ab31491c32e9f47db2adc0c12047301f3bacc71597"
dependencies = [
"byteorder",
"concurrent_lru",
"fancy-regex",
"miniz_oxide",
"flate2",
"pathdiff",
"radix_trie",
"rustc-hash",
@@ -3954,7 +3954,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -4108,9 +4108,9 @@ dependencies = [
[[package]]
name = "rawdb"
version = "0.3.4"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c184e0f054f8b9018424fb226dbf122ceae050eb5f900e45d0809ff9fdc7b0d0"
checksum = "9c06cc46cc4b6da16c9981293906062230f822879c2c88367a40673db5e44414"
dependencies = [
"libc",
"memmap2",
@@ -4138,15 +4138,6 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "redb"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae323eb086579a3769daa2c753bb96deb95993c534711e0dbe881b5192906a06"
dependencies = [
"libc",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
@@ -4182,7 +4173,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -4270,7 +4261,7 @@ checksum = "bd83f5f173ff41e00337d97f6572e416d022ef8a19f371817259ae960324c482"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -4363,9 +4354,9 @@ dependencies = [
[[package]]
name = "schemars"
version = "1.0.5"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1317c3bf3e7df961da95b0a56a172a02abead31276215a0497241a7624b487ce"
checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289"
dependencies = [
"chrono",
"dyn-clone",
@@ -4378,14 +4369,14 @@ dependencies = [
[[package]]
name = "schemars_derive"
version = "1.0.5"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f760a6150d45dd66ec044983c124595ae76912e77ed0b44124cb3e415cce5d9"
checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633"
dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -4484,7 +4475,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -4495,7 +4486,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -4749,7 +4740,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -4774,9 +4765,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.108"
version = "2.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917"
checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f"
dependencies = [
"proc-macro2",
"quote",
@@ -4797,7 +4788,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -4865,7 +4856,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -4876,7 +4867,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -4955,7 +4946,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -5101,7 +5092,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -5191,7 +5182,7 @@ checksum = "ee6ff59666c9cbaec3533964505d39154dc4e0a56151fdea30a09ed0301f62e2"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
"termcolor",
]
@@ -5330,9 +5321,9 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]]
name = "vecdb"
version = "0.3.4"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44fef339cc0a6103cea9df44c1133d8275585b90dbe2faa6e3f60e6d2bac47b6"
checksum = "f0cff5037d7e5d66344dff249c366a527b530ef3c723e04f4a90fa11d5f6a9fc"
dependencies = [
"allocative",
"ctrlc",
@@ -5349,12 +5340,12 @@ dependencies = [
[[package]]
name = "vecdb_derive"
version = "0.3.4"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c09556497d1695d5ba268ebeaa1d2547c2c597a60098fde60fe78028b463c4c1"
checksum = "9f73046e065e63abb2ad8d1616a08ca8acf7d543f04ffb0222f6fc44e0d3eeec"
dependencies = [
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -5441,7 +5432,7 @@ dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
"wasm-bindgen-shared",
]
@@ -5544,7 +5535,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -5555,7 +5546,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -5818,7 +5809,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
"synstructure",
]
@@ -5839,7 +5830,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]
@@ -5859,7 +5850,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
"synstructure",
]
@@ -5893,7 +5884,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.108",
"syn 2.0.109",
]
[[package]]

11
Cargo.toml
View File

@@ -52,8 +52,8 @@ brk_store = { version = "0.0.111", path = "crates/brk_store" }
brk_types = { version = "0.0.111", path = "crates/brk_types" }
brk_traversable = { version = "0.0.111", path = "crates/brk_traversable", features = ["derive"] }
brk_traversable_derive = { version = "0.0.111", path = "crates/brk_traversable_derive" }
# byteview = "=0.6.1"
byteview = "~0.8.0"
byteview = "=0.6.1"
# byteview = "~0.8.0"
derive_deref = "1.1.1"
fjall2 = { version = "2.11.5", package = "brk_fjall" }
# fjall2 = { path = "../fjall2", package = "brk_fjall" }
@@ -66,9 +66,8 @@ log = "0.4.28"
minreq = { version = "2.14.1", features = ["https", "serde_json"] }
parking_lot = "0.12.5"
rayon = "1.11.0"
redb = "3.1.0"
rustc-hash = "2.1.1"
schemars = "1.0.5"
schemars = "1.1.0"
serde = "1.0.228"
serde_bytes = "0.11.19"
serde_derive = "1.0.228"
@@ -76,7 +75,7 @@ serde_json = { version = "1.0.145", features = ["float_roundtrip"] }
sonic-rs = "0.5.6"
tokio = { version = "1.48.0", features = ["rt-multi-thread"] }
# vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"] }
vecdb = { version = "0.3.4", features = ["derive"] }
vecdb = { version = "0.3.7", features = ["derive"] }
zerocopy = { version = "0.8.27", features = ["derive"] }
[workspace.metadata.release]
@@ -88,6 +87,6 @@ tag-message = "release: v{{version}}"
[workspace.metadata.dist]
cargo-dist-version = "0.30.2"
ci = "github"
# allow-dirty = ["ci"]
allow-dirty = ["ci"]
installers = []
targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]

19
crates/brk_cli/src/paths.rs
View File

@@ -15,22 +15,3 @@ pub fn dot_brk_log_path() -> PathBuf {
pub fn default_brk_path() -> PathBuf {
dot_brk_path()
}
pub fn default_bitcoin_path() -> PathBuf {
if env::consts::OS == "macos" {
default_mac_bitcoin_path()
} else {
default_linux_bitcoin_path()
}
}
pub fn default_linux_bitcoin_path() -> PathBuf {
Path::new(&std::env::var("HOME").unwrap()).join(".bitcoin")
}
pub fn default_mac_bitcoin_path() -> PathBuf {
Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin")
}

10
crates/brk_computer/examples/computer.rs
View File

@@ -1,4 +1,5 @@
use std::{
env,
path::Path,
thread::{self, sleep},
time::{Duration, Instant},
@@ -25,17 +26,14 @@ pub fn main() -> Result<()> {
fn run() -> Result<()> {
brk_logger::init(Some(Path::new(".log")))?;
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
let bitcoin_dir = Client::default_bitcoin_path();
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk");
// let outputs_dir = Path::new("../../_outputs");
let client = Client::new(
"http://localhost:8332",
Client::default_url(),
Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?;

13
crates/brk_computer/src/blks.rs
View File

@@ -6,7 +6,8 @@ use brk_reader::Reader;
use brk_traversable::Traversable;
use brk_types::{BlkPosition, Height, TxIndex, Version};
use vecdb::{
AnyStoredVec, AnyVec, CompressedVec, Database, Exit, GenericStoredVec, PAGE_SIZE, VecIterator,
AnyStoredVec, AnyVec, CompressedVec, Database, Exit, GenericStoredVec, PAGE_SIZE,
VecIteratorExtended,
};
use super::Indexes;
@@ -58,7 +59,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, starting_indexes, reader, exit)?;
self.db.flush_then_punch()?;
self.db.compact()?;
Ok(())
}
@@ -75,14 +76,14 @@ impl Vecs {
let Some(min_height) = indexer
.vecs
.txindex_to_height
.iter()
.get_inner(min_txindex)
.iter()?
.get(min_txindex)
.map(|h| h.min(starting_indexes.height))
else {
return Ok(());
};
let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter();
let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter()?;
parser
.read(
@@ -99,7 +100,7 @@ impl Vecs {
exit,
)?;
let txindex = height_to_first_txindex_iter.unwrap_get_inner(height);
let txindex = height_to_first_txindex_iter.get_unwrap(height);
block.tx_metadata().iter().enumerate().try_for_each(
|(index, metadata)| -> Result<()> {

47
crates/brk_computer/src/chain.rs
View File

@@ -12,7 +12,7 @@ use brk_types::{
};
use vecdb::{
AnyCloneableIterableVec, AnyIterableVec, Database, EagerVec, Exit, LazyVecFrom1, LazyVecFrom2,
LazyVecFrom3, PAGE_SIZE, StoredIndex, VecIterator,
LazyVecFrom3, PAGE_SIZE, StoredIndex, VecIterator, VecIteratorExtended,
};
use crate::grouped::{
@@ -1123,7 +1123,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, price, exit)?;
self.db.flush_then_punch()?;
self.db.compact()?;
Ok(())
}
@@ -1152,9 +1152,8 @@ impl Vecs {
starting_indexes.height,
&indexes.height_to_timestamp_fixed,
|(h, t, ..)| {
while t.difference_in_days_between(
height_to_timestamp_fixed_iter.unwrap_get_inner(prev),
) > 0
while t.difference_in_days_between(height_to_timestamp_fixed_iter.unsafe_get(prev))
> 0
{
prev.increment();
if prev > h {
@@ -1216,7 +1215,7 @@ impl Vecs {
&indexer.vecs.height_to_timestamp,
|(height, timestamp, ..)| {
let interval = height.decremented().map_or(Timestamp::ZERO, |prev_h| {
let prev_timestamp = height_to_timestamp_iter.unwrap_get_inner(prev_h);
let prev_timestamp = height_to_timestamp_iter.unsafe_get(prev_h);
timestamp
.checked_sub(prev_timestamp)
.unwrap_or(Timestamp::ZERO)
@@ -1266,19 +1265,19 @@ impl Vecs {
Some(&self.height_to_vbytes),
)?;
let mut height_to_timestamp_iter = indexer.vecs.height_to_timestamp.iter();
let mut height_to_timestamp_iter = indexer.vecs.height_to_timestamp.iter()?;
self.difficultyepoch_to_timestamp.compute_transform(
starting_indexes.difficultyepoch,
&indexes.difficultyepoch_to_first_height,
|(i, h, ..)| (i, height_to_timestamp_iter.unwrap_get_inner(h)),
|(i, h, ..)| (i, height_to_timestamp_iter.unsafe_get(h)),
exit,
)?;
self.halvingepoch_to_timestamp.compute_transform(
starting_indexes.halvingepoch,
&indexes.halvingepoch_to_first_height,
|(i, h, ..)| (i, height_to_timestamp_iter.unwrap_get_inner(h)),
|(i, h, ..)| (i, height_to_timestamp_iter.unsafe_get(h)),
exit,
)?;
@@ -1292,9 +1291,8 @@ impl Vecs {
|(di, height, ..)| {
(
di,
height_to_difficultyepoch_iter.unwrap_get_inner(
height + (*height_count_iter.unwrap_get_inner(di) - 1),
),
height_to_difficultyepoch_iter
.unsafe_get(height + (*height_count_iter.unsafe_get(di) - 1)),
)
},
exit,
@@ -1312,9 +1310,8 @@ impl Vecs {
|(di, height, ..)| {
(
di,
height_to_halvingepoch_iter.unwrap_get_inner(
height + (*height_count_iter.unwrap_get_inner(di) - 1),
),
height_to_halvingepoch_iter
.unsafe_get(height + (*height_count_iter.unsafe_get(di) - 1)),
)
},
exit,
@@ -1365,7 +1362,7 @@ impl Vecs {
&indexer.vecs.height_to_first_txindex,
&indexer.vecs.txindex_to_txid,
|txindex| {
let v = txindex_to_txversion_iter.unwrap_get_inner(txindex);
let v = txindex_to_txversion_iter.unsafe_get(txindex);
v == txversion
},
exit,
@@ -1385,7 +1382,7 @@ impl Vecs {
let value = if txoutindex == TxOutIndex::COINBASE {
Sats::MAX
} else {
txoutindex_to_value_iter.unwrap_get_inner(txoutindex)
txoutindex_to_value_iter.unsafe_get(txoutindex)
};
(txinindex, value)
},
@@ -1525,14 +1522,14 @@ impl Vecs {
&indexer.vecs.height_to_first_txindex,
|(height, txindex, ..)| {
let first_txoutindex = txindex_to_first_txoutindex_iter
.unwrap_get_inner(txindex)
.unsafe_get(txindex)
.to_usize();
let output_count = txindex_to_output_count_iter.unwrap_get_inner(txindex);
let output_count = txindex_to_output_count_iter.unsafe_get(txindex);
let mut sats = Sats::ZERO;
(first_txoutindex..first_txoutindex + usize::from(output_count)).for_each(
|txoutindex| {
sats += txoutindex_to_value_iter
.unwrap_get_inner(TxOutIndex::from(txoutindex));
.unsafe_get(TxOutIndex::from(txoutindex));
},
);
(height, sats)
@@ -1556,7 +1553,7 @@ impl Vecs {
let range = *h - (*count - 1)..=*h;
let sum = range
.map(Height::from)
.map(|h| height_to_coinbase_iter.unwrap_get_inner(h))
.map(|h| height_to_coinbase_iter.unsafe_get(h))
.sum::<Sats>();
(h, sum)
},
@@ -1575,7 +1572,7 @@ impl Vecs {
let range = *h - (*count - 1)..=*h;
let sum = range
.map(Height::from)
.map(|h| height_to_coinbase_iter.unwrap_get_inner(h))
.map(|h| height_to_coinbase_iter.unsafe_get(h))
.sum::<Dollars>();
(h, sum)
},
@@ -1593,7 +1590,7 @@ impl Vecs {
starting_indexes.height,
self.indexes_to_coinbase.sats.height.as_ref().unwrap(),
|(height, coinbase, ..)| {
let fees = indexes_to_fee_sum_iter.unwrap_get_inner(height);
let fees = indexes_to_fee_sum_iter.unsafe_get(height);
(height, coinbase.checked_sub(fees).unwrap())
},
exit,
@@ -1787,8 +1784,8 @@ impl Vecs {
starting_indexes.height,
self.indexes_to_output_count.height.unwrap_cumulative(),
|(h, output_count, ..)| {
let input_count = input_count_iter.unwrap_get_inner(h);
let opreturn_count = opreturn_count_iter.unwrap_get_inner(h);
let input_count = input_count_iter.unsafe_get(h);
let opreturn_count = opreturn_count_iter.unsafe_get(h);
let block_count = u64::from(h + 1_usize);
// -1 > genesis output is unspendable
let mut utxo_count =

4
crates/brk_computer/src/cointime.rs
View File

@@ -298,7 +298,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexes, starting_indexes, price, chain, stateful, exit)?;
self.db.flush_then_punch()?;
self.db.compact()?;
Ok(())
}
@@ -339,7 +339,7 @@ impl Vecs {
starting_indexes.height,
self.indexes_to_coinblocks_created.height.as_ref().unwrap(),
|(i, created, ..)| {
let destroyed = coinblocks_destroyed_iter.unwrap_get_inner(i);
let destroyed = coinblocks_destroyed_iter.unsafe_get(i);
(i, created.checked_sub(destroyed).unwrap())
},
exit,

2
crates/brk_computer/src/constants.rs
View File

@@ -177,7 +177,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexes, starting_indexes, exit)?;
self.db.flush_then_punch()?;
self.db.compact()?;
Ok(())
}

20
crates/brk_computer/src/fetched.rs
View File

@@ -60,7 +60,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.db.flush_then_punch()?;
self.db.compact()?;
Ok(())
}
@@ -76,16 +76,18 @@ impl Vecs {
.height
.min(Height::from(self.height_to_price_ohlc_in_cents.len()));
height_to_timestamp
.iter_at(index)
.iter()?
.skip(index.to_usize())
.enumerate()
.try_for_each(|(i, v)| -> Result<()> {
self.height_to_price_ohlc_in_cents.forced_push_at(
i,
i.into(),
self.fetcher
.get_height(
i,
i.into(),
v,
i.decremented().map(|prev_i| {
height_to_timestamp.into_iter().unwrap_get_inner(prev_i)
height_to_timestamp.into_iter().get_unwrap_at(prev_i)
}),
)
.unwrap(),
@@ -101,14 +103,16 @@ impl Vecs {
let mut prev = None;
indexes
.dateindex_to_date
.iter_at(index)
.iter()
.skip(index.to_usize())
.enumerate()
.try_for_each(|(i, d)| -> Result<()> {
if prev.is_none() {
let i = i.to_usize();
prev.replace(if i > 0 {
self.dateindex_to_price_ohlc_in_cents
.into_iter()
.unwrap_get_inner_(i - 1)
.get_unwrap_at(i - 1)
} else {
OHLCCents::default()
});
@@ -129,7 +133,7 @@ impl Vecs {
prev.replace(ohlc.clone());
self.dateindex_to_price_ohlc_in_cents
.forced_push_at(i, ohlc, exit)?;
.forced_push_at_(i, ohlc, exit)?;
Ok(())
})?;

44
crates/brk_computer/src/grouped/builder_eager.rs
View File

@@ -219,9 +219,13 @@ where
let cumulative_vec = self.cumulative.as_mut().unwrap();
let mut cumulative = index.decremented().map_or(T::from(0_usize), |index| {
cumulative_vec.iter().unwrap_get_inner(index)
cumulative_vec.iter().unsafe_get(index)
});
source.iter_at(index).try_for_each(|(i, v)| -> Result<()> {
source
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, v)| -> Result<()> {
cumulative += v;
cumulative_vec.forced_push_at(i, cumulative, exit)?;
Ok(())
@@ -256,14 +260,13 @@ where
let mut cumulative = cumulative_vec.map(|cumulative_vec| {
index.decremented().map_or(T::from(0_usize), |index| {
cumulative_vec.iter().unwrap_get_inner(index)
cumulative_vec.iter().unsafe_get(index)
})
});
first_indexes
.iter_at(index)
.try_for_each(|(index, first_index)| -> Result<()> {
let count_index = count_indexes_iter.unwrap_get_inner(index);
first_indexes.iter().skip(index).enumerate().try_for_each(
|(index, first_index)| -> Result<()> {
let count_index = count_indexes_iter.unsafe_get(index);
if let Some(first) = self.first.as_mut() {
let f = source_iter
@@ -278,7 +281,7 @@ where
panic!("should compute last if count can be 0")
}
let last_index = first_index + (count_index - 1);
let v = source_iter.unwrap_get_inner(last_index);
let v = source_iter.unsafe_get(last_index);
// .context("to work")
// .inspect_err(|_| {
// dbg!(first_index, count_index, last_index);
@@ -382,7 +385,8 @@ where
}
Ok(())
})?;
},
)?;
self.safe_flush(exit)?;
@@ -427,20 +431,16 @@ where
let mut cumulative = self.cumulative.as_mut().map(|cumulative_vec| {
index.decremented().map_or(T::from(0_usize), |index| {
cumulative_vec.iter().unwrap_get_inner(index)
cumulative_vec.iter().unsafe_get(index)
})
});
first_indexes
.iter_at(index)
.try_for_each(|(index, first_index, ..)| -> Result<()> {
let count_index = count_indexes_iter.unwrap_get_inner(index);
first_indexes.iter().skip(index).enumerate().try_for_each(
|(index, first_index, ..)| -> Result<()> {
let count_index = count_indexes_iter.unsafe_get(index);
if let Some(first) = self.first.as_mut() {
let v = source_first_iter
.as_mut()
.unwrap()
.unwrap_get_inner(first_index);
let v = source_first_iter.as_mut().unwrap().unsafe_get(first_index);
first.forced_push_at(index, v, exit)?;
}
@@ -450,10 +450,7 @@ where
panic!("should compute last if count can be 0")
}
let last_index = first_index + (count_index - 1);
let v = source_last_iter
.as_mut()
.unwrap()
.unwrap_get_inner(last_index);
let v = source_last_iter.as_mut().unwrap().unsafe_get(last_index);
last.forced_push_at(index, v, exit)?;
}
@@ -529,7 +526,8 @@ where
}
Ok(())
})?;
},
)?;
self.safe_flush(exit)?;

150
crates/brk_computer/src/grouped/from_txindex.rs
View File

@@ -255,132 +255,77 @@ impl ComputedVecsFromTxindex<Bitcoin> {
if let Some(first) = self.height.first.as_mut() {
first.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_first()
.into_iter()
.unwrap_get_inner(height),
),
Bitcoin::from(sats.height.unwrap_first().into_iter().unsafe_get(height)),
exit,
)?;
}
if let Some(average) = self.height.average.as_mut() {
average.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_average()
.into_iter()
.unwrap_get_inner(height),
),
Bitcoin::from(sats.height.unwrap_average().into_iter().unsafe_get(height)),
exit,
)?;
}
if let Some(sum) = self.height.sum.as_mut() {
sum.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_sum()
.into_iter()
.unwrap_get_inner(height),
),
Bitcoin::from(sats.height.unwrap_sum().into_iter().unsafe_get(height)),
exit,
)?;
}
if let Some(max) = self.height.max.as_mut() {
max.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_max()
.into_iter()
.unwrap_get_inner(height),
),
Bitcoin::from(sats.height.unwrap_max().into_iter().unsafe_get(height)),
exit,
)?;
}
if let Some(pct90) = self.height.pct90.as_mut() {
pct90.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_pct90()
.into_iter()
.unwrap_get_inner(height),
),
Bitcoin::from(sats.height.unwrap_pct90().into_iter().unsafe_get(height)),
exit,
)?;
}
if let Some(pct75) = self.height.pct75.as_mut() {
pct75.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_pct75()
.into_iter()
.unwrap_get_inner(height),
),
Bitcoin::from(sats.height.unwrap_pct75().into_iter().unsafe_get(height)),
exit,
)?;
}
if let Some(median) = self.height.median.as_mut() {
median.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_median()
.into_iter()
.unwrap_get_inner(height),
),
Bitcoin::from(sats.height.unwrap_median().into_iter().unsafe_get(height)),
exit,
)?;
}
if let Some(pct25) = self.height.pct25.as_mut() {
pct25.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_pct25()
.into_iter()
.unwrap_get_inner(height),
),
Bitcoin::from(sats.height.unwrap_pct25().into_iter().unsafe_get(height)),
exit,
)?;
}
if let Some(pct10) = self.height.pct10.as_mut() {
pct10.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_pct10()
.into_iter()
.unwrap_get_inner(height),
),
Bitcoin::from(sats.height.unwrap_pct10().into_iter().unsafe_get(height)),
exit,
)?;
}
if let Some(min) = self.height.min.as_mut() {
min.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_min()
.into_iter()
.unwrap_get_inner(height),
),
Bitcoin::from(sats.height.unwrap_min().into_iter().unsafe_get(height)),
exit,
)?;
}
if let Some(last) = self.height.last.as_mut() {
last.forced_push_at(
height,
Bitcoin::from(
sats.height
.unwrap_last()
.into_iter()
.unwrap_get_inner(height),
),
Bitcoin::from(sats.height.unwrap_last().into_iter().unsafe_get(height)),
exit,
)?;
}
@@ -391,7 +336,7 @@ impl ComputedVecsFromTxindex<Bitcoin> {
sats.height
.unwrap_cumulative()
.into_iter()
.unwrap_get_inner(height),
.unsafe_get(height),
),
exit,
)?;
@@ -433,17 +378,12 @@ impl ComputedVecsFromTxindex<Dollars> {
(starting_index.to_usize()..indexer.vecs.height_to_weight.len())
.map(Height::from)
.try_for_each(|height| -> Result<()> {
let price = *close_iter.unwrap_get_inner(height);
let price = *close_iter.unsafe_get(height);
if let Some(first) = self.height.first.as_mut() {
first.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_first()
.into_iter()
.unwrap_get_inner(height),
price * bitcoin.height.unwrap_first().into_iter().unsafe_get(height),
exit,
)?;
}
@@ -455,55 +395,35 @@ impl ComputedVecsFromTxindex<Dollars> {
.height
.unwrap_average()
.into_iter()
.unwrap_get_inner(height),
.unsafe_get(height),
exit,
)?;
}
if let Some(sum) = self.height.sum.as_mut() {
sum.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_sum()
.into_iter()
.unwrap_get_inner(height),
price * bitcoin.height.unwrap_sum().into_iter().unsafe_get(height),
exit,
)?;
}
if let Some(max) = self.height.max.as_mut() {
max.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_max()
.into_iter()
.unwrap_get_inner(height),
price * bitcoin.height.unwrap_max().into_iter().unsafe_get(height),
exit,
)?;
}
if let Some(pct90) = self.height.pct90.as_mut() {
pct90.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_pct90()
.into_iter()
.unwrap_get_inner(height),
price * bitcoin.height.unwrap_pct90().into_iter().unsafe_get(height),
exit,
)?;
}
if let Some(pct75) = self.height.pct75.as_mut() {
pct75.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_pct75()
.into_iter()
.unwrap_get_inner(height),
price * bitcoin.height.unwrap_pct75().into_iter().unsafe_get(height),
exit,
)?;
}
@@ -515,55 +435,35 @@ impl ComputedVecsFromTxindex<Dollars> {
.height
.unwrap_median()
.into_iter()
.unwrap_get_inner(height),
.unsafe_get(height),
exit,
)?;
}
if let Some(pct25) = self.height.pct25.as_mut() {
pct25.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_pct25()
.into_iter()
.unwrap_get_inner(height),
price * bitcoin.height.unwrap_pct25().into_iter().unsafe_get(height),
exit,
)?;
}
if let Some(pct10) = self.height.pct10.as_mut() {
pct10.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_pct10()
.into_iter()
.unwrap_get_inner(height),
price * bitcoin.height.unwrap_pct10().into_iter().unsafe_get(height),
exit,
)?;
}
if let Some(min) = self.height.min.as_mut() {
min.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_min()
.into_iter()
.unwrap_get_inner(height),
price * bitcoin.height.unwrap_min().into_iter().unsafe_get(height),
exit,
)?;
}
if let Some(last) = self.height.last.as_mut() {
last.forced_push_at(
height,
price
* bitcoin
.height
.unwrap_last()
.into_iter()
.unwrap_get_inner(height),
price * bitcoin.height.unwrap_last().into_iter().unsafe_get(height),
exit,
)?;
}
@@ -575,7 +475,7 @@ impl ComputedVecsFromTxindex<Dollars> {
.height
.unwrap_cumulative()
.into_iter()
.unwrap_get_inner(height),
.unsafe_get(height),
exit,
)?;
}

8
crates/brk_computer/src/grouped/ratio_from_dateindex.rs
View File

@@ -394,7 +394,9 @@ impl ComputedRatioVecsFromDateIndex {
.dateindex
.as_ref()
.unwrap()
.iter_at(starting_dateindex)
.iter()
.skip(starting_dateindex)
.enumerate()
.try_for_each(|(index, ratio)| -> Result<()> {
if index < min_ratio_date {
self.ratio_pct5
@@ -545,7 +547,7 @@ impl ComputedRatioVecsFromDateIndex {
starting_indexes.dateindex,
date_to_price,
|(i, price, ..)| {
let multiplier = iter.unwrap_get_inner(i);
let multiplier = iter.unsafe_get(i);
(i, price * multiplier)
},
exit,
@@ -562,7 +564,7 @@ impl ComputedRatioVecsFromDateIndex {
starting_indexes.dateindex,
date_to_price,
|(i, price, ..)| {
let multiplier = iter.unwrap_get_inner(i);
let multiplier = iter.unsafe_get(i);
(i, price * multiplier)
},
exit,

8
crates/brk_computer/src/grouped/sd_from_dateindex.rs
View File

@@ -499,7 +499,9 @@ impl ComputedStandardDeviationVecsFromDateIndex {
let mut m3sd = self.m3sd.as_mut().map(|c| c.dateindex.as_mut().unwrap());
source
.iter_at(starting_dateindex)
.iter()
.skip(starting_dateindex)
.enumerate()
.try_for_each(|(index, ratio)| -> Result<()> {
if index < min_date {
self.sd.dateindex.as_mut().unwrap().forced_push_at(
@@ -548,7 +550,7 @@ impl ComputedStandardDeviationVecsFromDateIndex {
let pos = sorted.binary_search(&ratio).unwrap_or_else(|pos| pos);
sorted.insert(pos, ratio);
let avg = sma_iter.unwrap_get_inner(index);
let avg = sma_iter.unsafe_get(index);
let population = index.checked_sub(min_date).unwrap().to_usize() as f32 + 1.0;
@@ -637,7 +639,7 @@ impl ComputedStandardDeviationVecsFromDateIndex {
starting_indexes.dateindex,
price,
|(i, price, ..)| {
let multiplier = iter.unwrap_get_inner(i);
let multiplier = iter.unsafe_get(i);
(i, price * multiplier)
},
exit,

12
crates/brk_computer/src/indexes.rs
View File

@@ -509,7 +509,7 @@ impl Vecs {
exit: &Exit,
) -> Result<Indexes> {
let idxs = self.compute_(indexer, starting_indexes, exit)?;
self.db.flush_then_punch()?;
self.db.compact()?;
Ok(idxs)
}
@@ -558,7 +558,7 @@ impl Vecs {
prev_timestamp_fixed.replace(
height_to_timestamp_fixed_iter
.into_iter()
.unwrap_get_inner(prev_h),
.unsafe_get(prev_h),
);
}
let timestamp_fixed =
@@ -946,12 +946,8 @@ pub struct Indexes {
impl Indexes {
pub fn update_from_height(&mut self, height: Height, indexes: &Vecs) {
self.indexes.height = height;
self.dateindex = DateIndex::try_from(
indexes
.height_to_date_fixed
.into_iter()
.unwrap_get_inner(height),
)
self.dateindex =
DateIndex::try_from(indexes.height_to_date_fixed.into_iter().unsafe_get(height))
.unwrap();
self.weekindex = WeekIndex::from(self.dateindex);
self.monthindex = MonthIndex::from(self.dateindex);

138
crates/brk_computer/src/lib.rs
View File

@@ -12,38 +12,38 @@ use log::info;
use vecdb::{Exit, Format};
mod blks;
mod chain;
mod cointime;
mod constants;
// mod chain;
// mod cointime;
// mod constants;
mod fetched;
mod grouped;
// mod grouped;
mod indexes;
mod market;
mod pools;
mod price;
mod stateful;
// mod market;
// mod pools;
// mod price;
// mod stateful;
mod states;
mod traits;
mod utils;
use indexes::Indexes;
pub use pools::*;
// pub use pools::*;
pub use states::PriceToAmount;
use states::*;
#[derive(Clone, Traversable)]
pub struct Computer {
pub chain: chain::Vecs,
pub cointime: cointime::Vecs,
pub constants: constants::Vecs,
pub blks: blks::Vecs,
// pub chain: chain::Vecs,
// pub cointime: cointime::Vecs,
// pub constants: constants::Vecs,
pub fetched: Option<fetched::Vecs>,
pub indexes: indexes::Vecs,
pub market: market::Vecs,
pub pools: pools::Vecs,
pub blks: blks::Vecs,
pub price: Option<price::Vecs>,
pub stateful: stateful::Vecs,
// pub market: market::Vecs,
// pub pools: pools::Vecs,
// pub price: Option<price::Vecs>,
// pub stateful: stateful::Vecs,
}
const VERSION: Version = Version::new(4);
@@ -81,69 +81,69 @@ impl Computer {
Ok((indexes, fetched, blks))
})?;
let (price, constants, market) = thread::scope(|s| -> Result<_> {
let constants_handle = big_thread().spawn_scoped(s, || {
constants::Vecs::forced_import(&computed_path, VERSION, &indexes)
})?;
// let (price, constants, market) = thread::scope(|s| -> Result<_> {
// let constants_handle = big_thread().spawn_scoped(s, || {
// constants::Vecs::forced_import(&computed_path, VERSION, &indexes)
// })?;
let market_handle = big_thread().spawn_scoped(s, || {
market::Vecs::forced_import(&computed_path, VERSION, &indexes)
})?;
// let market_handle = big_thread().spawn_scoped(s, || {
// market::Vecs::forced_import(&computed_path, VERSION, &indexes)
// })?;
let price = fetched
.is_some()
.then(|| price::Vecs::forced_import(&computed_path, VERSION, &indexes).unwrap());
// let price = fetched
// .is_some()
// .then(|| price::Vecs::forced_import(&computed_path, VERSION, &indexes).unwrap());
let constants = constants_handle.join().unwrap()?;
let market = market_handle.join().unwrap()?;
// let constants = constants_handle.join().unwrap()?;
// let market = market_handle.join().unwrap()?;
Ok((price, constants, market))
})?;
// Ok((price, constants, market))
// })?;
let (chain, pools, cointime) = thread::scope(|s| -> Result<_> {
let chain_handle = big_thread().spawn_scoped(s, || {
chain::Vecs::forced_import(
&computed_path,
VERSION,
indexer,
&indexes,
price.as_ref(),
)
})?;
// let (chain, pools, cointime) = thread::scope(|s| -> Result<_> {
// let chain_handle = big_thread().spawn_scoped(s, || {
// chain::Vecs::forced_import(
// &computed_path,
// VERSION,
// indexer,
// &indexes,
// price.as_ref(),
// )
// })?;
let pools_handle = big_thread().spawn_scoped(s, || {
pools::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref())
})?;
// let pools_handle = big_thread().spawn_scoped(s, || {
// pools::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref())
// })?;
let cointime =
cointime::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref())?;
// let cointime =
// cointime::Vecs::forced_import(&computed_path, VERSION, &indexes, price.as_ref())?;
let chain = chain_handle.join().unwrap()?;
let pools = pools_handle.join().unwrap()?;
// let chain = chain_handle.join().unwrap()?;
// let pools = pools_handle.join().unwrap()?;
Ok((chain, pools, cointime))
})?;
// Ok((chain, pools, cointime))
// })?;
// Threads inside
let stateful = stateful::Vecs::forced_import(
&computed_path,
VERSION,
Format::Compressed,
&indexes,
price.as_ref(),
)?;
// // Threads inside
// let stateful = stateful::Vecs::forced_import(
// &computed_path,
// VERSION,
// Format::Compressed,
// &indexes,
// price.as_ref(),
// )?;
Ok(Self {
constants,
market,
stateful,
chain,
// constants,
// market,
// stateful,
// chain,
blks,
pools,
cointime,
// pools,
// cointime,
indexes,
fetched,
price,
// price,
})
}
@@ -182,13 +182,6 @@ impl Computer {
Ok(())
});
// let blks = scope.spawn(|| -> Result<()> {
// info!("Computing blks...");
// self.blks
// .compute(indexer, &self.indexes, &starting_indexes, parser, exit)?;
// Ok(())
// });
let chain = scope.spawn(|| -> Result<()> {
info!("Computing chain...");
self.chain.compute(
@@ -207,7 +200,6 @@ impl Computer {
}
constants.join().unwrap()?;
// blks.join().unwrap()?;
chain.join().unwrap()?;
Ok(())
})?;

13
crates/brk_computer/src/market.rs
View File

@@ -3,7 +3,10 @@ use std::{path::Path, thread};
use brk_error::Result;
use brk_traversable::Traversable;
use brk_types::{Date, DateIndex, Dollars, Height, Sats, StoredF32, StoredU16, Version};
use vecdb::{Database, EagerVec, Exit, PAGE_SIZE, StoredIndex, VecIterator};
use vecdb::{
Database, EagerVec, Exit, GenericStoredVec, PAGE_SIZE, StoredIndex, VecIterator,
VecIteratorExtended,
};
use crate::{
grouped::{ComputedStandardDeviationVecsFromDateIndex, Source, StandardDeviationVecsOptions},
@@ -1532,7 +1535,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(price, starting_indexes, exit)?;
self.db.flush_then_punch()?;
self.db.compact()?;
Ok(())
}
@@ -1591,12 +1594,12 @@ impl Vecs {
if prev.is_none() {
let i = i.to_usize();
prev.replace(if i > 0 {
slf.into_iter().unwrap_get_inner_(i - 1)
slf.one_shot_get_any_or_read_(i - 1).unwrap().unwrap()
} else {
StoredU16::default()
});
}
let days = if *high_iter.unwrap_get_inner(i) == ath {
let days = if *high_iter.unsafe_get(i) == ath {
StoredU16::default()
} else {
prev.unwrap() + StoredU16::new(1)
@@ -1622,7 +1625,7 @@ impl Vecs {
if prev.is_none() {
let i = i.to_usize();
prev.replace(if i > 0 {
slf.into_iter().unwrap_get_inner_(i - 1)
slf.one_shot_get_any_or_read_(i - 1)
} else {
StoredU16::ZERO
});

35
crates/brk_computer/src/pools/mod.rs
View File

@@ -81,7 +81,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, chain, price, exit)?;
self.db.flush_then_punch()?;
self.db.compact()?;
Ok(())
}
@@ -155,47 +155,40 @@ impl Vecs {
.iter()
.skip(min)
.try_for_each(|(height, coinbase_tag)| -> Result<()> {
let txindex = height_to_first_txindex_iter.unwrap_get_inner(height);
let txoutindex = txindex_to_first_txoutindex_iter.unwrap_get_inner(txindex);
let outputcount = txindex_to_output_count_iter.unwrap_get_inner(txindex);
let txindex = height_to_first_txindex_iter.unsafe_get(height);
let txoutindex = txindex_to_first_txoutindex_iter.unsafe_get(txindex);
let outputcount = txindex_to_output_count_iter.unsafe_get(txindex);
let pool = (*txoutindex..(*txoutindex + *outputcount))
.map(TxOutIndex::from)
.find_map(|txoutindex| {
let outputtype = txoutindex_to_outputtype_iter.unwrap_get_inner(txoutindex);
let typeindex = txoutindex_to_typeindex_iter.unwrap_get_inner(txoutindex);
let outputtype = txoutindex_to_outputtype_iter.unsafe_get(txoutindex);
let typeindex = txoutindex_to_typeindex_iter.unsafe_get(txoutindex);
match outputtype {
OutputType::P2PK65 => Some(AddressBytes::from(
p2pk65addressindex_to_p2pk65bytes_iter
.unwrap_get_inner(typeindex.into()),
p2pk65addressindex_to_p2pk65bytes_iter.unsafe_get(typeindex.into()),
)),
OutputType::P2PK33 => Some(AddressBytes::from(
p2pk33addressindex_to_p2pk33bytes_iter
.unwrap_get_inner(typeindex.into()),
p2pk33addressindex_to_p2pk33bytes_iter.unsafe_get(typeindex.into()),
)),
OutputType::P2PKH => Some(AddressBytes::from(
p2pkhaddressindex_to_p2pkhbytes_iter
.unwrap_get_inner(typeindex.into()),
p2pkhaddressindex_to_p2pkhbytes_iter.unsafe_get(typeindex.into()),
)),
OutputType::P2SH => Some(AddressBytes::from(
p2shaddressindex_to_p2shbytes_iter
.unwrap_get_inner(typeindex.into()),
p2shaddressindex_to_p2shbytes_iter.unsafe_get(typeindex.into()),
)),
OutputType::P2WPKH => Some(AddressBytes::from(
p2wpkhaddressindex_to_p2wpkhbytes_iter
.unwrap_get_inner(typeindex.into()),
p2wpkhaddressindex_to_p2wpkhbytes_iter.unsafe_get(typeindex.into()),
)),
OutputType::P2WSH => Some(AddressBytes::from(
p2wshaddressindex_to_p2wshbytes_iter
.unwrap_get_inner(typeindex.into()),
p2wshaddressindex_to_p2wshbytes_iter.unsafe_get(typeindex.into()),
)),
OutputType::P2TR => Some(AddressBytes::from(
p2traddressindex_to_p2trbytes_iter
.unwrap_get_inner(typeindex.into()),
p2traddressindex_to_p2trbytes_iter.unsafe_get(typeindex.into()),
)),
OutputType::P2A => Some(AddressBytes::from(
p2aaddressindex_to_p2abytes_iter.unwrap_get_inner(typeindex.into()),
p2aaddressindex_to_p2abytes_iter.unsafe_get(typeindex.into()),
)),
_ => None,
}

2
crates/brk_computer/src/pools/vecs.rs
View File

@@ -354,7 +354,7 @@ impl Vecs {
if prev.is_none() {
let i = i.to_usize();
prev.replace(if i > 0 {
slf.into_iter().unwrap_get_inner_(i - 1)
slf.one_shot_get_any_or_read_(i - 1)
} else {
StoredU16::ZERO
});

188
crates/brk_computer/src/price.rs
View File

@@ -8,7 +8,7 @@ use brk_types::{
};
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, PAGE_SIZE,
RawVec,
RawVec, StoredIndex,
};
use crate::{fetched, grouped::Source};
@@ -343,7 +343,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexes, starting_indexes, fetched, exit)?;
self.db.flush_then_punch()?;
self.db.compact()?;
Ok(())
}
@@ -387,10 +387,15 @@ impl Vecs {
.min(Height::from(self.height_to_price_ohlc.len()));
fetched
.height_to_price_ohlc_in_cents
.iter_at(index)
.iter()?
.skip(index.to_usize())
.enumerate()
.try_for_each(|(i, v)| -> Result<()> {
self.height_to_price_ohlc
.forced_push_at(i, OHLCDollars::from(v), exit)?;
self.height_to_price_ohlc.forced_push_at(
Height::from(i),
OHLCDollars::from(v),
exit,
)?;
Ok(())
})?;
self.height_to_price_ohlc.safe_flush(exit)?;
@@ -428,10 +433,15 @@ impl Vecs {
.min(DateIndex::from(self.dateindex_to_price_ohlc.len()));
fetched
.dateindex_to_price_ohlc_in_cents
.iter_at(index)
.iter()?
.skip(index.to_usize())
.enumerate()
.try_for_each(|(i, v)| -> Result<()> {
self.dateindex_to_price_ohlc
.forced_push_at(i, OHLCDollars::from(v), exit)?;
self.dateindex_to_price_ohlc.forced_push_at(
DateIndex::from(i),
OHLCDollars::from(v),
exit,
)?;
Ok(())
})?;
self.dateindex_to_price_ohlc.safe_flush(exit)?;
@@ -537,13 +547,15 @@ impl Vecs {
self.timeindexes_to_price_close
.weekindex
.unwrap_last()
.iter_at(index)
.iter()
.skip(index.to_usize())
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
let open = weekindex_first_iter.unwrap_get_inner(i);
let high = weekindex_max_iter.unwrap_get_inner(i);
let low = weekindex_min_iter.unwrap_get_inner(i);
let open = weekindex_first_iter.unsafe_get_(i);
let high = weekindex_max_iter.unsafe_get_(i);
let low = weekindex_min_iter.unsafe_get_(i);
self.weekindex_to_price_ohlc.forced_push_at(
i,
WeekIndex::from(i),
OHLCDollars {
open,
high,
@@ -577,11 +589,13 @@ impl Vecs {
self.chainindexes_to_price_close
.difficultyepoch
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
let open = difficultyepoch_first_iter.unwrap_get_inner(i);
let high = difficultyepoch_max_iter.unwrap_get_inner(i);
let low = difficultyepoch_min_iter.unwrap_get_inner(i);
let open = difficultyepoch_first_iter.unsafe_get(i);
let high = difficultyepoch_max_iter.unsafe_get(i);
let low = difficultyepoch_min_iter.unsafe_get(i);
self.difficultyepoch_to_price_ohlc.forced_push_at(
i,
OHLCDollars {
@@ -613,11 +627,13 @@ impl Vecs {
self.timeindexes_to_price_close
.monthindex
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
let open = monthindex_first_iter.unwrap_get_inner(i);
let high = monthindex_max_iter.unwrap_get_inner(i);
let low = monthindex_min_iter.unwrap_get_inner(i);
let open = monthindex_first_iter.unsafe_get(i);
let high = monthindex_max_iter.unsafe_get(i);
let low = monthindex_min_iter.unsafe_get(i);
self.monthindex_to_price_ohlc.forced_push_at(
i,
OHLCDollars {
@@ -653,11 +669,13 @@ impl Vecs {
self.timeindexes_to_price_close
.quarterindex
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
let open = quarterindex_first_iter.unwrap_get_inner(i);
let high = quarterindex_max_iter.unwrap_get_inner(i);
let low = quarterindex_min_iter.unwrap_get_inner(i);
let open = quarterindex_first_iter.unsafe_get(i);
let high = quarterindex_max_iter.unsafe_get(i);
let low = quarterindex_min_iter.unsafe_get(i);
self.quarterindex_to_price_ohlc.forced_push_at(
i,
OHLCDollars {
@@ -693,11 +711,13 @@ impl Vecs {
self.timeindexes_to_price_close
.semesterindex
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
let open = semesterindex_first_iter.unwrap_get_inner(i);
let high = semesterindex_max_iter.unwrap_get_inner(i);
let low = semesterindex_min_iter.unwrap_get_inner(i);
let open = semesterindex_first_iter.unsafe_get(i);
let high = semesterindex_max_iter.unsafe_get(i);
let low = semesterindex_min_iter.unsafe_get(i);
self.semesterindex_to_price_ohlc.forced_push_at(
i,
OHLCDollars {
@@ -725,11 +745,13 @@ impl Vecs {
self.timeindexes_to_price_close
.yearindex
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
let open = yearindex_first_iter.unwrap_get_inner(i);
let high = yearindex_max_iter.unwrap_get_inner(i);
let low = yearindex_min_iter.unwrap_get_inner(i);
let open = yearindex_first_iter.unsafe_get(i);
let high = yearindex_max_iter.unsafe_get(i);
let low = yearindex_min_iter.unsafe_get(i);
self.yearindex_to_price_ohlc.forced_push_at(
i,
OHLCDollars {
@@ -768,11 +790,13 @@ impl Vecs {
self.timeindexes_to_price_close
.decadeindex
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
let open = decadeindex_first_iter.unwrap_get_inner(i);
let high = decadeindex_max_iter.unwrap_get_inner(i);
let low = decadeindex_min_iter.unwrap_get_inner(i);
let open = decadeindex_first_iter.unsafe_get(i);
let high = decadeindex_max_iter.unsafe_get(i);
let low = decadeindex_min_iter.unsafe_get(i);
self.decadeindex_to_price_ohlc.forced_push_at(
i,
OHLCDollars {
@@ -883,14 +907,16 @@ impl Vecs {
.min(Height::from(self.height_to_price_ohlc_in_sats.len()));
self.chainindexes_to_price_close_in_sats
.height
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
self.height_to_price_ohlc_in_sats.forced_push_at(
i,
OHLCSats {
open: height_first_iter.unwrap_get_inner(i),
high: height_max_iter.unwrap_get_inner(i),
low: height_min_iter.unwrap_get_inner(i),
open: height_first_iter.unsafe_get(i),
high: height_max_iter.unsafe_get(i),
low: height_min_iter.unsafe_get(i),
close,
},
exit,
@@ -924,14 +950,16 @@ impl Vecs {
.dateindex
.as_ref()
.unwrap()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
self.dateindex_to_price_ohlc_in_sats.forced_push_at(
i,
OHLCSats {
open: dateindex_first_iter.unwrap_get_inner(i),
high: dateindex_max_iter.unwrap_get_inner(i),
low: dateindex_min_iter.unwrap_get_inner(i),
open: dateindex_first_iter.unsafe_get(i),
high: dateindex_max_iter.unsafe_get(i),
low: dateindex_min_iter.unsafe_get(i),
close,
},
exit,
@@ -961,14 +989,16 @@ impl Vecs {
self.timeindexes_to_price_close_in_sats
.weekindex
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
self.weekindex_to_price_ohlc_in_sats.forced_push_at(
i,
OHLCSats {
open: weekindex_first_iter.unwrap_get_inner(i),
high: weekindex_max_iter.unwrap_get_inner(i),
low: weekindex_min_iter.unwrap_get_inner(i),
open: weekindex_first_iter.unsafe_get(i),
high: weekindex_max_iter.unsafe_get(i),
low: weekindex_min_iter.unsafe_get(i),
close,
},
exit,
@@ -998,14 +1028,16 @@ impl Vecs {
self.chainindexes_to_price_close_in_sats
.difficultyepoch
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
self.difficultyepoch_to_price_ohlc_in_sats.forced_push_at(
i,
OHLCSats {
open: difficultyepoch_first_iter.unwrap_get_inner(i),
high: difficultyepoch_max_iter.unwrap_get_inner(i),
low: difficultyepoch_min_iter.unwrap_get_inner(i),
open: difficultyepoch_first_iter.unsafe_get(i),
high: difficultyepoch_max_iter.unsafe_get(i),
low: difficultyepoch_min_iter.unsafe_get(i),
close,
},
exit,
@@ -1036,14 +1068,16 @@ impl Vecs {
self.timeindexes_to_price_close_in_sats
.monthindex
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
self.monthindex_to_price_ohlc_in_sats.forced_push_at(
i,
OHLCSats {
open: monthindex_first_iter.unwrap_get_inner(i),
high: monthindex_max_iter.unwrap_get_inner(i),
low: monthindex_min_iter.unwrap_get_inner(i),
open: monthindex_first_iter.unsafe_get(i),
high: monthindex_max_iter.unsafe_get(i),
low: monthindex_min_iter.unsafe_get(i),
close,
},
exit,
@@ -1073,14 +1107,16 @@ impl Vecs {
self.timeindexes_to_price_close_in_sats
.quarterindex
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
self.quarterindex_to_price_ohlc_in_sats.forced_push_at(
i,
OHLCSats {
open: quarterindex_first_iter.unwrap_get_inner(i),
high: quarterindex_max_iter.unwrap_get_inner(i),
low: quarterindex_min_iter.unwrap_get_inner(i),
open: quarterindex_first_iter.unsafe_get(i),
high: quarterindex_max_iter.unsafe_get(i),
low: quarterindex_min_iter.unsafe_get(i),
close,
},
exit,
@@ -1110,14 +1146,16 @@ impl Vecs {
self.timeindexes_to_price_close_in_sats
.semesterindex
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
self.semesterindex_to_price_ohlc_in_sats.forced_push_at(
i,
OHLCSats {
open: semesterindex_first_iter.unwrap_get_inner(i),
high: semesterindex_max_iter.unwrap_get_inner(i),
low: semesterindex_min_iter.unwrap_get_inner(i),
open: semesterindex_first_iter.unsafe_get(i),
high: semesterindex_max_iter.unsafe_get(i),
low: semesterindex_min_iter.unsafe_get(i),
close,
},
exit,
@@ -1147,14 +1185,16 @@ impl Vecs {
self.timeindexes_to_price_close_in_sats
.yearindex
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
self.yearindex_to_price_ohlc_in_sats.forced_push_at(
i,
OHLCSats {
open: yearindex_first_iter.unwrap_get_inner(i),
high: yearindex_max_iter.unwrap_get_inner(i),
low: yearindex_min_iter.unwrap_get_inner(i),
open: yearindex_first_iter.unsafe_get(i),
high: yearindex_max_iter.unsafe_get(i),
low: yearindex_min_iter.unsafe_get(i),
close,
},
exit,
@@ -1187,14 +1227,16 @@ impl Vecs {
self.timeindexes_to_price_close_in_sats
.decadeindex
.unwrap_last()
.iter_at(index)
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, close)| -> Result<()> {
self.decadeindex_to_price_ohlc_in_sats.forced_push_at(
i,
OHLCSats {
open: decadeindex_first_iter.unwrap_get_inner(i),
high: decadeindex_max_iter.unwrap_get_inner(i),
low: decadeindex_min_iter.unwrap_get_inner(i),
open: decadeindex_first_iter.unsafe_get(i),
high: decadeindex_max_iter.unsafe_get(i),
low: decadeindex_min_iter.unsafe_get(i),
close,
},
exit,

2
crates/brk_computer/src/stateful/address_cohort.rs
View File

@@ -112,7 +112,7 @@ impl DynCohortVecs for Vecs {
self.state.as_mut().unwrap().addr_count = *self
.height_to_addr_count
.into_iter()
.unwrap_get_inner(prev_height);
.unsafe_get(prev_height);
}
Ok(starting_height)

36
crates/brk_computer/src/stateful/addresstype/addresscount.rs
View File

@@ -13,34 +13,14 @@ impl From<(&AddressTypeToHeightToAddressCount, Height)> for AddressTypeToAddress
fn from((groups, starting_height): (&AddressTypeToHeightToAddressCount, Height)) -> Self {
if let Some(prev_height) = starting_height.decremented() {
Self(ByAddressType {
p2pk65: groups
.p2pk65
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2pk33: groups
.p2pk33
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2pkh: groups
.p2pkh
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2sh: groups.p2sh.into_iter().unwrap_get_inner(prev_height).into(),
p2wpkh: groups
.p2wpkh
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2wsh: groups
.p2wsh
.into_iter()
.unwrap_get_inner(prev_height)
.into(),
p2tr: groups.p2tr.into_iter().unwrap_get_inner(prev_height).into(),
p2a: groups.p2a.into_iter().unwrap_get_inner(prev_height).into(),
p2pk65: groups.p2pk65.into_iter().unsafe_get(prev_height).into(),
p2pk33: groups.p2pk33.into_iter().unsafe_get(prev_height).into(),
p2pkh: groups.p2pkh.into_iter().unsafe_get(prev_height).into(),
p2sh: groups.p2sh.into_iter().unsafe_get(prev_height).into(),
p2wpkh: groups.p2wpkh.into_iter().unsafe_get(prev_height).into(),
p2wsh: groups.p2wsh.into_iter().unsafe_get(prev_height).into(),
p2tr: groups.p2tr.into_iter().unsafe_get(prev_height).into(),
p2a: groups.p2a.into_iter().unsafe_get(prev_height).into(),
})
} else {
Default::default()

16
crates/brk_computer/src/stateful/common.rs
View File

@@ -1338,19 +1338,15 @@ impl Vecs {
prev_height = state.import_at_or_before(prev_height)?;
}
state.supply.value = self
.height_to_supply
.into_iter()
.unwrap_get_inner(prev_height);
state.supply.value = self.height_to_supply.into_iter().unsafe_get(prev_height);
state.supply.utxo_count = *self
.height_to_utxo_count
.into_iter()
.unwrap_get_inner(prev_height);
.unsafe_get(prev_height);
if let Some(height_to_realized_cap) = self.height_to_realized_cap.as_mut() {
state.realized.as_mut().unwrap().cap = height_to_realized_cap
.into_iter()
.unwrap_get_inner(prev_height);
state.realized.as_mut().unwrap().cap =
height_to_realized_cap.into_iter().unsafe_get(prev_height);
}
Ok(prev_height.incremented())
@@ -2106,11 +2102,11 @@ impl Vecs {
starting_indexes.dateindex,
&indexes.dateindex_to_first_height,
|(i, height, ..)| {
let count = dateindex_to_height_count_iter.unwrap_get_inner(i);
let count = dateindex_to_height_count_iter.unsafe_get(i);
if count == StoredU64::default() {
unreachable!()
}
let supply = height_to_supply_iter.unwrap_get_inner(height + (*count - 1));
let supply = height_to_supply_iter.unsafe_get(height + (*count - 1));
(i, supply)
},
exit,

58
crates/brk_computer/src/stateful/mod.rs
View File

@@ -502,7 +502,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, chain, price, starting_indexes, exit)?;
self.db.flush_then_punch()?;
self.db.compact()?;
Ok(())
}
@@ -712,10 +712,10 @@ impl Vecs {
.enumerate()
.map(|(height, supply)| {
let height = Height::from(height);
let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height);
let timestamp = height_to_timestamp_fixed_iter.unsafe_get(height);
let price = height_to_price_close_iter
.as_mut()
.map(|i| *i.unwrap_get_inner(height));
.map(|i| *i.unsafe_get(height));
BlockState {
timestamp,
price,
@@ -794,14 +794,14 @@ impl Vecs {
let mut unspendable_supply = if let Some(prev_height) = starting_height.decremented() {
self.height_to_unspendable_supply
.into_iter()
.unwrap_get_inner(prev_height)
.unsafe_get(prev_height)
} else {
Sats::ZERO
};
let mut opreturn_supply = if let Some(prev_height) = starting_height.decremented() {
self.height_to_opreturn_supply
.into_iter()
.unwrap_get_inner(prev_height)
.unsafe_get(prev_height)
} else {
Sats::ZERO
};
@@ -847,20 +847,18 @@ impl Vecs {
v.state.as_mut().unwrap().reset_single_iteration_values()
});
let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height);
let timestamp = height_to_timestamp_fixed_iter.unsafe_get(height);
let price = height_to_price_close_iter
.as_mut()
.map(|i| *i.unwrap_get_inner(height));
let first_txindex = height_to_first_txindex_iter.unwrap_get_inner(height);
.map(|i| *i.unsafe_get(height));
let first_txindex = height_to_first_txindex_iter.unsafe_get(height);
let first_txoutindex = height_to_first_txoutindex_iter
.unwrap_get_inner(height)
.unsafe_get(height)
.to_usize();
let first_txinindex = height_to_first_txinindex_iter
.unwrap_get_inner(height)
.to_usize();
let tx_count = height_to_tx_count_iter.unwrap_get_inner(height);
let output_count = height_to_output_count_iter.unwrap_get_inner(height);
let input_count = height_to_input_count_iter.unwrap_get_inner(height);
let first_txinindex = height_to_first_txinindex_iter.unsafe_get(height).to_usize();
let tx_count = height_to_tx_count_iter.unsafe_get(height);
let output_count = height_to_output_count_iter.unsafe_get(height);
let input_count = height_to_input_count_iter.unsafe_get(height);
let txoutindex_to_txindex = build_txoutindex_to_txindex(
first_txindex,
@@ -876,28 +874,28 @@ impl Vecs {
let first_addressindexes: ByAddressType<TypeIndex> = ByAddressType {
p2a: height_to_first_p2aaddressindex_iter
.unwrap_get_inner(height)
.unsafe_get(height)
.into(),
p2pk33: height_to_first_p2pk33addressindex_iter
.unwrap_get_inner(height)
.unsafe_get(height)
.into(),
p2pk65: height_to_first_p2pk65addressindex_iter
.unwrap_get_inner(height)
.unsafe_get(height)
.into(),
p2pkh: height_to_first_p2pkhaddressindex_iter
.unwrap_get_inner(height)
.unsafe_get(height)
.into(),
p2sh: height_to_first_p2shaddressindex_iter
.unwrap_get_inner(height)
.unsafe_get(height)
.into(),
p2tr: height_to_first_p2traddressindex_iter
.unwrap_get_inner(height)
.unsafe_get(height)
.into(),
p2wpkh: height_to_first_p2wpkhaddressindex_iter
.unwrap_get_inner(height)
.unsafe_get(height)
.into(),
p2wsh: height_to_first_p2wshaddressindex_iter
.unwrap_get_inner(height)
.unsafe_get(height)
.into(),
};
@@ -1211,7 +1209,7 @@ impl Vecs {
.into_iter()
.map(|state| state.value)
.sum::<Sats>()
+ height_to_unclaimed_rewards_iter.unwrap_get_inner(height);
+ height_to_unclaimed_rewards_iter.unsafe_get(height);
opreturn_supply += transacted.by_type.unspendable.opreturn.value;
@@ -1258,16 +1256,16 @@ impl Vecs {
self.addresstype_to_height_to_empty_addr_count
.forced_push_at(height, &addresstype_to_empty_addr_count, exit)?;
let date = height_to_date_fixed_iter.unwrap_get_inner(height);
let date = height_to_date_fixed_iter.unsafe_get(height);
let dateindex = DateIndex::try_from(date).unwrap();
let date_first_height = dateindex_to_first_height_iter.unwrap_get_inner(dateindex);
let date_height_count = dateindex_to_height_count_iter.unwrap_get_inner(dateindex);
let date_first_height = dateindex_to_first_height_iter.unsafe_get(dateindex);
let date_height_count = dateindex_to_height_count_iter.unsafe_get(dateindex);
let is_date_last_height = date_first_height
+ Height::from(date_height_count).decremented().unwrap()
== height;
let date_price = dateindex_to_price_close_iter
.as_mut()
.map(|v| is_date_last_height.then(|| *v.unwrap_get_inner(dateindex)));
.map(|v| is_date_last_height.then(|| *v.unsafe_get(dateindex)));
let dateindex = is_date_last_height.then_some(dateindex);
@@ -2137,7 +2135,7 @@ fn build_txoutindex_to_txindex<'a>(
let block_first_txindex = block_first_txindex.to_usize();
for tx_offset in 0..block_tx_count as usize {
let txindex = TxIndex::from(block_first_txindex + tx_offset);
let output_count = u64::from(txindex_to_output_count.unwrap_get_inner(txindex));
let output_count = u64::from(txindex_to_output_count.unsafe_get(txindex));
for _ in 0..output_count {
vec.push(txindex);
@@ -2157,7 +2155,7 @@ fn build_txinindex_to_txindex<'a>(
let block_first_txindex = block_first_txindex.to_usize();
for tx_offset in 0..block_tx_count as usize {
let txindex = TxIndex::from(block_first_txindex + tx_offset);
let input_count = u64::from(txindex_to_input_count.unwrap_get_inner(txindex));
let input_count = u64::from(txindex_to_input_count.unsafe_get(txindex));
for _ in 0..input_count {
vec.push(txindex);

52
crates/brk_computer/src/traits.rs
View File

@@ -39,15 +39,19 @@ impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
let mut other_iter = closes.iter();
let mut prev = None;
let index = max_from.min(DateIndex::from(self.len()));
closes.iter_at(index).try_for_each(|(i, closes)| {
let index = max_from.to_usize().min(self.len());
closes
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, closes)| {
let price = *closes;
let i_usize = i.to_usize();
if prev.is_none() {
if i_usize == 0 {
prev.replace(Sats::ZERO);
} else {
prev.replace(self.into_iter().unwrap_get_inner_(i_usize - 1));
prev.replace(self.one_shot_get_any_or_read_(i_usize - 1));
}
}
@@ -57,7 +61,7 @@ impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
stack = prev.unwrap() + Sats::from(Bitcoin::from(DCA_AMOUNT / price));
if i_usize >= len {
let prev_price = *other_iter.unwrap_get_inner_(i_usize - len);
let prev_price = *other_iter.get_(i_usize - len);
if prev_price != Dollars::ZERO {
stack = stack
.checked_sub(Sats::from(Bitcoin::from(DCA_AMOUNT / prev_price)))
@@ -90,14 +94,18 @@ impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
let mut prev = None;
let index = max_from.min(DateIndex::from(self.len()));
closes.iter_at(index).try_for_each(|(i, closes)| {
closes
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, closes)| {
let price = *closes;
let i_usize = i.to_usize();
if prev.is_none() {
if i_usize == 0 {
prev.replace(Sats::ZERO);
} else {
prev.replace(self.into_iter().unwrap_get_inner_(i_usize - 1));
prev.replace(self.one_shot_get_any_or_read_(i_usize - 1));
}
}
@@ -151,7 +159,11 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
let first_price_date = DateIndex::try_from(Date::new(2010, 7, 12)).unwrap();
stacks.iter_at(index).try_for_each(|(i, stack)| {
stacks
.iter()
.skip(index.to_usize())
.enumerate()
.try_for_each(|(i, stack)| {
let mut avg_price = Dollars::from(f64::NAN);
if i > first_price_date {
avg_price = DCA_AMOUNT
@@ -183,7 +195,11 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
let from_usize = from.to_usize();
stacks.iter_at(index).try_for_each(|(i, stack)| {
stacks
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, stack)| {
let mut avg_price = Dollars::from(f64::NAN);
if i >= from {
avg_price = DCA_AMOUNT * (i.to_usize() + 1 - from_usize) / Bitcoin::from(stack);
@@ -220,7 +236,10 @@ where
)?;
let index = max_from.min(I::from(self.len()));
sats.iter_at(index).try_for_each(|(i, sats)| {
sats.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, sats)| {
let (i, v) = (i, Bitcoin::from(sats));
self.forced_push_at(i, v, exit)
})?;
@@ -257,8 +276,12 @@ where
let mut price_iter = price.iter();
let index = max_from.min(I::from(self.len()));
bitcoin.iter_at(index).try_for_each(|(i, bitcoin)| {
let dollars = price_iter.unwrap_get_inner(i);
bitcoin
.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, bitcoin)| {
let dollars = price_iter.unsafe_get(i);
let (i, v) = (i, *dollars * bitcoin);
self.forced_push_at(i, v, exit)
})?;
@@ -295,11 +318,14 @@ where
let index = max_from.min(I::from(self.len()));
let mut close_iter = close.iter();
ath.iter_at(index).try_for_each(|(i, ath)| {
ath.iter()
.skip(index)
.enumerate()
.try_for_each(|(i, ath)| {
if ath == Dollars::ZERO {
self.forced_push_at(i, StoredF32::default(), exit)
} else {
let close = *close_iter.unwrap_get_inner(i);
let close = *close_iter.unsafe_get(i);
let drawdown = StoredF32::from((*ath - *close) / *ath * -100.0);
self.forced_push_at(i, drawdown, exit)
}

2
crates/brk_indexer/Cargo.toml
View File

@@ -24,6 +24,6 @@ fjall2 = { workspace = true }
fjall3 = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }
redb = { workspace = true }
rustc-hash = { workspace = true }
vecdb = { workspace = true }
rand = "0.9.2"

11
crates/brk_indexer/examples/indexer.rs
View File

@@ -1,5 +1,5 @@
use std::{
fs,
env, fs,
path::Path,
thread::sleep,
time::{Duration, Instant},
@@ -15,18 +15,15 @@ use vecdb::Exit;
fn main() -> Result<()> {
brk_logger::init(Some(Path::new(".log")))?;
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
let bitcoin_dir = Client::default_bitcoin_path();
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk");
fs::create_dir_all(&outputs_dir)?;
// let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk");
let client = Client::new(
"http://localhost:8332",
Client::default_url(),
Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?;

75
crates/brk_indexer/examples/indexer_single_vs_multi.rs
View File

@@ -9,7 +9,7 @@ use brk_error::Result;
use brk_indexer::Indexer;
use brk_types::TxInIndex;
use rayon::prelude::*;
use vecdb::{GenericStoredVec, StoredIndex};
use vecdb::{AnyVec, GenericStoredVec, StoredIndex};
fn main() -> Result<()> {
brk_logger::init(Some(Path::new(".log")))?;
@@ -20,8 +20,8 @@ fn main() -> Result<()> {
let indexer = Indexer::forced_import(&outputs_dir)?;
let vecs = indexer.vecs;
let output_len = vecs.txoutindex_to_value.len_();
let input_len = vecs.txinindex_to_outpoint.len_();
let output_len = vecs.txoutindex_to_value.len();
let input_len = vecs.txinindex_to_outpoint.len();
dbg!(output_len, input_len);
// Simulate processing blocks
@@ -244,11 +244,11 @@ fn run_method1(
.map(|i| {
(
vecs.txoutindex_to_value
.unwrap_read_(i, &txoutindex_to_value_reader),
.read_at_unwrap(i, &txoutindex_to_value_reader),
vecs.txoutindex_to_outputtype
.unwrap_read_(i, &txoutindex_to_outputtype_reader),
.read_at_unwrap(i, &txoutindex_to_outputtype_reader),
vecs.txoutindex_to_typeindex
.unwrap_read_(i, &txoutindex_to_typeindex_reader),
.read_at_unwrap(i, &txoutindex_to_typeindex_reader),
)
})
.collect();
@@ -261,20 +261,20 @@ fn run_method1(
.filter_map(|i| {
let outpoint = vecs
.txinindex_to_outpoint
.unwrap_read_(i, &txinindex_to_outpoint_reader);
.read_at_unwrap(i, &txinindex_to_outpoint_reader);
if outpoint.is_coinbase() {
return None;
}
let first_txoutindex = vecs.txindex_to_first_txoutindex.unwrap_read_(
let first_txoutindex = vecs.txindex_to_first_txoutindex.read_at_unwrap(
outpoint.txindex().to_usize(),
&txindex_to_first_txoutindex_reader,
);
let txoutindex = first_txoutindex.to_usize() + usize::from(outpoint.vout());
let value = vecs
.txoutindex_to_value
.unwrap_read_(txoutindex, &txoutindex_to_value_reader);
.read_at_unwrap(txoutindex, &txoutindex_to_value_reader);
Some(u64::from(value))
})
.sum();
@@ -350,14 +350,14 @@ fn run_method2(
return None;
}
let first_txoutindex = vecs.txindex_to_first_txoutindex.unwrap_read_(
let first_txoutindex = vecs.txindex_to_first_txoutindex.read_at_unwrap(
outpoint.txindex().to_usize(),
&txindex_to_first_txoutindex_reader,
);
let txoutindex = first_txoutindex.to_usize() + usize::from(outpoint.vout());
let value = vecs
.txoutindex_to_value
.unwrap_read_(txoutindex, &txoutindex_to_value_reader);
.read_at_unwrap(txoutindex, &txoutindex_to_value_reader);
Some(u64::from(value))
})
.sum();
@@ -447,14 +447,14 @@ fn run_method4(
return None;
}
let first_txoutindex = vecs.txindex_to_first_txoutindex.unwrap_read_(
let first_txoutindex = vecs.txindex_to_first_txoutindex.read_at_unwrap(
outpoint.txindex().to_usize(),
&txindex_to_first_txoutindex_reader,
);
let txoutindex = first_txoutindex.to_usize() + usize::from(outpoint.vout());
let value = vecs
.txoutindex_to_value
.unwrap_read_(txoutindex, &txoutindex_to_value_reader);
.read_at_unwrap(txoutindex, &txoutindex_to_value_reader);
Some(u64::from(value))
})
.sum();
@@ -491,11 +491,11 @@ fn run_method5(
.map(|i| {
(
vecs.txoutindex_to_value
.unwrap_read_(i, &txoutindex_to_value_reader),
.read_at_unwrap(i, &txoutindex_to_value_reader),
vecs.txoutindex_to_outputtype
.unwrap_read_(i, &txoutindex_to_outputtype_reader),
.read_at_unwrap(i, &txoutindex_to_outputtype_reader),
vecs.txoutindex_to_typeindex
.unwrap_read_(i, &txoutindex_to_typeindex_reader),
.read_at_unwrap(i, &txoutindex_to_typeindex_reader),
)
})
.collect();
@@ -509,20 +509,20 @@ fn run_method5(
.filter_map(|i| {
let outpoint = vecs
.txinindex_to_outpoint
.unwrap_read_(i, &txinindex_to_outpoint_reader);
.read_at_unwrap(i, &txinindex_to_outpoint_reader);
if outpoint.is_coinbase() {
return None;
}
let first_txoutindex = vecs.txindex_to_first_txoutindex.unwrap_read_(
let first_txoutindex = vecs.txindex_to_first_txoutindex.read_at_unwrap(
outpoint.txindex().to_usize(),
&txindex_to_first_txoutindex_reader,
);
let txoutindex = first_txoutindex.to_usize() + usize::from(outpoint.vout());
let value = vecs
.txoutindex_to_value
.unwrap_read_(txoutindex, &txoutindex_to_value_reader);
.read_at_unwrap(txoutindex, &txoutindex_to_value_reader);
Some(u64::from(value))
})
.sum();
@@ -585,16 +585,17 @@ fn run_method6(
let txoutindex_to_value_reader = vecs.txoutindex_to_value.create_reader();
// Prefetch all first_txoutindexes in parallel
let first_txoutindexes: Vec<Option<_>> = outpoints
let first_txoutindexes: Vec<Option<_>> =
outpoints
.par_iter()
.map(|op| {
if op.is_coinbase() {
return None;
}
Some(
vecs.txindex_to_first_txoutindex
.unwrap_read_(op.txindex().to_usize(), &txindex_to_first_txoutindex_reader),
)
Some(vecs.txindex_to_first_txoutindex.read_at_unwrap(
op.txindex().to_usize(),
&txindex_to_first_txoutindex_reader,
))
})
.collect();
@@ -607,7 +608,7 @@ fn run_method6(
let txoutindex = first_txoutindex.to_usize() + usize::from(op.vout());
let value = vecs
.txoutindex_to_value
.unwrap_read_(txoutindex, &txoutindex_to_value_reader);
.read_at_unwrap(txoutindex, &txoutindex_to_value_reader);
Some(u64::from(value))
})
.sum();
@@ -648,11 +649,11 @@ fn run_method7(
.map(|i| {
(
vecs.txoutindex_to_value
.unwrap_read_(i, &txoutindex_to_value_reader),
.read_at_unwrap(i, &txoutindex_to_value_reader),
vecs.txoutindex_to_outputtype
.unwrap_read_(i, &txoutindex_to_outputtype_reader),
.read_at_unwrap(i, &txoutindex_to_outputtype_reader),
vecs.txoutindex_to_typeindex
.unwrap_read_(i, &txoutindex_to_typeindex_reader),
.read_at_unwrap(i, &txoutindex_to_typeindex_reader),
)
})
.collect();
@@ -664,20 +665,20 @@ fn run_method7(
.filter_map(|i| {
let outpoint = vecs
.txinindex_to_outpoint
.unwrap_read_(i, &txinindex_to_outpoint_reader);
.read_at_unwrap(i, &txinindex_to_outpoint_reader);
if outpoint.is_coinbase() {
return None;
}
let first_txoutindex = vecs.txindex_to_first_txoutindex.unwrap_read_(
let first_txoutindex = vecs.txindex_to_first_txoutindex.read_at_unwrap(
outpoint.txindex().to_usize(),
&txindex_to_first_txoutindex_reader,
);
let txoutindex = first_txoutindex.to_usize() + usize::from(outpoint.vout());
let value = vecs
.txoutindex_to_value
.unwrap_read_(txoutindex, &txoutindex_to_value_reader);
.read_at_unwrap(txoutindex, &txoutindex_to_value_reader);
Some(u64::from(value))
})
.sum();
@@ -720,11 +721,11 @@ fn run_method8(
let i = block_start + offset;
(
vecs.txoutindex_to_value
.unwrap_read_(i, &txoutindex_to_value_reader),
.read_at_unwrap(i, &txoutindex_to_value_reader),
vecs.txoutindex_to_outputtype
.unwrap_read_(i, &txoutindex_to_outputtype_reader),
.read_at_unwrap(i, &txoutindex_to_outputtype_reader),
vecs.txoutindex_to_typeindex
.unwrap_read_(i, &txoutindex_to_typeindex_reader),
.read_at_unwrap(i, &txoutindex_to_typeindex_reader),
)
})
.collect::<Vec<_>>()
@@ -744,20 +745,20 @@ fn run_method8(
let i = input_block_start + offset;
let outpoint = vecs
.txinindex_to_outpoint
.unwrap_read_(i, &txinindex_to_outpoint_reader);
.read_at_unwrap(i, &txinindex_to_outpoint_reader);
if outpoint.is_coinbase() {
return None;
}
let first_txoutindex = vecs.txindex_to_first_txoutindex.unwrap_read_(
let first_txoutindex = vecs.txindex_to_first_txoutindex.read_at_unwrap(
outpoint.txindex().to_usize(),
&txindex_to_first_txoutindex_reader,
);
let txoutindex = first_txoutindex.to_usize() + usize::from(outpoint.vout());
let value = vecs
.txoutindex_to_value
.unwrap_read_(txoutindex, &txoutindex_to_value_reader);
.read_at_unwrap(txoutindex, &txoutindex_to_value_reader);
Some(u64::from(value))
})
.collect::<Vec<_>>()

88
crates/brk_indexer/src/lib.rs
View File

@@ -8,31 +8,30 @@ use brk_iterator::Blocks;
use brk_rpc::Client;
use brk_store::AnyStore;
use brk_types::{
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutPoint, OutputType, Sats,
StoredBool, Timestamp, TxInIndex, TxIndex, TxOutIndex, Txid, TxidPrefix, TypeIndex,
TypeIndexAndOutPoint, TypeIndexAndTxIndex, Unit, Version, Vin, Vout,
AddressBytes, AddressBytesHash, AddressTypeAddressIndexOutPoint,
AddressTypeAddressIndexTxIndex, BlockHashPrefix, Height, OutPoint, OutputType, Sats,
StoredBool, Timestamp, TxInIndex, TxIndex, TxOutIndex, Txid, TxidPrefix, TypeIndex, Unit,
Version, Vin, Vout,
};
use log::{error, info};
use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use vecdb::{AnyVec, Exit, GenericStoredVec, Reader, VecIteratorExtended};
mod indexes;
// mod stores_redb;
// mod stores_v2;
mod stores_v3;
mod stores_v2;
// mod stores_v3;
mod vecs;
pub use indexes::*;
// pub use stores_redb::*;
// pub use stores_v2::*;
pub use stores_v3::*;
pub use stores_v2::*;
// pub use stores_v3::*;
pub use vecs::*;
// One version for all data sources
// Increment on **change _OR_ addition**
const VERSION: Version = Version::new(23);
const SNAPSHOT_BLOCK_RANGE: usize = 1_000;
const COLLISIONS_CHECKED_UP_TO: Height = Height::new(0);
const COLLISIONS_CHECKED_UP_TO: Height = Height::new(920_000);
#[derive(Clone)]
pub struct Indexer {
@@ -262,7 +261,7 @@ impl Indexer {
let vout = Vout::from(outpoint.vout);
let txoutindex = vecs.txindex_to_first_txoutindex.get_pushed_or_read(prev_txindex, &readers.txindex_to_first_txoutindex)?
let txoutindex = vecs.txindex_to_first_txoutindex.get_pushed_or_read_with(prev_txindex, &readers.txindex_to_first_txoutindex)?
.ok_or(Error::Str("Expect txoutindex to not be none"))
.inspect_err(|_| {
dbg!(outpoint.txid, prev_txindex, vout);
@@ -271,7 +270,7 @@ impl Indexer {
let outpoint = OutPoint::new(prev_txindex, vout);
let outputtype = vecs.txoutindex_to_outputtype.get_pushed_or_read(txoutindex, &readers.txoutindex_to_outputtype)?
let outputtype = vecs.txoutindex_to_outputtype.get_pushed_or_read_with(txoutindex, &readers.txoutindex_to_outputtype)?
.ok_or(Error::Str("Expect outputtype to not be none"))?;
let mut tuple = (
@@ -286,7 +285,7 @@ impl Indexer {
if outputtype.is_address() {
let typeindex = vecs
.txoutindex_to_typeindex
.get_pushed_or_read(txoutindex, &readers.txoutindex_to_typeindex)?
.get_pushed_or_read_with(txoutindex, &readers.txoutindex_to_typeindex)?
.ok_or(Error::Str("Expect typeindex to not be none"))?;
tuple.3 = Some((outputtype, typeindex));
}
@@ -374,56 +373,56 @@ impl Indexer {
let prev_addressbytes_opt = match outputtype {
OutputType::P2PK65 => vecs
.p2pk65addressindex_to_p2pk65bytes
.get_pushed_or_read(
.get_pushed_or_read_with(
typeindex.into(),
&readers.p2pk65addressindex_to_p2pk65bytes,
)?
.map(AddressBytes::from),
OutputType::P2PK33 => vecs
.p2pk33addressindex_to_p2pk33bytes
.get_pushed_or_read(
.get_pushed_or_read_with(
typeindex.into(),
&readers.p2pk33addressindex_to_p2pk33bytes,
)?
.map(AddressBytes::from),
OutputType::P2PKH => vecs
.p2pkhaddressindex_to_p2pkhbytes
.get_pushed_or_read(
.get_pushed_or_read_with(
typeindex.into(),
&readers.p2pkhaddressindex_to_p2pkhbytes,
)?
.map(AddressBytes::from),
OutputType::P2SH => vecs
.p2shaddressindex_to_p2shbytes
.get_pushed_or_read(
.get_pushed_or_read_with(
typeindex.into(),
&readers.p2shaddressindex_to_p2shbytes,
)?
.map(AddressBytes::from),
OutputType::P2WPKH => vecs
.p2wpkhaddressindex_to_p2wpkhbytes
.get_pushed_or_read(
.get_pushed_or_read_with(
typeindex.into(),
&readers.p2wpkhaddressindex_to_p2wpkhbytes,
)?
.map(AddressBytes::from),
OutputType::P2WSH => vecs
.p2wshaddressindex_to_p2wshbytes
.get_pushed_or_read(
.get_pushed_or_read_with(
typeindex.into(),
&readers.p2wshaddressindex_to_p2wshbytes,
)?
.map(AddressBytes::from),
OutputType::P2TR => vecs
.p2traddressindex_to_p2trbytes
.get_pushed_or_read(
.get_pushed_or_read_with(
typeindex.into(),
&readers.p2traddressindex_to_p2trbytes,
)?
.map(AddressBytes::from),
OutputType::P2A => vecs
.p2aaddressindex_to_p2abytes
.get_pushed_or_read(
.get_pushed_or_read_with(
typeindex.into(),
&readers.p2aaddressindex_to_p2abytes,
)?
@@ -573,12 +572,17 @@ impl Indexer {
if outputtype.is_unspendable() {
continue;
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
stores
.addresstype_to_typeindex_and_txindex
.get_mut(outputtype)
.unwrap()
.addresstype_to_addressindex_and_txindex
.insert_if_needed(
TypeIndexAndTxIndex::from((typeindex, txindex)),
AddressTypeAddressIndexTxIndex::from((
addresstype,
addressindex,
txindex,
)),
Unit,
height,
);
@@ -589,12 +593,17 @@ impl Indexer {
if same_block_spent_outpoints.contains(&outpoint) {
same_block_output_info.insert(outpoint, (outputtype, typeindex));
} else if outputtype.is_address() {
let addresstype = outputtype;
let addressindex = typeindex;
stores
.addresstype_to_typeindex_and_unspentoutpoint
.get_mut(outputtype)
.unwrap()
.addresstype_to_addressindex_and_unspentoutpoint
.insert_if_needed(
TypeIndexAndOutPoint::from((typeindex, outpoint)),
AddressTypeAddressIndexOutPoint::from((
addresstype,
addressindex,
outpoint,
)),
Unit,
height,
);
@@ -607,7 +616,7 @@ impl Indexer {
// let i = Instant::now();
for (txinindex, input_source) in txins {
let (vin, txindex, outpoint, outputtype_typeindex_opt) = match input_source {
let (vin, txindex, outpoint, addresstype_addressindex_opt) = match input_source {
InputSource::PreviousBlock(tuple) => tuple,
InputSource::SameBlock((txindex, txin, vin, outpoint)) => {
let mut tuple = (vin, txindex, outpoint, None);
@@ -636,23 +645,28 @@ impl Indexer {
vecs.txinindex_to_outpoint
.push_if_needed(txinindex, outpoint)?;
let Some((outputtype, typeindex)) = outputtype_typeindex_opt else {
let Some((addresstype, addressindex)) = addresstype_addressindex_opt else {
continue;
};
stores
.addresstype_to_typeindex_and_txindex
.get_mut_unwrap(outputtype)
.addresstype_to_addressindex_and_txindex
.insert_if_needed(
TypeIndexAndTxIndex::from((typeindex, txindex)),
AddressTypeAddressIndexTxIndex::from((addresstype, addressindex, txindex)),
Unit,
height,
);
stores
.addresstype_to_typeindex_and_unspentoutpoint
.get_mut_unwrap(outputtype)
.remove_if_needed(TypeIndexAndOutPoint::from((typeindex, outpoint)), height);
.addresstype_to_addressindex_and_unspentoutpoint
.remove_if_needed(
AddressTypeAddressIndexOutPoint::from((
addresstype,
addressindex,
outpoint,
)),
height,
);
}
// println!("txins.into_iter(): {:?}", i.elapsed());

408
crates/brk_indexer/src/stores_redb.rs
View File

@@ -1,408 +0,0 @@
use std::{fs, path::Path, sync::Arc};
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_store::{AnyStore, StoreRedb as Store};
use brk_types::{
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutPoint, StoredString, TxIndex,
TxOutIndex, TxidPrefix, TypeIndex, TypeIndexAndOutPoint, TypeIndexAndTxIndex, Unit, Version,
Vout,
};
use rayon::prelude::*;
use redb::Database;
use vecdb::{AnyVec, GenericStoredVec, StoredIndex, VecIterator, VecIteratorExtended};
use crate::Indexes;
use super::Vecs;
#[derive(Clone)]
pub struct Stores {
pub database: Arc<Database>,
pub addressbyteshash_to_typeindex: Store<AddressBytesHash, TypeIndex>,
pub blockhashprefix_to_height: Store<BlockHashPrefix, Height>,
pub height_to_coinbase_tag: Store<Height, StoredString>,
pub txidprefix_to_txindex: Store<TxidPrefix, TxIndex>,
pub addresstype_to_typeindex_and_txindex: ByAddressType<Store<TypeIndexAndTxIndex, Unit>>,
pub addresstype_to_typeindex_and_unspentoutpoint:
ByAddressType<Store<TypeIndexAndOutPoint, Unit>>,
}
impl Stores {
pub fn forced_import(parent: &Path, version: Version) -> Result<Self> {
let pathbuf = parent.join("stores");
let path = pathbuf.as_path();
fs::create_dir_all(&pathbuf)?;
let database = Arc::new(match brk_store::open_redb_database(path) {
Ok(database) => database,
Err(_) => {
fs::remove_dir_all(path)?;
return Self::forced_import(path, version);
}
});
let database_ref = &database;
let create_addressindex_and_txindex_store = |index| {
Store::import(
database_ref,
path,
&format!("a2t{}", index),
version,
Some(false),
)
};
let create_addressindex_and_unspentoutpoint_store =
|index| Store::import(database_ref, path, &format!("a2u{}", index), version, None);
Ok(Self {
database: database.clone(),
height_to_coinbase_tag: Store::import(database_ref, path, "h2c", version, None)?,
addressbyteshash_to_typeindex: Store::import(database_ref, path, "a2t", version, None)?,
blockhashprefix_to_height: Store::import(database_ref, path, "b2h", version, None)?,
txidprefix_to_txindex: Store::import(database_ref, path, "t2t", version, None)?,
addresstype_to_typeindex_and_txindex: ByAddressType::new_with_index(
create_addressindex_and_txindex_store,
)?,
addresstype_to_typeindex_and_unspentoutpoint: ByAddressType::new_with_index(
create_addressindex_and_unspentoutpoint_store,
)?,
})
}
pub fn starting_height(&self) -> Height {
self.iter_any_store()
.map(|store| {
// let height =
store.height().map(Height::incremented).unwrap_or_default()
// dbg!((height, store.name()));
})
.min()
.unwrap()
}
pub fn commit(&mut self, height: Height) -> Result<()> {
[
&mut self.addressbyteshash_to_typeindex as &mut dyn AnyStore,
&mut self.blockhashprefix_to_height,
&mut self.height_to_coinbase_tag,
&mut self.txidprefix_to_txindex,
]
// .into_iter() // Changed from par_iter_mut()
.into_par_iter() // Changed from par_iter_mut()
.chain(
self.addresstype_to_typeindex_and_txindex
// .iter_mut()
.par_iter_mut()
.map(|s| s as &mut dyn AnyStore),
)
.chain(
self.addresstype_to_typeindex_and_unspentoutpoint
// .iter_mut()
.par_iter_mut()
.map(|s| s as &mut dyn AnyStore),
)
.try_for_each(|store| store.commit(height))?;
Ok(())
// self.database
// .persist(PersistMode::SyncAll)
// .map_err(|e| e.into())
}
fn iter_any_store(&self) -> impl Iterator<Item = &dyn AnyStore> {
[
&self.addressbyteshash_to_typeindex as &dyn AnyStore,
&self.blockhashprefix_to_height,
&self.height_to_coinbase_tag,
&self.txidprefix_to_txindex,
]
.into_iter()
.chain(
self.addresstype_to_typeindex_and_txindex
.iter()
.map(|s| s as &dyn AnyStore),
)
.chain(
self.addresstype_to_typeindex_and_unspentoutpoint
.iter()
.map(|s| s as &dyn AnyStore),
)
}
pub fn rollback_if_needed(
&mut self,
vecs: &mut Vecs,
starting_indexes: &Indexes,
) -> Result<()> {
if self.addressbyteshash_to_typeindex.is_empty()?
&& self.blockhashprefix_to_height.is_empty()?
&& self.txidprefix_to_txindex.is_empty()?
&& self.height_to_coinbase_tag.is_empty()?
&& self
.addresstype_to_typeindex_and_txindex
.iter()
.map(|s| s.is_empty())
.collect::<Result<Vec<_>>>()?
.into_iter()
.all(|empty| empty)
&& self
.addresstype_to_typeindex_and_unspentoutpoint
.iter()
.map(|s| s.is_empty())
.collect::<Result<Vec<_>>>()?
.into_iter()
.all(|empty| empty)
{
return Ok(());
}
if starting_indexes.height != Height::ZERO {
vecs.height_to_blockhash
.iter()?
.skip(starting_indexes.height.to_usize())
.map(BlockHashPrefix::from)
.for_each(|prefix| {
self.blockhashprefix_to_height.remove(prefix);
});
(starting_indexes.height.to_usize()..vecs.height_to_blockhash.len())
.map(Height::from)
.for_each(|h| {
self.height_to_coinbase_tag.remove(h);
});
if let Ok(mut index) = vecs
.height_to_first_p2pk65addressindex
.one_shot_read(starting_indexes.height)
{
let mut p2pk65addressindex_to_p2pk65bytes_iter =
vecs.p2pk65addressindex_to_p2pk65bytes.iter()?;
while let Some(typedbytes) = p2pk65addressindex_to_p2pk65bytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressBytesHash::from(&bytes);
self.addressbyteshash_to_typeindex.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2pk33addressindex
.one_shot_read(starting_indexes.height)
{
let mut p2pk33addressindex_to_p2pk33bytes_iter =
vecs.p2pk33addressindex_to_p2pk33bytes.iter()?;
while let Some(typedbytes) = p2pk33addressindex_to_p2pk33bytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressBytesHash::from(&bytes);
self.addressbyteshash_to_typeindex.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2pkhaddressindex
.one_shot_read(starting_indexes.height)
{
let mut p2pkhaddressindex_to_p2pkhbytes_iter =
vecs.p2pkhaddressindex_to_p2pkhbytes.iter()?;
while let Some(typedbytes) = p2pkhaddressindex_to_p2pkhbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressBytesHash::from(&bytes);
self.addressbyteshash_to_typeindex.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2shaddressindex
.one_shot_read(starting_indexes.height)
{
let mut p2shaddressindex_to_p2shbytes_iter =
vecs.p2shaddressindex_to_p2shbytes.iter()?;
while let Some(typedbytes) = p2shaddressindex_to_p2shbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressBytesHash::from(&bytes);
self.addressbyteshash_to_typeindex.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2traddressindex
.one_shot_read(starting_indexes.height)
{
let mut p2traddressindex_to_p2trbytes_iter =
vecs.p2traddressindex_to_p2trbytes.iter()?;
while let Some(typedbytes) = p2traddressindex_to_p2trbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressBytesHash::from(&bytes);
self.addressbyteshash_to_typeindex.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2wpkhaddressindex
.one_shot_read(starting_indexes.height)
{
let mut p2wpkhaddressindex_to_p2wpkhbytes_iter =
vecs.p2wpkhaddressindex_to_p2wpkhbytes.iter()?;
while let Some(typedbytes) = p2wpkhaddressindex_to_p2wpkhbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressBytesHash::from(&bytes);
self.addressbyteshash_to_typeindex.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2wshaddressindex
.one_shot_read(starting_indexes.height)
{
let mut p2wshaddressindex_to_p2wshbytes_iter =
vecs.p2wshaddressindex_to_p2wshbytes.iter()?;
while let Some(typedbytes) = p2wshaddressindex_to_p2wshbytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressBytesHash::from(&bytes);
self.addressbyteshash_to_typeindex.remove(hash);
index.increment();
}
}
if let Ok(mut index) = vecs
.height_to_first_p2aaddressindex
.one_shot_read(starting_indexes.height)
{
let mut p2aaddressindex_to_p2abytes_iter =
vecs.p2aaddressindex_to_p2abytes.iter()?;
while let Some(typedbytes) = p2aaddressindex_to_p2abytes_iter.get(index) {
let bytes = AddressBytes::from(typedbytes);
let hash = AddressBytesHash::from(&bytes);
self.addressbyteshash_to_typeindex.remove(hash);
index.increment();
}
}
} else {
unreachable!();
// self.blockhashprefix_to_height.reset()?;
// self.addressbyteshash_to_typeindex.reset()?;
}
if starting_indexes.txindex != TxIndex::ZERO {
vecs.txindex_to_txid
.iter()?
.enumerate()
.skip(starting_indexes.txindex.to_usize())
.for_each(|(txindex, txid)| {
let txindex = TxIndex::from(txindex);
let txidprefix = TxidPrefix::from(&txid);
// "d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599"
let is_not_first_dup = txindex != TxIndex::new(142783)
|| txidprefix != TxidPrefix::from([153, 133, 216, 41, 84, 225, 15, 34]);
// "e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468"
let is_not_second_dup = txindex != TxIndex::new(142841)
|| txidprefix != TxidPrefix::from([104, 180, 95, 88, 182, 116, 233, 78]);
if is_not_first_dup && is_not_second_dup {
self.txidprefix_to_txindex.remove(txidprefix);
}
});
} else {
unreachable!();
// self.txidprefix_to_txindex.reset()?;
}
if starting_indexes.txoutindex != TxOutIndex::ZERO {
let mut txoutindex_to_txindex_iter = vecs.txoutindex_to_txindex.iter()?;
let mut txindex_to_first_txoutindex_iter = vecs.txindex_to_first_txoutindex.iter()?;
vecs.txoutindex_to_outputtype
.iter()?
.enumerate()
.skip(starting_indexes.txoutindex.to_usize())
.zip(
vecs.txoutindex_to_typeindex
.iter()?
.skip(starting_indexes.txoutindex.to_usize()),
)
.filter(|((_, outputtype), _)| outputtype.is_address())
.for_each(|((txoutindex, outputtype), typeindex)| {
let txindex = txoutindex_to_txindex_iter.unsafe_get_(txoutindex);
let vout = Vout::from(
txoutindex.to_usize()
- txindex_to_first_txoutindex_iter
.unsafe_get(txindex)
.to_usize(),
);
let outpoint = OutPoint::new(txindex, vout);
self.addresstype_to_typeindex_and_unspentoutpoint
.get_mut(outputtype)
.unwrap()
.remove(TypeIndexAndOutPoint::from((typeindex, outpoint)));
});
// Add back outputs that were spent after the rollback point
let mut txindex_to_first_txoutindex_iter = vecs.txindex_to_first_txoutindex.iter()?;
let mut txoutindex_to_outputtype_iter = vecs.txoutindex_to_outputtype.iter()?;
let mut txoutindex_to_typeindex_iter = vecs.txoutindex_to_typeindex.iter()?;
vecs.txinindex_to_outpoint
.iter()?
.skip(starting_indexes.txinindex.to_usize())
.for_each(|outpoint| {
if outpoint.is_coinbase() {
return;
}
let txindex = outpoint.txindex();
let vout = outpoint.vout();
// Calculate txoutindex from txindex and vout
let txoutindex = txindex_to_first_txoutindex_iter.unsafe_get(txindex) + vout;
// Only process if this output was created before the rollback point
if txoutindex < starting_indexes.txoutindex {
let outputtype = txoutindex_to_outputtype_iter.unsafe_get(txoutindex);
if outputtype.is_address() {
let typeindex = txoutindex_to_typeindex_iter.unsafe_get(txoutindex);
self.addresstype_to_typeindex_and_unspentoutpoint
.get_mut(outputtype)
.unwrap()
.insert(TypeIndexAndOutPoint::from((typeindex, outpoint)), Unit);
}
}
});
} else {
unreachable!();
// self.addresstype_to_typeindex_and_txindex
// .iter_mut()
// .try_for_each(|s| s.reset())?;
// self.addresstype_to_typeindex_and_unspentoutpoint
// .iter_mut()
// .try_for_each(|s| s.reset())?;
}
self.commit(starting_indexes.height.decremented().unwrap_or_default())?;
Ok(())
}
}

154
crates/brk_indexer/src/stores_v2.rs
View File

@@ -1,12 +1,11 @@
use std::{fs, path::Path};
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_store::{AnyStore, StoreFjallV2 as Store};
use brk_types::{
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutPoint, StoredString, TxIndex,
TxOutIndex, TxidPrefix, TypeIndex, TypeIndexAndOutPoint, TypeIndexAndTxIndex, Unit, Version,
Vout,
AddressBytes, AddressBytesHash, AddressTypeAddressIndexOutPoint,
AddressTypeAddressIndexTxIndex, BlockHashPrefix, Height, OutPoint, StoredString, TxIndex,
TxOutIndex, TxidPrefix, TypeIndex, Unit, Version, Vout,
};
use fjall2::{PersistMode, TransactionalKeyspace};
use rayon::prelude::*;
@@ -24,9 +23,9 @@ pub struct Stores {
pub blockhashprefix_to_height: Store<BlockHashPrefix, Height>,
pub height_to_coinbase_tag: Store<Height, StoredString>,
pub txidprefix_to_txindex: Store<TxidPrefix, TxIndex>,
pub addresstype_to_typeindex_and_txindex: ByAddressType<Store<TypeIndexAndTxIndex, Unit>>,
pub addresstype_to_typeindex_and_unspentoutpoint:
ByAddressType<Store<TypeIndexAndOutPoint, Unit>>,
pub addresstype_to_addressindex_and_txindex: Store<AddressTypeAddressIndexTxIndex, Unit>,
pub addresstype_to_addressindex_and_unspentoutpoint:
Store<AddressTypeAddressIndexOutPoint, Unit>,
}
impl Stores {
@@ -46,31 +45,26 @@ impl Stores {
let keyspace_ref = &keyspace;
let create_addressindex_and_txindex_store = |index| {
Store::import(
keyspace_ref,
path,
&format!("a2t{}", index),
version,
Some(false),
)
};
let create_addressindex_and_unspentoutpoint_store =
|index| Store::import(keyspace_ref, path, &format!("a2u{}", index), version, None);
Ok(Self {
keyspace: keyspace.clone(),
height_to_coinbase_tag: Store::import(keyspace_ref, path, "h2c", version, None)?,
addressbyteshash_to_typeindex: Store::import(keyspace_ref, path, "a2t", version, None)?,
blockhashprefix_to_height: Store::import(keyspace_ref, path, "b2h", version, None)?,
txidprefix_to_txindex: Store::import(keyspace_ref, path, "t2t", version, None)?,
addresstype_to_typeindex_and_txindex: ByAddressType::new_with_index(
create_addressindex_and_txindex_store,
height_to_coinbase_tag: Store::import(keyspace_ref, path, "h2c", version, true)?,
addressbyteshash_to_typeindex: Store::import(keyspace_ref, path, "a2t", version, true)?,
blockhashprefix_to_height: Store::import(keyspace_ref, path, "b2h", version, true)?,
txidprefix_to_txindex: Store::import(keyspace_ref, path, "t2t", version, true)?,
addresstype_to_addressindex_and_txindex: Store::import(
keyspace_ref,
path,
"aat",
version,
false,
)?,
addresstype_to_typeindex_and_unspentoutpoint: ByAddressType::new_with_index(
create_addressindex_and_unspentoutpoint_store,
addresstype_to_addressindex_and_unspentoutpoint: Store::import(
keyspace_ref,
path,
"aau",
version,
false,
)?,
})
}
@@ -92,18 +86,10 @@ impl Stores {
&mut self.blockhashprefix_to_height,
&mut self.height_to_coinbase_tag,
&mut self.txidprefix_to_txindex,
&mut self.addresstype_to_addressindex_and_txindex,
&mut self.addresstype_to_addressindex_and_unspentoutpoint,
]
.into_par_iter() // Changed from par_iter_mut()
.chain(
self.addresstype_to_typeindex_and_txindex
.par_iter_mut()
.map(|s| s as &mut dyn AnyStore),
)
.chain(
self.addresstype_to_typeindex_and_unspentoutpoint
.par_iter_mut()
.map(|s| s as &mut dyn AnyStore),
)
.try_for_each(|store| store.commit(height))?;
self.keyspace
@@ -117,18 +103,10 @@ impl Stores {
&self.blockhashprefix_to_height,
&self.height_to_coinbase_tag,
&self.txidprefix_to_txindex,
&self.addresstype_to_addressindex_and_txindex,
&self.addresstype_to_addressindex_and_unspentoutpoint,
]
.into_iter()
.chain(
self.addresstype_to_typeindex_and_txindex
.iter()
.map(|s| s as &dyn AnyStore),
)
.chain(
self.addresstype_to_typeindex_and_unspentoutpoint
.iter()
.map(|s| s as &dyn AnyStore),
)
}
pub fn rollback_if_needed(
@@ -140,20 +118,10 @@ impl Stores {
&& self.blockhashprefix_to_height.is_empty()?
&& self.txidprefix_to_txindex.is_empty()?
&& self.height_to_coinbase_tag.is_empty()?
&& self.addresstype_to_addressindex_and_txindex.is_empty()?
&& self
.addresstype_to_typeindex_and_txindex
.iter()
.map(|s| s.is_empty())
.collect::<Result<Vec<_>>>()?
.into_iter()
.all(|empty| empty)
&& self
.addresstype_to_typeindex_and_unspentoutpoint
.iter()
.map(|s| s.is_empty())
.collect::<Result<Vec<_>>>()?
.into_iter()
.all(|empty| empty)
.addresstype_to_addressindex_and_unspentoutpoint
.is_empty()?
{
return Ok(());
}
@@ -175,7 +143,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2pk65addressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2pk65addressindex_to_p2pk65bytes_iter =
vecs.p2pk65addressindex_to_p2pk65bytes.iter()?;
@@ -190,7 +158,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2pk33addressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2pk33addressindex_to_p2pk33bytes_iter =
vecs.p2pk33addressindex_to_p2pk33bytes.iter()?;
@@ -205,7 +173,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2pkhaddressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2pkhaddressindex_to_p2pkhbytes_iter =
vecs.p2pkhaddressindex_to_p2pkhbytes.iter()?;
@@ -220,7 +188,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2shaddressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2shaddressindex_to_p2shbytes_iter =
vecs.p2shaddressindex_to_p2shbytes.iter()?;
@@ -235,7 +203,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2traddressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2traddressindex_to_p2trbytes_iter =
vecs.p2traddressindex_to_p2trbytes.iter()?;
@@ -250,7 +218,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2wpkhaddressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2wpkhaddressindex_to_p2wpkhbytes_iter =
vecs.p2wpkhaddressindex_to_p2wpkhbytes.iter()?;
@@ -265,7 +233,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2wshaddressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2wshaddressindex_to_p2wshbytes_iter =
vecs.p2wshaddressindex_to_p2wshbytes.iter()?;
@@ -280,7 +248,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2aaddressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2aaddressindex_to_p2abytes_iter =
vecs.p2aaddressindex_to_p2abytes.iter()?;
@@ -338,21 +306,28 @@ impl Stores {
.skip(starting_indexes.txoutindex.to_usize()),
)
.filter(|((_, outputtype), _)| outputtype.is_address())
.for_each(|((txoutindex, outputtype), typeindex)| {
let txindex = txoutindex_to_txindex_iter.unsafe_get_(txoutindex);
.for_each(|((txoutindex, addresstype), addressindex)| {
let txindex = txoutindex_to_txindex_iter.get_unwrap_at(txoutindex);
self.addresstype_to_addressindex_and_txindex.remove(
AddressTypeAddressIndexTxIndex::from((addresstype, addressindex, txindex)),
);
let vout = Vout::from(
txoutindex.to_usize()
- txindex_to_first_txoutindex_iter
.unsafe_get(txindex)
.get_unwrap(txindex)
.to_usize(),
);
let outpoint = OutPoint::new(txindex, vout);
self.addresstype_to_typeindex_and_unspentoutpoint
.get_mut(outputtype)
.unwrap()
.remove(TypeIndexAndOutPoint::from((typeindex, outpoint)));
self.addresstype_to_addressindex_and_unspentoutpoint.remove(
AddressTypeAddressIndexOutPoint::from((
addresstype,
addressindex,
outpoint,
)),
);
});
// Add back outputs that were spent after the rollback point
@@ -371,19 +346,32 @@ impl Stores {
let vout = outpoint.vout();
// Calculate txoutindex from txindex and vout
let txoutindex = txindex_to_first_txoutindex_iter.unsafe_get(txindex) + vout;
let txoutindex = txindex_to_first_txoutindex_iter.get_unwrap(txindex) + vout;
// Only process if this output was created before the rollback point
if txoutindex < starting_indexes.txoutindex {
let outputtype = txoutindex_to_outputtype_iter.unsafe_get(txoutindex);
let outputtype = txoutindex_to_outputtype_iter.get_unwrap(txoutindex);
if outputtype.is_address() {
let typeindex = txoutindex_to_typeindex_iter.unsafe_get(txoutindex);
let addresstype = outputtype;
let addressindex = txoutindex_to_typeindex_iter.get_unwrap(txoutindex);
self.addresstype_to_typeindex_and_unspentoutpoint
.get_mut(outputtype)
.unwrap()
.insert(TypeIndexAndOutPoint::from((typeindex, outpoint)), Unit);
self.addresstype_to_addressindex_and_txindex.remove(
AddressTypeAddressIndexTxIndex::from((
addresstype,
addressindex,
txindex,
)),
);
self.addresstype_to_addressindex_and_unspentoutpoint.insert(
AddressTypeAddressIndexOutPoint::from((
addresstype,
addressindex,
outpoint,
)),
Unit,
);
}
}
});

197
crates/brk_indexer/src/stores_v3.rs
View File

@@ -1,15 +1,13 @@
use std::{fs, path::Path};
use brk_error::Result;
use brk_grouper::ByAddressType;
use brk_store::{AnyStore, StoreFjallV3 as Store};
use brk_types::{
AddressBytes, AddressBytesHash, BlockHashPrefix, Height, OutPoint, StoredString, TxIndex,
TxOutIndex, TxidPrefix, TypeIndex, TypeIndexAndOutPoint, TypeIndexAndTxIndex, Unit, Version,
Vout,
AddressBytes, AddressBytesHash, AddressTypeAddressIndexOutPoint,
AddressTypeAddressIndexTxIndex, BlockHashPrefix, Height, OutPoint, StoredString, TxIndex,
TxOutIndex, TxidPrefix, TypeIndex, Unit, Version, Vout,
};
use fjall3::{Database, PersistMode};
use log::info;
use rayon::prelude::*;
use vecdb::{AnyVec, GenericStoredVec, StoredIndex, VecIterator, VecIteratorExtended};
@@ -25,9 +23,9 @@ pub struct Stores {
pub blockhashprefix_to_height: Store<BlockHashPrefix, Height>,
pub height_to_coinbase_tag: Store<Height, StoredString>,
pub txidprefix_to_txindex: Store<TxidPrefix, TxIndex>,
pub addresstype_to_typeindex_and_txindex: ByAddressType<Store<TypeIndexAndTxIndex, Unit>>,
pub addresstype_to_typeindex_and_unspentoutpoint:
ByAddressType<Store<TypeIndexAndOutPoint, Unit>>,
pub addresstype_to_addressindex_and_txindex: Store<AddressTypeAddressIndexTxIndex, Unit>,
pub addresstype_to_addressindex_and_unspentoutpoint:
Store<AddressTypeAddressIndexOutPoint, Unit>,
}
impl Stores {
@@ -47,31 +45,56 @@ impl Stores {
let database_ref = &database;
let create_addressindex_and_txindex_store = |index| {
Store::import(
database_ref,
path,
&format!("a2t{}", index),
version,
Some(false),
)
};
let create_addressindex_and_unspentoutpoint_store =
|index| Store::import(database_ref, path, &format!("a2u{}", index), version, None);
Ok(Self {
database: database.clone(),
height_to_coinbase_tag: Store::import(database_ref, path, "h2c", version, None)?,
addressbyteshash_to_typeindex: Store::import(database_ref, path, "a2t", version, None)?,
blockhashprefix_to_height: Store::import(database_ref, path, "b2h", version, None)?,
txidprefix_to_txindex: Store::import(database_ref, path, "t2t", version, None)?,
addresstype_to_typeindex_and_txindex: ByAddressType::new_with_index(
create_addressindex_and_txindex_store,
height_to_coinbase_tag: Store::import(
database_ref,
path,
"height_to_coinbase_tag",
version,
true,
true,
)?,
addresstype_to_typeindex_and_unspentoutpoint: ByAddressType::new_with_index(
create_addressindex_and_unspentoutpoint_store,
addressbyteshash_to_typeindex: Store::import(
database_ref,
path,
"addressbyteshash_to_typeindex",
version,
true,
false,
)?,
blockhashprefix_to_height: Store::import(
database_ref,
path,
"blockhashprefix_to_height",
version,
true,
false,
)?,
txidprefix_to_txindex: Store::import(
database_ref,
path,
"txidprefix_to_txindex",
version,
true,
false,
)?,
addresstype_to_addressindex_and_txindex: Store::import(
database_ref,
path,
"addresstype_to_addressindex_and_txindex",
version,
false,
false,
)?,
addresstype_to_addressindex_and_unspentoutpoint: Store::import(
database_ref,
path,
"addresstype_to_addressindex_and_unspentoutpoint",
version,
false,
false,
)?,
})
}
@@ -88,37 +111,17 @@ impl Stores {
}
pub fn commit(&mut self, height: Height) -> Result<()> {
info!(
"database.write_buffer_size = {}",
self.database.write_buffer_size()
);
info!("database.journal_count = {}", self.database.journal_count());
[
&mut self.addressbyteshash_to_typeindex as &mut dyn AnyStore,
&mut self.blockhashprefix_to_height,
&mut self.height_to_coinbase_tag,
&mut self.txidprefix_to_txindex,
&mut self.addresstype_to_addressindex_and_txindex,
&mut self.addresstype_to_addressindex_and_unspentoutpoint,
]
.into_par_iter() // Changed from par_iter_mut()
.chain(
self.addresstype_to_typeindex_and_txindex
.par_iter_mut()
.map(|s| s as &mut dyn AnyStore),
)
.chain(
self.addresstype_to_typeindex_and_unspentoutpoint
.par_iter_mut()
.map(|s| s as &mut dyn AnyStore),
)
.try_for_each(|store| store.commit(height))?;
info!(
"database.write_buffer_size = {}",
self.database.write_buffer_size()
);
info!("database.journal_count = {}", self.database.journal_count());
self.database
.persist(PersistMode::SyncAll)
.map_err(|e| e.into())
@@ -130,18 +133,10 @@ impl Stores {
&self.blockhashprefix_to_height,
&self.height_to_coinbase_tag,
&self.txidprefix_to_txindex,
&self.addresstype_to_addressindex_and_txindex,
&self.addresstype_to_addressindex_and_unspentoutpoint,
]
.into_iter()
.chain(
self.addresstype_to_typeindex_and_txindex
.iter()
.map(|s| s as &dyn AnyStore),
)
.chain(
self.addresstype_to_typeindex_and_unspentoutpoint
.iter()
.map(|s| s as &dyn AnyStore),
)
}
pub fn rollback_if_needed(
@@ -153,20 +148,10 @@ impl Stores {
&& self.blockhashprefix_to_height.is_empty()?
&& self.txidprefix_to_txindex.is_empty()?
&& self.height_to_coinbase_tag.is_empty()?
&& self.addresstype_to_addressindex_and_txindex.is_empty()?
&& self
.addresstype_to_typeindex_and_txindex
.iter()
.map(|s| s.is_empty())
.collect::<Result<Vec<_>>>()?
.into_iter()
.all(|empty| empty)
&& self
.addresstype_to_typeindex_and_unspentoutpoint
.iter()
.map(|s| s.is_empty())
.collect::<Result<Vec<_>>>()?
.into_iter()
.all(|empty| empty)
.addresstype_to_addressindex_and_unspentoutpoint
.is_empty()?
{
return Ok(());
}
@@ -188,7 +173,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2pk65addressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2pk65addressindex_to_p2pk65bytes_iter =
vecs.p2pk65addressindex_to_p2pk65bytes.iter()?;
@@ -203,7 +188,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2pk33addressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2pk33addressindex_to_p2pk33bytes_iter =
vecs.p2pk33addressindex_to_p2pk33bytes.iter()?;
@@ -218,7 +203,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2pkhaddressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2pkhaddressindex_to_p2pkhbytes_iter =
vecs.p2pkhaddressindex_to_p2pkhbytes.iter()?;
@@ -233,7 +218,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2shaddressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2shaddressindex_to_p2shbytes_iter =
vecs.p2shaddressindex_to_p2shbytes.iter()?;
@@ -248,7 +233,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2traddressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2traddressindex_to_p2trbytes_iter =
vecs.p2traddressindex_to_p2trbytes.iter()?;
@@ -263,7 +248,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2wpkhaddressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2wpkhaddressindex_to_p2wpkhbytes_iter =
vecs.p2wpkhaddressindex_to_p2wpkhbytes.iter()?;
@@ -278,7 +263,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2wshaddressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2wshaddressindex_to_p2wshbytes_iter =
vecs.p2wshaddressindex_to_p2wshbytes.iter()?;
@@ -293,7 +278,7 @@ impl Stores {
if let Ok(mut index) = vecs
.height_to_first_p2aaddressindex
.one_shot_read(starting_indexes.height)
.read(starting_indexes.height)
{
let mut p2aaddressindex_to_p2abytes_iter =
vecs.p2aaddressindex_to_p2abytes.iter()?;
@@ -351,21 +336,28 @@ impl Stores {
.skip(starting_indexes.txoutindex.to_usize()),
)
.filter(|((_, outputtype), _)| outputtype.is_address())
.for_each(|((txoutindex, outputtype), typeindex)| {
let txindex = txoutindex_to_txindex_iter.unsafe_get_(txoutindex);
.for_each(|((txoutindex, addresstype), addressindex)| {
let txindex = txoutindex_to_txindex_iter.get_unwrap_at(txoutindex);
self.addresstype_to_addressindex_and_txindex.remove(
AddressTypeAddressIndexTxIndex::from((addresstype, addressindex, txindex)),
);
let vout = Vout::from(
txoutindex.to_usize()
- txindex_to_first_txoutindex_iter
.unsafe_get(txindex)
.get_unwrap(txindex)
.to_usize(),
);
let outpoint = OutPoint::new(txindex, vout);
self.addresstype_to_typeindex_and_unspentoutpoint
.get_mut(outputtype)
.unwrap()
.remove(TypeIndexAndOutPoint::from((typeindex, outpoint)));
self.addresstype_to_addressindex_and_unspentoutpoint.remove(
AddressTypeAddressIndexOutPoint::from((
addresstype,
addressindex,
outpoint,
)),
);
});
// Add back outputs that were spent after the rollback point
@@ -384,19 +376,32 @@ impl Stores {
let vout = outpoint.vout();
// Calculate txoutindex from txindex and vout
let txoutindex = txindex_to_first_txoutindex_iter.unsafe_get(txindex) + vout;
let txoutindex = txindex_to_first_txoutindex_iter.get_unwrap(txindex) + vout;
// Only process if this output was created before the rollback point
if txoutindex < starting_indexes.txoutindex {
let outputtype = txoutindex_to_outputtype_iter.unsafe_get(txoutindex);
let outputtype = txoutindex_to_outputtype_iter.get_unwrap(txoutindex);
if outputtype.is_address() {
let typeindex = txoutindex_to_typeindex_iter.unsafe_get(txoutindex);
let addresstype = outputtype;
let addressindex = txoutindex_to_typeindex_iter.get_unwrap(txoutindex);
self.addresstype_to_typeindex_and_unspentoutpoint
.get_mut(outputtype)
.unwrap()
.insert(TypeIndexAndOutPoint::from((typeindex, outpoint)), Unit);
self.addresstype_to_addressindex_and_txindex.remove(
AddressTypeAddressIndexTxIndex::from((
addresstype,
addressindex,
txindex,
)),
);
self.addresstype_to_addressindex_and_unspentoutpoint.insert(
AddressTypeAddressIndexOutPoint::from((
addresstype,
addressindex,
outpoint,
)),
Unit,
);
}
}
});

9
crates/brk_iterator/examples/iterator.rs
View File

@@ -1,4 +1,4 @@
use std::{path::Path, time::Instant};
use std::time::Instant;
use brk_error::Result;
use brk_iterator::Blocks;
@@ -7,13 +7,10 @@ use brk_rpc::{Auth, Client};
use brk_types::Height;
fn main() -> Result<()> {
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
let bitcoin_dir = Client::default_bitcoin_path();
let client = Client::new(
"http://localhost:8332",
Client::default_url(),
Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?;

3
crates/brk_logger/src/lib.rs
View File

@@ -23,7 +23,8 @@ pub fn init(path: Option<&Path>) -> io::Result<()> {
});
Builder::from_env(Env::default().default_filter_or(
"debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
"info,bitcoin=off,bitcoincore-rpc=off,fjall=off,lsm-tree=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
// "debug,bitcoin=off,bitcoincore-rpc=off,rolldown=off,rolldown=off,rmcp=off,brk_rmcp=off,tracing=off,aide=off,brk_aide=off",
))
.format(move |buf, record| {
let date_time = Timestamp::now()

9
crates/brk_monitor/examples/mempool.rs
View File

@@ -1,4 +1,4 @@
use std::{path::Path, thread, time::Duration};
use std::{thread, time::Duration};
use brk_error::Result;
use brk_monitor::Mempool;
@@ -6,14 +6,11 @@ use brk_rpc::{Auth, Client};
fn main() -> Result<()> {
// Connect to Bitcoin Core
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
let bitcoin_dir = Client::default_bitcoin_path();
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
let client = Client::new(
"http://localhost:8332",
Client::default_url(),
Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?;

13
crates/brk_query/examples/main.rs
View File

@@ -1,4 +1,4 @@
use std::{fs, path::Path};
use std::{env, fs, path::Path};
use brk_computer::Computer;
use brk_error::Result;
@@ -10,24 +10,21 @@ use brk_types::Index;
use vecdb::Exit;
pub fn main() -> Result<()> {
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
let bitcoin_dir = Client::default_bitcoin_path();
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
let blocks_dir = bitcoin_dir.join("blocks");
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk");
fs::create_dir_all(&outputs_dir)?;
// let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk");
let client = Client::new(
"http://localhost:8332",
Client::default_url(),
Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?;
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
let outputs_dir = Path::new(&env::var("HOME").unwrap()).join(".brk");
// let outputs_dir = Path::new("../../_outputs");
let exit = Exit::new();

9
crates/brk_reader/examples/reader.rs
View File

@@ -1,5 +1,3 @@
use std::path::Path;
use brk_error::Result;
use brk_reader::Reader;
use brk_rpc::{Auth, Client};
@@ -8,13 +6,10 @@ use brk_rpc::{Auth, Client};
fn main() -> Result<()> {
let i = std::time::Instant::now();
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
let bitcoin_dir = Client::default_bitcoin_path();
let client = Client::new(
"http://localhost:8332",
Client::default_url(),
Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?;

9
crates/brk_rpc/examples/rpc.rs
View File

@@ -1,19 +1,14 @@
use std::path::Path;
use bitcoincore_rpc::RpcApi;
use brk_rpc::{Auth, Client};
fn main() {
brk_logger::init(None).unwrap();
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
let bitcoin_dir = Client::default_bitcoin_path();
let auth = Auth::CookieFile(bitcoin_dir.join(".cookie"));
let client = Client::new("http://localhost:8332", auth).unwrap();
let client = Client::new(Client::default_url(), auth).unwrap();
loop {
println!("{:?}", client.call(|c| c.get_block_count()).unwrap());

25
crates/brk_rpc/src/lib.rs
View File

@@ -1,3 +1,5 @@
use std::env;
use std::path::{Path, PathBuf};
use std::thread::sleep;
use std::{mem, sync::Arc, time::Duration};
@@ -270,4 +272,27 @@ impl Client {
{
self.0.call_once(f)
}
pub fn default_url() -> &'static str {
"http://localhost:8332"
}
pub fn default_bitcoin_path() -> PathBuf {
if env::consts::OS == "macos" {
Self::default_mac_bitcoin_path()
} else {
Self::default_linux_bitcoin_path()
}
}
pub fn default_linux_bitcoin_path() -> PathBuf {
Path::new(&env::var("HOME").unwrap()).join(".bitcoin")
}
pub fn default_mac_bitcoin_path() -> PathBuf {
Path::new(&env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin")
}
}

7
crates/brk_server/examples/main.rs
View File

@@ -29,17 +29,14 @@ pub fn main() -> Result<()> {
fn run() -> Result<()> {
brk_logger::init(Some(Path::new(".log")))?;
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
.join("Library")
.join("Application Support")
.join("Bitcoin");
let bitcoin_dir = Client::default_bitcoin_path();
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
// let outputs_dir = Path::new("../../_outputs");
let client = Client::new(
"http://localhost:8332",
Client::default_url(),
Auth::CookieFile(bitcoin_dir.join(".cookie")),
)?;

1
crates/brk_store/Cargo.toml
View File

@@ -21,5 +21,4 @@ fjall2 = { workspace = true }
fjall3 = { workspace = true }
log = { workspace = true }
parking_lot = { workspace = true }
redb = { workspace = true }
rustc-hash = { workspace = true }

18
crates/brk_store/src/fjall_v2/mod.rs
View File

@@ -28,9 +28,7 @@ pub struct StoreFjallV2<Key, Value> {
const MAJOR_FJALL_VERSION: Version = Version::TWO;
pub fn open_keyspace(path: &Path) -> fjall2::Result<TransactionalKeyspace> {
fjall2::Config::new(path.join("fjall"))
.max_write_buffer_size(32 * 1024 * 1024)
.open_transactional()
fjall2::Config::new(path.join("fjall")).open_transactional()
}
impl<K, V> StoreFjallV2<K, V>
@@ -42,16 +40,14 @@ where
fn open_partition_handle(
keyspace: &TransactionalKeyspace,
name: &str,
bloom_filters: Option<bool>,
bloom_filters: bool,
) -> Result<TransactionalPartitionHandle> {
let mut options = PartitionCreateOptions::default()
.max_memtable_size(8 * 1024 * 1024)
.manual_journal_persist(true);
let mut options = PartitionCreateOptions::default().manual_journal_persist(true);
if bloom_filters.is_some_and(|b| !b) {
options = options.bloom_filter_bits(None);
} else {
if bloom_filters {
options = options.bloom_filter_bits(Some(5));
} else {
options = options.bloom_filter_bits(None);
}
keyspace.open_partition(name, options).map_err(|e| e.into())
@@ -62,7 +58,7 @@ where
path: &Path,
name: &str,
version: Version,
bloom_filters: Option<bool>,
bloom_filters: bool,
) -> Result<Self> {
fs::create_dir_all(path)?;

92
crates/brk_store/src/fjall_v3/mod.rs
View File

@@ -29,7 +29,6 @@ const MAJOR_FJALL_VERSION: Version = Version::new(3);
pub fn open_fjall3_database(path: &Path) -> fjall3::Result<Database> {
Database::builder(path.join("fjall"))
.max_write_buffer_size(32 * 1024 * 1024)
.cache_size(1024 * 1024 * 1024)
.open()
}
@@ -40,34 +39,13 @@ where
V: Debug + Clone + From<ByteView>,
ByteView: From<K> + From<V>,
{
fn open_keyspace(
database: &Database,
name: &str,
bloom_filters: Option<bool>,
) -> Result<Keyspace> {
let mut options = KeyspaceCreateOptions::default()
.manual_journal_persist(true)
.filter_block_pinning_policy(PinningPolicy::all(false))
.index_block_pinning_policy(PinningPolicy::all(false))
.max_memtable_size(8 * 1024 * 1024);
if bloom_filters.is_some_and(|b| !b) {
options = options.filter_policy(FilterPolicy::disabled());
} else {
options = options.filter_policy(FilterPolicy::all(FilterPolicyEntry::Bloom(
BloomConstructionPolicy::BitsPerKey(5.0),
)));
}
database.keyspace(name, options).map_err(|e| e.into())
}
pub fn import(
database: &Database,
path: &Path,
name: &str,
version: Version,
bloom_filters: Option<bool>,
bloom_filters: bool,
sequential: bool,
) -> Result<Self> {
fs::create_dir_all(path)?;
@@ -76,7 +54,7 @@ where
&path.join(format!("meta/{name}")),
MAJOR_FJALL_VERSION + version,
|| {
Self::open_keyspace(database, name, bloom_filters).inspect_err(|e| {
Self::open_keyspace(database, name, bloom_filters, sequential).inspect_err(|e| {
eprintln!("{e}");
eprintln!("Delete {path:?} and try again");
})
@@ -93,6 +71,34 @@ where
})
}
fn open_keyspace(
database: &Database,
name: &str,
bloom_filters: bool,
sequential: bool,
) -> Result<Keyspace> {
let mut options = KeyspaceCreateOptions::default().manual_journal_persist(true);
if bloom_filters {
options = options.filter_policy(FilterPolicy::new(&[
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(10.0)),
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(7.5)),
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(5.0)),
FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(2.5)),
]));
} else {
options = options.filter_policy(FilterPolicy::disabled());
}
if sequential {
options = options
.filter_block_pinning_policy(PinningPolicy::all(false))
.index_block_pinning_policy(PinningPolicy::all(false));
}
database.keyspace(name, options).map_err(|e| e.into())
}
#[inline]
pub fn get<'a>(&'a self, key: &'a K) -> Result<Option<Cow<'a, V>>>
where
@@ -114,6 +120,9 @@ where
#[inline]
pub fn is_empty(&self) -> Result<bool> {
// self.database
// .read_tx()
// .is_empty(&self.keyspace)
self.keyspace.is_empty().map_err(|e| e.into())
}
@@ -178,6 +187,7 @@ where
}
let mut batch = self.database.batch();
// let mut batch = self.database.inner().batch();
let mut items = mem::take(&mut self.puts)
.into_iter()
.map(|(key, value)| Item::Value { key, value })
@@ -188,26 +198,14 @@ where
)
.collect::<Vec<_>>();
items.sort_unstable();
batch.data = items
batch.ingest(
items
.into_iter()
.map(|i| i.fjall(&self.keyspace))
.collect::<Vec<_>>();
.map(|i| i.fjalled(&self.keyspace))
.collect::<Vec<_>>(),
);
batch.commit_keyspace(&self.keyspace)?;
// let mut wtx = self.database.write_tx();
// let mut dels = self.dels.drain().collect::<Vec<_>>();
// dels.sort_unstable();
// dels.into_iter()
// .for_each(|key| wtx.remove(&self.keyspace, ByteView::from(key)));
// let mut puts = self.puts.drain().collect::<Vec<_>>();
// puts.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));
// puts.into_iter().for_each(|(key, value)| {
// wtx.insert(&self.keyspace, ByteView::from(key), ByteView::from(value))
// });
// wtx.commit()?;
// batch.commit_keyspace(self.keyspace.inner())?;
Ok(())
}
@@ -265,20 +263,22 @@ impl<K, V> Item<K, V> {
}
}
pub fn fjall(self, keyspace: &Keyspace) -> fjall3::Item
pub fn fjalled(self, keyspace: &Keyspace) -> fjall3::Item
where
K: Into<ByteView>,
V: Into<ByteView>,
{
let keyspace_id = keyspace.id;
// let keyspace_id = keyspace.inner().id;
match self {
Item::Value { key, value } => fjall3::Item {
keyspace_id: keyspace.id,
keyspace_id,
key: key.into().into(),
value: value.into().into(),
value_type: ValueType::Value,
},
Item::Tomb(key) => fjall3::Item {
keyspace_id: keyspace.id,
keyspace_id,
key: key.into().into(),
value: [].into(),
value_type: ValueType::WeakTombstone,

2
crates/brk_store/src/lib.rs
View File

@@ -3,9 +3,7 @@
mod any;
mod fjall_v2;
mod fjall_v3;
mod redb;
pub use any::*;
pub use fjall_v2::*;
pub use fjall_v3::*;
pub use redb::*;

77
crates/brk_store/src/redb/meta.rs
View File

@@ -1,77 +0,0 @@
use std::{
fs, io,
path::{Path, PathBuf},
};
use brk_error::Result;
use brk_types::Version;
use super::Height;
#[derive(Debug, Clone)]
pub struct StoreMeta {
pathbuf: PathBuf,
version: Version,
height: Option<Height>,
}
impl StoreMeta {
pub fn checked_open(path: &Path, version: Version) -> Result<Self> {
fs::create_dir_all(path)?;
if Version::try_from(Self::path_version_(path).as_path())
.is_ok_and(|prev_version| version != prev_version)
{
todo!();
}
let slf = Self {
pathbuf: path.to_owned(),
version,
height: Height::try_from(Self::path_height_(path).as_path()).ok(),
};
slf.version.write(&slf.path_version())?;
Ok(slf)
}
pub fn version(&self) -> Version {
self.version
}
pub fn export(&mut self, height: Height) -> io::Result<()> {
self.height = Some(height);
height.write(&self.path_height())
}
pub fn path(&self) -> &Path {
&self.pathbuf
}
fn path_version(&self) -> PathBuf {
Self::path_version_(&self.pathbuf)
}
fn path_version_(path: &Path) -> PathBuf {
path.join("version")
}
#[inline]
pub fn height(&self) -> Option<Height> {
self.height
}
#[inline]
pub fn needs(&self, height: Height) -> bool {
self.height.is_none_or(|self_height| height > self_height)
}
#[inline]
pub fn has(&self, height: Height) -> bool {
!self.needs(height)
}
fn path_height(&self) -> PathBuf {
Self::path_height_(&self.pathbuf)
}
fn path_height_(path: &Path) -> PathBuf {
path.join("height")
}
}

232
crates/brk_store/src/redb/mod.rs
View File

@@ -1,232 +0,0 @@
use std::{
borrow::{Borrow, Cow},
fmt::Debug,
fs,
hash::Hash,
mem::{self, transmute},
path::Path,
sync::Arc,
};
use brk_error::Result;
use brk_types::{Height, Version};
use parking_lot::RwLock;
use redb::{
Builder, Database, Durability, Key, ReadOnlyTable, ReadableDatabase, ReadableTableMetadata,
TableDefinition, Value,
};
mod meta;
use meta::*;
use rustc_hash::{FxHashMap, FxHashSet};
use crate::any::AnyStore;
#[derive(Clone)]
pub struct StoreRedb<K, V>
where
K: Key + 'static,
V: Value + 'static,
{
meta: StoreMeta,
name: &'static str,
db: Arc<Database>,
table: Arc<RwLock<Option<ReadOnlyTable<K, V>>>>,
puts: FxHashMap<K, V>,
dels: FxHashSet<K>,
}
const MAJOR_FJALL_VERSION: Version = Version::new(3);
pub fn open_redb_database(path: &Path) -> redb::Result<Database> {
let db = Builder::new()
.set_cache_size(4 * 1024 * 1024 * 1024)
.create(path.join("store.redb"))
.unwrap();
Ok(db)
}
impl<K, V> StoreRedb<K, V>
where
K: Key + Ord + Eq + Hash + 'static,
V: Value + Clone + 'static,
{
pub fn import(
db: &Arc<Database>,
path: &Path,
name: &str,
version: Version,
_bloom_filters: Option<bool>,
) -> Result<Self> {
fs::create_dir_all(path)?;
let meta = StoreMeta::checked_open(
&path.join(format!("meta/{name}")),
MAJOR_FJALL_VERSION + version,
)?;
{
let mut wtx = db.begin_write().unwrap();
wtx.set_durability(Durability::Immediate).unwrap();
let definition: TableDefinition<K, V> = TableDefinition::new(name);
let table = wtx.open_table(definition).unwrap();
drop(table);
wtx.commit().unwrap();
}
let definition: TableDefinition<K, V> = TableDefinition::new(name);
let table = db.begin_read().unwrap().open_table(definition).unwrap();
Ok(Self {
db: db.clone(),
meta,
name: Box::leak(Box::new(name.to_string())),
table: Arc::new(RwLock::new(Some(table))),
puts: FxHashMap::default(),
dels: FxHashSet::default(),
})
}
// In case my hack doesn't work:
// https://github.com/cberner/redb/issues/869
#[inline]
pub fn get<'a>(&'a self, key: &'a K) -> Result<Option<Cow<'a, V>>>
where
&'a K: Borrow<K::SelfType<'a>>,
V: From<V::SelfType<'static>>,
{
if let Some(v) = self.puts.get(key) {
Ok(Some(Cow::Borrowed(v)))
} else if let Some(value) = self.table.read().as_ref().unwrap().get(key).unwrap() {
let selftype: <V as Value>::SelfType<'static> = unsafe { transmute(value.value()) };
let owned: V = selftype.into();
Ok(Some(Cow::Owned(owned)))
} else {
Ok(None)
}
}
#[inline]
pub fn is_empty(&self) -> Result<bool> {
Ok(self.table.read().as_ref().unwrap().len().unwrap() == 0)
}
#[inline]
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
if self.needs(height) {
self.insert(key, value);
}
}
#[inline]
pub fn insert(&mut self, key: K, value: V) {
let _ = self.dels.is_empty() || self.dels.remove(&key);
self.puts.insert(key, value);
}
#[inline]
pub fn remove(&mut self, key: K) {
// Hot path: key was recently inserted
if self.puts.remove(&key).is_some() {
return;
}
let newly_inserted = self.dels.insert(key);
debug_assert!(newly_inserted, "Double deletion at {:?}", self.meta.path());
}
#[inline]
pub fn remove_if_needed(&mut self, key: K, height: Height) {
if self.needs(height) {
self.remove(key)
}
}
#[inline]
fn has(&self, height: Height) -> bool {
self.meta.has(height)
}
#[inline]
fn needs(&self, height: Height) -> bool {
self.meta.needs(height)
}
}
impl<K, V> AnyStore for StoreRedb<K, V>
where
// K: Debug + Clone + From<ByteView> + Ord + Eq + Hash,
// V: Debug + Clone + From<ByteView>,
K: Debug + Clone + Key + Ord + Eq + Hash + 'static + Borrow<K::SelfType<'static>>,
V: Debug + Clone + Value + 'static + Borrow<V::SelfType<'static>>,
// ByteView: From<K> + From<V>,
Self: Send + Sync,
{
fn commit(&mut self, height: Height) -> Result<()> {
if self.has(height) {
return Ok(());
}
self.meta.export(height)?;
if self.puts.is_empty() && self.dels.is_empty() {
return Ok(());
}
// let mut _rtx_lock = self._rtx.write();
// drop(_rtx_lock.take());
let mut table_lock = self.table.write();
drop(table_lock.take());
let mut wtx = self.db.begin_write().unwrap();
wtx.set_durability(Durability::Immediate).unwrap();
let definition: TableDefinition<K, V> = TableDefinition::new(self.name);
let mut table = wtx.open_table(definition).unwrap();
mem::take(&mut self.puts)
.into_iter()
.for_each(|(key, value)| {
table.insert(key, value).unwrap();
});
mem::take(&mut self.dels).into_iter().for_each(|key| {
table.remove(key).unwrap();
});
drop(table);
wtx.commit().unwrap();
table_lock.replace(
self.db
.begin_read()
.unwrap()
.open_table(definition)
.unwrap(),
);
Ok(())
}
fn name(&self) -> &'static str {
self.name
}
fn height(&self) -> Option<Height> {
self.meta.height()
}
fn has(&self, height: Height) -> bool {
self.has(height)
}
fn needs(&self, height: Height) -> bool {
self.needs(height)
}
fn version(&self) -> Version {
self.meta.version()
}
}

1
crates/brk_types/Cargo.toml
View File

@@ -19,7 +19,6 @@ itoa = "1.0.15"
jiff = { workspace = true }
num_enum = "0.7.5"
rapidhash = "4.1.1"
redb = { workspace = true }
ryu = "1.0.20"
schemars = { workspace = true }
serde = { workspace = true }

40
crates/brk_types/src/addressbyteshash.rs
View File

@@ -1,8 +1,5 @@
use std::{cmp::Ordering, mem};
use byteview::ByteView;
use derive_deref::Deref;
use redb::{Key, TypeName, Value};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::AddressBytes;
@@ -51,40 +48,3 @@ impl From<&AddressBytesHash> for ByteView {
Self::new(value.as_bytes())
}
}
impl Value for AddressBytesHash {
type SelfType<'a> = AddressBytesHash;
type AsBytes<'a>
= [u8; mem::size_of::<u64>()]
where
Self: 'a;
fn fixed_width() -> Option<usize> {
Some(mem::size_of::<u64>())
}
fn from_bytes<'a>(data: &'a [u8]) -> AddressBytesHash
where
Self: 'a,
{
AddressBytesHash(u64::from_le_bytes(data.try_into().unwrap()))
}
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> [u8; mem::size_of::<u64>()]
where
Self: 'a,
Self: 'b,
{
value.0.to_le_bytes()
}
fn type_name() -> TypeName {
TypeName::new("AddressBytesHash")
}
}
impl Key for AddressBytesHash {
fn compare(data1: &[u8], data2: &[u8]) -> Ordering {
Self::from_bytes(data1).cmp(&Self::from_bytes(data2))
}
}

73
crates/brk_types/src/addresstypeaddressindexoutpoint.rs Normal file
View File

@@ -0,0 +1,73 @@
use std::hash::{Hash, Hasher};
use byteview::ByteView;
use serde::Serialize;
use zerocopy::IntoBytes;
use crate::{AddressTypeAddressIndexTxIndex, OutputType, Vout};
use super::{OutPoint, TypeIndex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize)]
#[repr(C)]
pub struct AddressTypeAddressIndexOutPoint {
addresstypeaddressindextxindex: AddressTypeAddressIndexTxIndex, // (u8; u64)
vout: Vout, // u16
}
impl Hash for AddressTypeAddressIndexOutPoint {
fn hash<H: Hasher>(&self, state: &mut H) {
let mut buf = [0u8; 11];
buf[..1].copy_from_slice(self.addresstypeaddressindextxindex.addresstype().as_bytes());
buf[1..9].copy_from_slice(
self.addresstypeaddressindextxindex
.addressindextxindex()
.as_bytes(),
);
buf[9..].copy_from_slice(self.vout.as_bytes());
state.write(&buf);
}
}
impl From<(OutputType, TypeIndex, OutPoint)> for AddressTypeAddressIndexOutPoint {
#[inline]
fn from((addresstype, addressindex, outpoint): (OutputType, TypeIndex, OutPoint)) -> Self {
Self {
addresstypeaddressindextxindex: AddressTypeAddressIndexTxIndex::from((
addresstype,
addressindex,
outpoint.txindex(),
)),
vout: outpoint.vout(),
}
}
}
impl From<ByteView> for AddressTypeAddressIndexOutPoint {
#[inline]
fn from(value: ByteView) -> Self {
Self {
addresstypeaddressindextxindex: AddressTypeAddressIndexTxIndex::from(&value[0..9]),
vout: Vout::from(&value[9..]),
}
}
}
impl From<AddressTypeAddressIndexOutPoint> for ByteView {
#[inline]
fn from(value: AddressTypeAddressIndexOutPoint) -> Self {
ByteView::from(&value)
}
}
impl From<&AddressTypeAddressIndexOutPoint> for ByteView {
#[inline]
fn from(value: &AddressTypeAddressIndexOutPoint) -> Self {
ByteView::from(
[
&ByteView::from(value.addresstypeaddressindextxindex),
value.vout.to_be_bytes().as_slice(),
]
.concat(),
)
}
}

88
crates/brk_types/src/addresstypeaddressindextxindex.rs Normal file
View File

@@ -0,0 +1,88 @@
use std::hash::{Hash, Hasher};
use byteview::ByteView;
use serde::Serialize;
use zerocopy::IntoBytes;
use crate::OutputType;
use super::{TxIndex, TypeIndex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Serialize)]
pub struct AddressTypeAddressIndexTxIndex {
addresstype: OutputType,
addressindextxindex: u64,
}
impl Hash for AddressTypeAddressIndexTxIndex {
fn hash<H: Hasher>(&self, state: &mut H) {
let mut buf = [0u8; 9];
buf[..1].copy_from_slice(self.addresstype.as_bytes());
buf[1..].copy_from_slice(self.addressindextxindex.as_bytes());
state.write(&buf);
}
}
impl AddressTypeAddressIndexTxIndex {
pub fn addresstype(&self) -> OutputType {
self.addresstype
}
pub fn addressindex(&self) -> u32 {
(self.addressindextxindex >> 32) as u32
}
pub fn txindex(&self) -> u32 {
self.addressindextxindex as u32
}
pub fn addressindextxindex(&self) -> u64 {
self.addressindextxindex
}
}
impl From<(OutputType, TypeIndex, TxIndex)> for AddressTypeAddressIndexTxIndex {
#[inline]
fn from((addresstype, addressindex, txindex): (OutputType, TypeIndex, TxIndex)) -> Self {
Self {
addresstype,
addressindextxindex: (u64::from(addressindex) << 32) | u64::from(txindex),
}
}
}
impl From<ByteView> for AddressTypeAddressIndexTxIndex {
#[inline]
fn from(value: ByteView) -> Self {
Self::from(&*value)
}
}
impl From<&[u8]> for AddressTypeAddressIndexTxIndex {
#[inline]
fn from(value: &[u8]) -> Self {
let addresstype = OutputType::from(&value[0..1]);
let addressindex = TypeIndex::from(&value[1..5]);
let txindex = TxIndex::from(&value[5..9]);
Self::from((addresstype, addressindex, txindex))
}
}
impl From<AddressTypeAddressIndexTxIndex> for ByteView {
#[inline]
fn from(value: AddressTypeAddressIndexTxIndex) -> Self {
ByteView::from(&value)
}
}
impl From<&AddressTypeAddressIndexTxIndex> for ByteView {
#[inline]
fn from(value: &AddressTypeAddressIndexTxIndex) -> Self {
ByteView::from(
[
value.addresstype.as_bytes(),
value.addressindextxindex.to_be_bytes().as_slice(),
]
.concat(),
)
}
}

40
crates/brk_types/src/blockhashprefix.rs
View File

@@ -1,8 +1,5 @@
use std::{cmp::Ordering, mem};
use byteview::ByteView;
use derive_deref::Deref;
use redb::{Key, TypeName, Value};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::copy_first_8bytes;
@@ -60,40 +57,3 @@ impl From<BlockHashPrefix> for ByteView {
Self::from(&value)
}
}
impl Value for BlockHashPrefix {
type SelfType<'a> = BlockHashPrefix;
type AsBytes<'a>
= [u8; mem::size_of::<u64>()]
where
Self: 'a;
fn fixed_width() -> Option<usize> {
Some(mem::size_of::<u64>())
}
fn from_bytes<'a>(data: &'a [u8]) -> BlockHashPrefix
where
Self: 'a,
{
BlockHashPrefix(u64::from_le_bytes(data.try_into().unwrap()))
}
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> [u8; mem::size_of::<u64>()]
where
Self: 'a,
Self: 'b,
{
value.0.to_le_bytes()
}
fn type_name() -> TypeName {
TypeName::new("BlockHashPrefix")
}
}
impl Key for BlockHashPrefix {
fn compare(data1: &[u8], data2: &[u8]) -> Ordering {
Self::from_bytes(data1).cmp(&Self::from_bytes(data2))
}
}

40
crates/brk_types/src/height.rs
View File

@@ -1,14 +1,11 @@
use std::{
cmp::Ordering,
fmt::Debug,
mem,
ops::{Add, AddAssign, Rem},
};
use allocative::Allocative;
use byteview::ByteView;
use derive_deref::Deref;
use redb::{Key, TypeName, Value};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use vecdb::{CheckedSub, PrintableIndex, Stamp, StoredCompressed};
@@ -283,40 +280,3 @@ impl std::fmt::Display for Height {
f.write_str(str)
}
}
impl Value for Height {
type SelfType<'a> = Height;
type AsBytes<'a>
= [u8; mem::size_of::<u32>()]
where
Self: 'a;
fn fixed_width() -> Option<usize> {
Some(mem::size_of::<u32>())
}
fn from_bytes<'a>(data: &'a [u8]) -> Height
where
Self: 'a,
{
Height(u32::from_le_bytes(data.try_into().unwrap()))
}
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> [u8; mem::size_of::<u32>()]
where
Self: 'a,
Self: 'b,
{
value.0.to_le_bytes()
}
fn type_name() -> TypeName {
TypeName::new("Height")
}
}
impl Key for Height {
fn compare(data1: &[u8], data2: &[u8]) -> Ordering {
Self::from_bytes(data1).cmp(&Self::from_bytes(data2))
}
}

8
crates/brk_types/src/lib.rs
View File

@@ -10,6 +10,8 @@ mod addressbyteshash;
mod addresschainstats;
mod addressmempoolstats;
mod addressstats;
mod addresstypeaddressindexoutpoint;
mod addresstypeaddressindextxindex;
mod anyaddressindex;
mod bitcoin;
mod blkmetadata;
@@ -92,8 +94,6 @@ mod txoutindex;
mod txstatus;
mod txversion;
mod typeindex;
mod typeindexandoutpoint;
mod typeindexandtxindex;
mod unit;
mod unknownoutputindex;
mod vin;
@@ -108,6 +108,8 @@ pub use addressbyteshash::*;
pub use addresschainstats::*;
pub use addressmempoolstats::*;
pub use addressstats::*;
pub use addresstypeaddressindexoutpoint::*;
pub use addresstypeaddressindextxindex::*;
pub use anyaddressindex::*;
pub use bitcoin::*;
pub use blkmetadata::*;
@@ -190,8 +192,6 @@ pub use txoutindex::*;
pub use txstatus::*;
pub use txversion::*;
pub use typeindex::*;
pub use typeindexandoutpoint::*;
pub use typeindexandtxindex::*;
pub use unit::*;
pub use unknownoutputindex::*;
pub use vin::*;

8
crates/brk_types/src/outputtype.rs
View File

@@ -20,6 +20,7 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
KnownLayout,
Serialize,
JsonSchema,
Hash,
)]
#[serde(rename_all = "lowercase")]
#[strum(serialize_all = "lowercase")]
@@ -906,3 +907,10 @@ impl TryFrom<OutputType> for AddressType {
})
}
}
impl From<&[u8]> for OutputType {
#[inline]
fn from(value: &[u8]) -> Self {
Self::read_from_bytes(value).unwrap()
}
}

34
crates/brk_types/src/stored_string.rs
View File

@@ -2,7 +2,6 @@ use std::{borrow::Cow, str};
use byteview::ByteView;
use derive_deref::Deref;
use redb::{TypeName, Value};
use serde::Serialize;
use vecdb::PrintableIndex;
@@ -68,36 +67,3 @@ impl PrintableIndex for StoredString {
&["string"]
}
}
impl Value for StoredString {
type SelfType<'a>
= StoredString
where
Self: 'a;
type AsBytes<'a>
= &'a str
where
Self: 'a;
fn fixed_width() -> Option<usize> {
None
}
fn from_bytes<'a>(data: &'a [u8]) -> StoredString
where
Self: 'a,
{
StoredString(str::from_utf8(data).unwrap().to_string())
}
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> &'a str
where
Self: 'b,
{
value.as_str()
}
fn type_name() -> TypeName {
TypeName::new("StoredString")
}
}

40
crates/brk_types/src/txidprefix.rs
View File

@@ -1,8 +1,5 @@
use std::{cmp::Ordering, mem};
use byteview::ByteView;
use derive_deref::Deref;
use redb::{Key, TypeName, Value};
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::copy_first_8bytes;
@@ -67,40 +64,3 @@ impl From<[u8; 8]> for TxidPrefix {
Self(u64::from_ne_bytes(value))
}
}
impl Value for TxidPrefix {
type SelfType<'a> = TxidPrefix;
type AsBytes<'a>
= [u8; mem::size_of::<u64>()]
where
Self: 'a;
fn fixed_width() -> Option<usize> {
Some(mem::size_of::<u64>())
}
fn from_bytes<'a>(data: &'a [u8]) -> TxidPrefix
where
Self: 'a,
{
TxidPrefix(u64::from_le_bytes(data.try_into().unwrap()))
}
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> [u8; mem::size_of::<u64>()]
where
Self: 'a,
Self: 'b,
{
value.0.to_le_bytes()
}
fn type_name() -> TypeName {
TypeName::new("TxidPrefix")
}
}
impl Key for TxidPrefix {
fn compare(data1: &[u8], data2: &[u8]) -> Ordering {
Self::from_bytes(data1).cmp(&Self::from_bytes(data2))
}
}

37
crates/brk_types/src/txindex.rs
View File

@@ -1,12 +1,8 @@
use std::{
mem,
ops::{Add, AddAssign},
};
use std::ops::{Add, AddAssign};
use allocative::Allocative;
use byteview::ByteView;
use derive_deref::{Deref, DerefMut};
use redb::{TypeName, Value};
use schemars::JsonSchema;
use serde::Serialize;
use vecdb::{CheckedSub, PrintableIndex, StoredCompressed};
@@ -170,34 +166,3 @@ impl std::fmt::Display for TxIndex {
f.write_str(str)
}
}
impl Value for TxIndex {
type SelfType<'a> = TxIndex;
type AsBytes<'a>
= [u8; mem::size_of::<u32>()]
where
Self: 'a;
fn fixed_width() -> Option<usize> {
Some(mem::size_of::<u32>())
}
fn from_bytes<'a>(data: &'a [u8]) -> TxIndex
where
Self: 'a,
{
TxIndex(u32::from_le_bytes(data.try_into().unwrap()))
}
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> [u8; mem::size_of::<u32>()]
where
Self: 'a,
Self: 'b,
{
value.0.to_le_bytes()
}
fn type_name() -> TypeName {
TypeName::new("TxIndex")
}
}

34
crates/brk_types/src/typeindex.rs
View File

@@ -1,7 +1,6 @@
use std::{mem, ops::Add};
use std::ops::Add;
use byteview::ByteView;
use redb::{TypeName, Value};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use vecdb::{CheckedSub, StoredCompressed};
@@ -151,34 +150,3 @@ impl std::fmt::Display for TypeIndex {
f.write_str(str)
}
}
impl Value for TypeIndex {
type SelfType<'a> = TypeIndex;
type AsBytes<'a>
= [u8; mem::size_of::<u32>()]
where
Self: 'a;
fn fixed_width() -> Option<usize> {
Some(mem::size_of::<u32>())
}
fn from_bytes<'a>(data: &'a [u8]) -> TypeIndex
where
Self: 'a,
{
TypeIndex(u32::from_le_bytes(data.try_into().unwrap()))
}
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> [u8; mem::size_of::<u32>()]
where
Self: 'a,
Self: 'b,
{
value.0.to_le_bytes()
}
fn type_name() -> TypeName {
TypeName::new("TypeIndex")
}
}

114
crates/brk_types/src/typeindexandoutpoint.rs
View File

@@ -1,114 +0,0 @@
use std::{
cmp::Ordering,
hash::{Hash, Hasher},
};
use byteview::ByteView;
use redb::{Key, TypeName, Value};
use serde::Serialize;
use zerocopy::IntoBytes;
use crate::{TypeIndexAndTxIndex, Vout};
use super::{OutPoint, TypeIndex};
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default, Serialize)]
#[repr(C)]
pub struct TypeIndexAndOutPoint {
typeindexandtxindex: TypeIndexAndTxIndex, // u64
vout: Vout, // u16
}
impl Hash for TypeIndexAndOutPoint {
fn hash<H: Hasher>(&self, state: &mut H) {
let mut buf = [0u8; 10];
buf[..8].copy_from_slice(self.typeindexandtxindex.as_bytes());
buf[8..].copy_from_slice(self.vout.as_bytes());
state.write(&buf);
}
}
impl From<(TypeIndex, OutPoint)> for TypeIndexAndOutPoint {
#[inline]
fn from(value: (TypeIndex, OutPoint)) -> Self {
Self {
typeindexandtxindex: TypeIndexAndTxIndex::from((value.0, value.1.txindex())),
vout: value.1.vout(),
}
}
}
impl From<ByteView> for TypeIndexAndOutPoint {
#[inline]
fn from(value: ByteView) -> Self {
Self {
typeindexandtxindex: TypeIndexAndTxIndex::from(&value[0..8]),
vout: Vout::from(&value[8..]),
}
}
}
impl From<TypeIndexAndOutPoint> for ByteView {
#[inline]
fn from(value: TypeIndexAndOutPoint) -> Self {
ByteView::from(&value)
}
}
impl From<&TypeIndexAndOutPoint> for ByteView {
#[inline]
fn from(value: &TypeIndexAndOutPoint) -> Self {
ByteView::from(
[
value.typeindexandtxindex.to_be_bytes().as_slice(),
value.vout.to_be_bytes().as_slice(),
]
.concat(),
)
}
}
impl Value for TypeIndexAndOutPoint {
type SelfType<'a> = TypeIndexAndOutPoint;
type AsBytes<'a>
= [u8; 10]
// 8 bytes (u64) + 2 bytes (u16)
where
Self: 'a;
fn fixed_width() -> Option<usize> {
Some(10) // 8 + 2
}
fn from_bytes<'a>(data: &'a [u8]) -> TypeIndexAndOutPoint
where
Self: 'a,
{
TypeIndexAndOutPoint {
typeindexandtxindex: TypeIndexAndTxIndex::from_bytes(&data[0..8]),
vout: Vout::from_bytes(&data[8..10]),
}
}
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> [u8; 10]
where
Self: 'a,
Self: 'b,
{
let mut bytes = [0u8; 10];
bytes[0..8].copy_from_slice(&<TypeIndexAndTxIndex as redb::Value>::as_bytes(
&value.typeindexandtxindex,
));
bytes[8..10].copy_from_slice(&<Vout as redb::Value>::as_bytes(&value.vout));
bytes
}
fn type_name() -> TypeName {
TypeName::new("TypeIndexAndOutPoint")
}
}
impl Key for TypeIndexAndOutPoint {
fn compare(data1: &[u8], data2: &[u8]) -> Ordering {
Self::from_bytes(data1).cmp(&Self::from_bytes(data2))
}
}

120
crates/brk_types/src/typeindexandtxindex.rs
View File

@@ -1,120 +0,0 @@
use std::{cmp::Ordering, mem};
use byteview::ByteView;
use redb::{Key, TypeName, Value};
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::{TxIndex, TypeIndex};
#[derive(
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Clone,
Copy,
Default,
Serialize,
Hash,
FromBytes,
Immutable,
IntoBytes,
KnownLayout,
)]
pub struct TypeIndexAndTxIndex(u64);
impl TypeIndexAndTxIndex {
pub fn typeindex(&self) -> u32 {
(self.0 >> 32) as u32
}
pub fn txindex(&self) -> u32 {
self.0 as u32
}
pub fn to_be_bytes(&self) -> [u8; 8] {
self.0.to_be_bytes()
}
}
impl From<(TypeIndex, TxIndex)> for TypeIndexAndTxIndex {
#[inline]
fn from((typeindex, txindex): (TypeIndex, TxIndex)) -> Self {
Self((u64::from(typeindex) << 32) | u64::from(txindex))
}
}
impl From<ByteView> for TypeIndexAndTxIndex {
#[inline]
fn from(value: ByteView) -> Self {
Self::from(&*value)
}
}
impl From<&[u8]> for TypeIndexAndTxIndex {
#[inline]
fn from(value: &[u8]) -> Self {
let typeindex = TypeIndex::from(&value[0..4]);
let txindex = TxIndex::from(&value[4..8]);
Self::from((typeindex, txindex))
}
}
impl From<TypeIndexAndTxIndex> for ByteView {
#[inline]
fn from(value: TypeIndexAndTxIndex) -> Self {
ByteView::from(&value)
}
}
impl From<&TypeIndexAndTxIndex> for ByteView {
#[inline]
fn from(value: &TypeIndexAndTxIndex) -> Self {
ByteView::from(value.0.to_be_bytes().as_slice())
}
}
impl From<TypeIndexAndTxIndex> for u64 {
#[inline]
fn from(value: TypeIndexAndTxIndex) -> Self {
value.0
}
}
impl Value for TypeIndexAndTxIndex {
type SelfType<'a> = TypeIndexAndTxIndex;
type AsBytes<'a>
= [u8; mem::size_of::<u64>()]
where
Self: 'a;
fn fixed_width() -> Option<usize> {
Some(mem::size_of::<u64>())
}
fn from_bytes<'a>(data: &'a [u8]) -> TypeIndexAndTxIndex
where
Self: 'a,
{
TypeIndexAndTxIndex(u64::from_le_bytes(data.try_into().unwrap()))
}
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> [u8; mem::size_of::<u64>()]
where
Self: 'a,
Self: 'b,
{
value.0.to_le_bytes()
}
fn type_name() -> TypeName {
TypeName::new("TypeIndexAndTxIndex")
}
}
impl Key for TypeIndexAndTxIndex {
fn compare(data1: &[u8], data2: &[u8]) -> Ordering {
Self::from_bytes(data1).cmp(&Self::from_bytes(data2))
}
}

36
crates/brk_types/src/unit.rs
View File

@@ -1,5 +1,4 @@
use byteview::ByteView;
use redb::{TypeName, Value};
#[derive(Debug, Clone)]
pub struct Unit;
@@ -16,38 +15,3 @@ impl From<Unit> for ByteView {
Self::new(&[])
}
}
impl Value for Unit {
type SelfType<'a>
= Unit
where
Self: 'a;
type AsBytes<'a>
= &'a [u8]
where
Self: 'a;
fn fixed_width() -> Option<usize> {
Some(0)
}
#[allow(clippy::unused_unit, clippy::semicolon_if_nothing_returned)]
fn from_bytes<'a>(_data: &'a [u8]) -> Unit
where
Self: 'a,
{
Unit
}
#[allow(clippy::ignored_unit_patterns)]
fn as_bytes<'a, 'b: 'a>(_: &'a Self::SelfType<'b>) -> &'a [u8]
where
Self: 'b,
{
&[]
}
fn type_name() -> TypeName {
TypeName::new("Unit")
}
}

34
crates/brk_types/src/vout.rs
View File

@@ -1,8 +1,5 @@
use std::mem;
use allocative::Allocative;
use derive_deref::Deref;
use redb::{TypeName, Value};
use schemars::JsonSchema;
use serde::Serialize;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
@@ -110,34 +107,3 @@ impl std::fmt::Display for Vout {
self.0.fmt(f)
}
}
impl Value for Vout {
type SelfType<'a> = Vout;
type AsBytes<'a>
= [u8; mem::size_of::<u16>()]
where
Self: 'a;
fn fixed_width() -> Option<usize> {
Some(mem::size_of::<u16>())
}
fn from_bytes<'a>(data: &'a [u8]) -> Vout
where
Self: 'a,
{
Vout(u16::from_le_bytes(data.try_into().unwrap()))
}
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> [u8; mem::size_of::<u16>()]
where
Self: 'a,
Self: 'b,
{
value.0.to_le_bytes()
}
fn type_name() -> TypeName {
TypeName::new("Vout")
}
}