mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-27 06:34:30 -07:00
Compare commits
26 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 nym21
|
be9569f3fb | ||
|
nym21
|
900e72f95a | ||
|
nym21
|
d2827f188b | ||
|
nym21
|
cf9903b759 | ||
|
nym21
|
23f96461f4 | ||
|
nym21
|
9f2fd26e98 | ||
|
nym21
|
78d837c080 | ||
|
nym21
|
241b9312b7 | ||
|
nym21
|
ed70ad7378 | ||
|
nym21
|
00213176d8 | ||
|
nym21
|
406650a45a | ||
|
nym21
|
56750ccf3c | ||
|
nym21
|
dfc286b393 | ||
|
nym21
|
49a66f72fc | ||
|
nym21
|
3f237689da | ||
|
nym21
|
cf1fb483b3 | ||
|
nym21
|
b10f5e3f67 | ||
|
nym21
|
c4fc24c513 | ||
|
nym21
|
3ac9c2d95e | ||
|
nym21
|
e5ab4dafc0 | ||
|
nym21
|
10ae1911c3 | ||
|
nym21
|
73ebcdf0d6 | ||
|
nym21
|
5347523921 | ||
|
nym21
|
7ef70b953b | ||
|
nym21
|
ccaca524fe | ||
|
nym21
|
dd51f91cab |
@@ -26,3 +26,6 @@ _*
|
||||
profile.json.gz
|
||||
flamegraph.svg
|
||||
*.trace
|
||||
|
||||
# AI
|
||||
CLAUDE.md
|
||||
|
||||
Generated
+58
-55
@@ -144,12 +144,6 @@ dependencies = [
|
||||
"derive_arbitrary",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arc-swap"
|
||||
version = "1.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
|
||||
|
||||
[[package]]
|
||||
name = "arcstr"
|
||||
version = "1.2.0"
|
||||
@@ -477,7 +471,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"brk_bundler",
|
||||
"brk_cli",
|
||||
@@ -492,12 +486,12 @@ dependencies = [
|
||||
"brk_parser",
|
||||
"brk_server",
|
||||
"brk_store",
|
||||
"brk_vec",
|
||||
"brk_vecs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brk_bundler"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"brk_rolldown",
|
||||
"log",
|
||||
@@ -508,7 +502,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_cli"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"bitcoincore-rpc",
|
||||
"brk_computer",
|
||||
@@ -519,7 +513,7 @@ dependencies = [
|
||||
"brk_logger",
|
||||
"brk_parser",
|
||||
"brk_server",
|
||||
"brk_vec",
|
||||
"brk_vecs",
|
||||
"clap",
|
||||
"clap_derive",
|
||||
"color-eyre",
|
||||
@@ -531,7 +525,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_computer"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"bitcoin",
|
||||
@@ -542,12 +536,9 @@ dependencies = [
|
||||
"brk_indexer",
|
||||
"brk_logger",
|
||||
"brk_parser",
|
||||
"brk_store",
|
||||
"brk_vec",
|
||||
"brk_vecs",
|
||||
"color-eyre",
|
||||
"derive_deref",
|
||||
"either",
|
||||
"fjall",
|
||||
"log",
|
||||
"rayon",
|
||||
"serde",
|
||||
@@ -557,7 +548,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_core"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"bitcoin",
|
||||
@@ -578,16 +569,17 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_exit"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"brk_logger",
|
||||
"ctrlc",
|
||||
"log",
|
||||
"parking_lot",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brk_fetcher"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"brk_core",
|
||||
"brk_logger",
|
||||
@@ -599,7 +591,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_indexer"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"bitcoincore-rpc",
|
||||
@@ -608,7 +600,7 @@ dependencies = [
|
||||
"brk_logger",
|
||||
"brk_parser",
|
||||
"brk_store",
|
||||
"brk_vec",
|
||||
"brk_vecs",
|
||||
"color-eyre",
|
||||
"fjall",
|
||||
"log",
|
||||
@@ -617,12 +609,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_interface"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"brk_computer",
|
||||
"brk_core",
|
||||
"brk_indexer",
|
||||
"brk_vec",
|
||||
"brk_vecs",
|
||||
"color-eyre",
|
||||
"derive_deref",
|
||||
"rmcp",
|
||||
@@ -635,7 +627,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_logger"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"color-eyre",
|
||||
"env_logger",
|
||||
@@ -645,7 +637,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_mcp"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"brk_interface",
|
||||
@@ -655,7 +647,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_parser"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"bitcoin",
|
||||
"bitcoincore-rpc",
|
||||
@@ -981,7 +973,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_server"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"bitcoincore-rpc",
|
||||
@@ -995,7 +987,7 @@ dependencies = [
|
||||
"brk_logger",
|
||||
"brk_mcp",
|
||||
"brk_parser",
|
||||
"brk_vec",
|
||||
"brk_vecs",
|
||||
"clap",
|
||||
"clap_derive",
|
||||
"color-eyre",
|
||||
@@ -1012,7 +1004,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brk_store"
|
||||
version = "0.0.81"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"brk_core",
|
||||
"byteview",
|
||||
@@ -1033,16 +1025,17 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brk_vec"
|
||||
version = "0.0.81"
|
||||
name = "brk_vecs"
|
||||
version = "0.0.82"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"brk_core",
|
||||
"brk_exit",
|
||||
"clap",
|
||||
"clap_derive",
|
||||
"libc",
|
||||
"log",
|
||||
"memmap2",
|
||||
"parking_lot",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -1125,9 +1118,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.29"
|
||||
version = "1.2.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362"
|
||||
checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
"libc",
|
||||
@@ -1721,9 +1714,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
|
||||
|
||||
[[package]]
|
||||
name = "fjall"
|
||||
version = "2.11.1"
|
||||
version = "2.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f5cb653019268f6dc8de3b254b633a2d233a775054349b804b9cfbf18bbe3426"
|
||||
checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"byteview",
|
||||
@@ -2037,9 +2030,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hyper-util"
|
||||
version = "0.1.15"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f66d5bd4c6f02bf0542fad85d626775bab9258cf795a4256dcaf3161114d1df"
|
||||
checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
@@ -2158,9 +2151,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "io-uring"
|
||||
version = "0.7.8"
|
||||
version = "0.7.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013"
|
||||
checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4"
|
||||
dependencies = [
|
||||
"bitflags 2.9.1",
|
||||
"cfg-if",
|
||||
@@ -2330,9 +2323,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.4"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638"
|
||||
checksum = "4488594b9328dee448adb906d8b126d9b7deb7cf5c22161ee591610bb1be83c0"
|
||||
dependencies = [
|
||||
"bitflags 2.9.1",
|
||||
"libc",
|
||||
@@ -2372,9 +2365,9 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
|
||||
|
||||
[[package]]
|
||||
name = "lsm-tree"
|
||||
version = "2.10.1"
|
||||
version = "2.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d2bd4cdc451a8dcf11329190afb9b78eb8988bed07a3da29b8d73d2e0c731ff"
|
||||
checksum = "55b6d7475a8dd22e749186968daacf8e2a77932b061b1bd263157987bbfc0c6c"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"crossbeam-skiplist",
|
||||
@@ -3105,6 +3098,16 @@ dependencies = [
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.11"
|
||||
@@ -3352,9 +3355,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quick_cache"
|
||||
version = "0.6.14"
|
||||
version = "0.6.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b450dad8382b1b95061d5ca1eb792081fb082adf48c678791fe917509596d5f"
|
||||
checksum = "8565e62e02af316570d4b492f17af1481d6c07cea60f4e7edd71700da5052ba9"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.15.4",
|
||||
@@ -3398,9 +3401,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.1"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
|
||||
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
|
||||
dependencies = [
|
||||
"rand_chacha 0.9.0",
|
||||
"rand_core 0.9.3",
|
||||
@@ -3446,9 +3449,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rapidhash"
|
||||
version = "1.4.0"
|
||||
version = "2.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9813f789f95ee4fe6b4d01834404d7cccacbc3f6c029343af910b3c2835eb9f1"
|
||||
checksum = "2297bf4643b32dc85bf5f622c9a41a268db6c6a6a2a2e3b405958a5b014ad9e1"
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
@@ -3472,9 +3475,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.13"
|
||||
version = "0.5.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6"
|
||||
checksum = "7e8af0dde094006011e6a740d4879319439489813bd0bcdc7d821beaeeff48ec"
|
||||
dependencies = [
|
||||
"bitflags 2.9.1",
|
||||
]
|
||||
@@ -3591,7 +3594,7 @@ dependencies = [
|
||||
"http-body-util",
|
||||
"paste",
|
||||
"pin-project-lite",
|
||||
"rand 0.9.1",
|
||||
"rand 0.9.2",
|
||||
"rmcp-macros",
|
||||
"schemars 1.0.4",
|
||||
"serde",
|
||||
@@ -3852,9 +3855,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.140"
|
||||
version = "1.0.141"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
|
||||
checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3"
|
||||
dependencies = [
|
||||
"indexmap 2.10.0",
|
||||
"itoa",
|
||||
|
||||
+23
-19
@@ -4,7 +4,7 @@ members = ["crates/*"]
|
||||
package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
|
||||
package.license = "MIT"
|
||||
package.edition = "2024"
|
||||
package.version = "0.0.81"
|
||||
package.version = "0.0.82"
|
||||
package.homepage = "https://bitcoinresearchkit.org"
|
||||
package.repository = "https://github.com/bitcoinresearchkit/brk"
|
||||
package.readme = "README.md"
|
||||
@@ -22,41 +22,45 @@ debug = true
|
||||
inherits = "release"
|
||||
|
||||
[workspace.dependencies]
|
||||
arc-swap = "1.7.1"
|
||||
axum = "0.8.4"
|
||||
bincode = { version = "2.0.1", features = ["serde"] }
|
||||
bitcoin = { version = "0.32.6", features = ["serde"] }
|
||||
bitcoincore-rpc = "0.19.0"
|
||||
brk_bundler = { version = "0.0.81", path = "crates/brk_bundler" }
|
||||
brk_cli = { version = "0.0.81", path = "crates/brk_cli" }
|
||||
brk_computer = { version = "0.0.81", path = "crates/brk_computer" }
|
||||
brk_core = { version = "0.0.81", path = "crates/brk_core" }
|
||||
brk_exit = { version = "0.0.81", path = "crates/brk_exit" }
|
||||
brk_fetcher = { version = "0.0.81", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0.0.81", path = "crates/brk_indexer" }
|
||||
brk_interface = { version = "0.0.81", path = "crates/brk_interface" }
|
||||
brk_logger = { version = "0.0.81", path = "crates/brk_logger" }
|
||||
brk_mcp = { version = "0.0.81", path = "crates/brk_mcp" }
|
||||
brk_parser = { version = "0.0.81", path = "crates/brk_parser" }
|
||||
brk_server = { version = "0.0.81", path = "crates/brk_server" }
|
||||
brk_store = { version = "0.0.81", path = "crates/brk_store" }
|
||||
brk_vec = { version = "0.0.81", path = "crates/brk_vec" }
|
||||
brk_bundler = { version = "0.0.82", path = "crates/brk_bundler" }
|
||||
brk_cli = { version = "0.0.82", path = "crates/brk_cli" }
|
||||
brk_computer = { version = "0.0.82", path = "crates/brk_computer" }
|
||||
brk_core = { version = "0.0.82", path = "crates/brk_core" }
|
||||
brk_exit = { version = "0.0.82", path = "crates/brk_exit" }
|
||||
brk_fetcher = { version = "0.0.82", path = "crates/brk_fetcher" }
|
||||
brk_indexer = { version = "0.0.82", path = "crates/brk_indexer" }
|
||||
brk_interface = { version = "0.0.82", path = "crates/brk_interface" }
|
||||
brk_logger = { version = "0.0.82", path = "crates/brk_logger" }
|
||||
brk_mcp = { version = "0.0.82", path = "crates/brk_mcp" }
|
||||
brk_parser = { version = "0.0.82", path = "crates/brk_parser" }
|
||||
brk_server = { version = "0.0.82", path = "crates/brk_server" }
|
||||
brk_store = { version = "0.0.82", path = "crates/brk_store" }
|
||||
brk_vecs = { version = "0.0.82", path = "crates/brk_vecs" }
|
||||
byteview = "=0.6.1"
|
||||
clap = { version = "4.5.41", features = ["string"] }
|
||||
clap_derive = "4.5.41"
|
||||
color-eyre = "0.6.5"
|
||||
derive_deref = "1.1.1"
|
||||
fjall = "2.11.1"
|
||||
fjall = "2.11.2"
|
||||
jiff = "0.2.15"
|
||||
libc = "0.2.174"
|
||||
log = { version = "0.4.27" }
|
||||
minreq = { version = "2.14.0", features = ["https", "serde_json"] }
|
||||
parking_lot = "0.12.4"
|
||||
rayon = "1.10.0"
|
||||
rmcp = { version = "0.3.0", features = ["transport-worker", "transport-streamable-http-server" ] }
|
||||
rmcp = { version = "0.3.0", features = [
|
||||
"transport-worker",
|
||||
"transport-streamable-http-server",
|
||||
] }
|
||||
schemars = "1.0.4"
|
||||
serde = { version = "1.0.219" }
|
||||
serde_bytes = "0.11.17"
|
||||
serde_derive = "1.0.219"
|
||||
serde_json = { version = "1.0.140", features = ["float_roundtrip"] }
|
||||
serde_json = { version = "1.0.141", features = ["float_roundtrip"] }
|
||||
tabled = "0.20.0"
|
||||
tokio = { version = "1.46.1", features = ["rt-multi-thread"] }
|
||||
zerocopy = { version = "0.8.26" }
|
||||
|
||||
@@ -22,7 +22,7 @@ full = [
|
||||
"interface",
|
||||
"server",
|
||||
"store",
|
||||
"vec",
|
||||
"vecs",
|
||||
]
|
||||
bundler = ["brk_bundler"]
|
||||
core = ["brk_core"]
|
||||
@@ -36,7 +36,7 @@ parser = ["brk_parser"]
|
||||
interface = ["brk_interface"]
|
||||
server = ["brk_server"]
|
||||
store = ["brk_store"]
|
||||
vec = ["brk_vec"]
|
||||
vecs = ["brk_vecs"]
|
||||
|
||||
[dependencies]
|
||||
brk_bundler = { workspace = true, optional = true }
|
||||
@@ -52,7 +52,7 @@ brk_parser = { workspace = true, optional = true }
|
||||
brk_interface = { workspace = true, optional = true }
|
||||
brk_server = { workspace = true, optional = true }
|
||||
brk_store = { workspace = true, optional = true }
|
||||
brk_vec = { workspace = true, optional = true }
|
||||
brk_vecs = { workspace = true, optional = true }
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
all-features = true
|
||||
|
||||
@@ -51,6 +51,6 @@ pub use brk_server as server;
|
||||
#[doc(inline)]
|
||||
pub use brk_store as store;
|
||||
|
||||
#[cfg(feature = "vec")]
|
||||
#[cfg(feature = "vecs")]
|
||||
#[doc(inline)]
|
||||
pub use brk_vec as vec;
|
||||
pub use brk_vecs as vecs;
|
||||
|
||||
@@ -17,7 +17,7 @@ brk_indexer = { workspace = true }
|
||||
brk_logger = { workspace = true }
|
||||
brk_parser = { workspace = true }
|
||||
brk_server = { workspace = true }
|
||||
brk_vec = { workspace = true }
|
||||
brk_vecs = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
clap_derive = { workspace = true }
|
||||
color-eyre = { workspace = true }
|
||||
|
||||
@@ -7,13 +7,12 @@ use bitcoincore_rpc::{self, Auth, Client};
|
||||
use brk_core::{default_bitcoin_path, default_brk_path, default_on_error, dot_brk_path};
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_server::Website;
|
||||
use brk_vec::{Computation, Format};
|
||||
use brk_vecs::{Computation, Format};
|
||||
use clap::Parser;
|
||||
use clap_derive::Parser;
|
||||
use color_eyre::eyre::eyre;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
|
||||
#[derive(Parser, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
|
||||
#[command(version, about)]
|
||||
pub struct Config {
|
||||
@@ -32,7 +31,6 @@ pub struct Config {
|
||||
#[arg(long, value_name = "PATH")]
|
||||
brkdir: Option<String>,
|
||||
|
||||
|
||||
/// Computation of computed datasets, `lazy` computes data whenever requested without saving it, `eager` computes the data once and saves it to disk, default: `lazy`, saved
|
||||
#[serde(default, deserialize_with = "default_on_error")]
|
||||
#[arg(short, long)]
|
||||
@@ -124,7 +122,6 @@ impl Config {
|
||||
config_saved.brkdir = Some(brkdir);
|
||||
}
|
||||
|
||||
|
||||
if let Some(computation) = config_args.computation.take() {
|
||||
config_saved.computation = Some(computation);
|
||||
}
|
||||
@@ -290,15 +287,10 @@ impl Config {
|
||||
.map_or_else(default_brk_path, |s| Self::fix_user_path(s.as_ref()))
|
||||
}
|
||||
|
||||
pub fn outputsdir(&self) -> PathBuf {
|
||||
self.brkdir().join("outputs")
|
||||
}
|
||||
|
||||
pub fn harsdir(&self) -> PathBuf {
|
||||
self.outputsdir().join("hars")
|
||||
self.brkdir().join("hars")
|
||||
}
|
||||
|
||||
|
||||
fn path_cookiefile(&self) -> PathBuf {
|
||||
self.rpccookiefile.as_ref().map_or_else(
|
||||
|| self.bitcoindir().join(".cookie"),
|
||||
|
||||
@@ -18,7 +18,7 @@ pub fn run() -> color_eyre::Result<()> {
|
||||
|
||||
let format = config.format();
|
||||
|
||||
let mut indexer = Indexer::forced_import(&config.outputsdir())?;
|
||||
let mut indexer = Indexer::forced_import(&config.brkdir())?;
|
||||
|
||||
let wait_for_synced_node = |rpc_client: &bitcoincore_rpc::Client| -> color_eyre::Result<()> {
|
||||
let is_synced = || -> color_eyre::Result<bool> {
|
||||
@@ -37,7 +37,7 @@ pub fn run() -> color_eyre::Result<()> {
|
||||
};
|
||||
|
||||
let mut computer = Computer::forced_import(
|
||||
&config.outputsdir(),
|
||||
&config.brkdir(),
|
||||
&indexer,
|
||||
config.computation(),
|
||||
config.fetcher(),
|
||||
|
||||
@@ -17,12 +17,9 @@ brk_fetcher = { workspace = true }
|
||||
brk_indexer = { workspace = true }
|
||||
brk_logger = { workspace = true }
|
||||
brk_parser = { workspace = true }
|
||||
brk_store = { workspace = true }
|
||||
brk_vec = { workspace = true }
|
||||
brk_vecs = { workspace = true }
|
||||
color-eyre = { workspace = true }
|
||||
derive_deref = { workspace = true }
|
||||
either = "1.15.0"
|
||||
fjall = { workspace = true }
|
||||
log = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
|
||||
@@ -5,7 +5,7 @@ use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_parser::Parser;
|
||||
use brk_vec::{Computation, Format};
|
||||
use brk_vecs::{Computation, Format};
|
||||
|
||||
pub fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use std::path::Path;
|
||||
use std::{path::Path, sync::Arc};
|
||||
|
||||
use brk_core::Version;
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, Computation, Format};
|
||||
use brk_vecs::{AnyCollectableVec, Computation, File, Format};
|
||||
use log::info;
|
||||
|
||||
use crate::{blocks, cointime, constants, fetched, indexes, market, mining, transactions};
|
||||
@@ -27,16 +27,19 @@ pub struct Vecs {
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
version: Version,
|
||||
indexer: &Indexer,
|
||||
fetch: bool,
|
||||
computation: Computation,
|
||||
format: Format,
|
||||
fetched_file: &Arc<File>,
|
||||
states_path: &Path,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let indexes = indexes::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
version + VERSION + Version::ZERO,
|
||||
indexer,
|
||||
computation,
|
||||
@@ -45,7 +48,8 @@ impl Vecs {
|
||||
|
||||
let fetched = fetch.then(|| {
|
||||
fetched::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
fetched_file,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
@@ -56,43 +60,44 @@ impl Vecs {
|
||||
|
||||
Ok(Self {
|
||||
blocks: blocks::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
&indexes,
|
||||
)?,
|
||||
mining: mining::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
&indexes,
|
||||
)?,
|
||||
constants: constants::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
&indexes,
|
||||
)?,
|
||||
market: market::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
&indexes,
|
||||
)?,
|
||||
stateful: stateful::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
&indexes,
|
||||
fetched.as_ref(),
|
||||
states_path,
|
||||
)?,
|
||||
transactions: transactions::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
version + VERSION + Version::ZERO,
|
||||
indexer,
|
||||
&indexes,
|
||||
@@ -101,7 +106,7 @@ impl Vecs {
|
||||
fetched.as_ref(),
|
||||
)?,
|
||||
cointime: cointime::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
version + VERSION + Version::ZERO,
|
||||
computation,
|
||||
format,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{
|
||||
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, StoredUsize,
|
||||
@@ -6,7 +6,7 @@ use brk_core::{
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Format};
|
||||
use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, EagerVec, File, Format};
|
||||
|
||||
use crate::grouped::Source;
|
||||
|
||||
@@ -34,7 +34,7 @@ pub struct Vecs {
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
version: Version,
|
||||
computation: Computation,
|
||||
format: Format,
|
||||
@@ -42,13 +42,13 @@ impl Vecs {
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
height_to_interval: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"interval",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"timestamp",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -58,7 +58,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_first(),
|
||||
)?,
|
||||
indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"block_interval",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -71,7 +71,7 @@ impl Vecs {
|
||||
.add_average(),
|
||||
)?,
|
||||
indexes_to_block_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"block_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -81,7 +81,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"block_weight",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -91,7 +91,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
indexes_to_block_size: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"block_size",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -101,13 +101,13 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
height_to_vbytes: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"vbytes",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"block_vbytes",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -117,13 +117,13 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
difficultyepoch_to_timestamp: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"timestamp",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
halvingepoch_to_timestamp: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"timestamp",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{Bitcoin, CheckedSub, Dollars, StoredF64, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
|
||||
use brk_vecs::{AnyCollectableVec, Computation, File, Format, VecIterator};
|
||||
|
||||
use super::{
|
||||
Indexes, fetched,
|
||||
@@ -46,7 +46,7 @@ pub struct Vecs {
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
version: Version,
|
||||
computation: Computation,
|
||||
format: Format,
|
||||
@@ -57,7 +57,7 @@ impl Vecs {
|
||||
|
||||
Ok(Self {
|
||||
indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"coinblocks_created",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -67,7 +67,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"coinblocks_stored",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -77,7 +77,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
indexes_to_liveliness: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"liveliness",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -87,7 +87,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_vaultedness: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"vaultedness",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -97,7 +97,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"activity_to_vaultedness_ratio",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -107,7 +107,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"vaulted_supply",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -118,7 +118,7 @@ impl Vecs {
|
||||
indexes,
|
||||
)?,
|
||||
indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"active_supply",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -129,7 +129,7 @@ impl Vecs {
|
||||
indexes,
|
||||
)?,
|
||||
indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"thermo_cap",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -139,7 +139,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_investor_cap: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"investor_cap",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -149,7 +149,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"vaulted_cap",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -159,7 +159,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_active_cap: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"active_cap",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -169,7 +169,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"vaulted_price",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -179,7 +179,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"vaulted_price",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -189,7 +189,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
indexes_to_active_price: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"active_price",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -199,7 +199,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"active_price",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -209,7 +209,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"true_market_mean",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -219,7 +219,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"true_market_mean",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -229,7 +229,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"cointime_value_destroyed",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -239,7 +239,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"cointime_value_created",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -249,7 +249,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"cointime_value_stored",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -259,7 +259,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
indexes_to_cointime_price: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"cointime_price",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -269,7 +269,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"cointime_cap",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -279,7 +279,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"cointime_price",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{StoredU8, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyVec, Computation, Format};
|
||||
use brk_vecs::{AnyCollectableVec, AnyVec, Computation, File, Format};
|
||||
|
||||
use crate::grouped::Source;
|
||||
|
||||
@@ -25,7 +25,7 @@ pub struct Vecs {
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
version: Version,
|
||||
computation: Computation,
|
||||
format: Format,
|
||||
@@ -33,7 +33,7 @@ impl Vecs {
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
constant_0: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"constant_0",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -43,7 +43,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
constant_1: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"constant_1",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -53,7 +53,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
constant_50: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"constant_50",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -63,7 +63,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
constant_100: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"constant_100",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{
|
||||
Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex,
|
||||
@@ -8,8 +8,8 @@ use brk_core::{
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, StoredIndex,
|
||||
use brk_vecs::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, File, Format, StoredIndex,
|
||||
VecIterator,
|
||||
};
|
||||
|
||||
@@ -76,103 +76,100 @@ const VERSION_IN_SATS: Version = Version::ZERO;
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
fetched_file: &Arc<File>,
|
||||
version: Version,
|
||||
computation: Computation,
|
||||
format: Format,
|
||||
indexes: &indexes::Vecs,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let mut fetched_path = path.to_owned();
|
||||
fetched_path.pop();
|
||||
fetched_path = fetched_path.join("fetched");
|
||||
|
||||
Ok(Self {
|
||||
dateindex_to_ohlc_in_cents: EagerVec::forced_import(
|
||||
&fetched_path,
|
||||
fetched_file,
|
||||
"ohlc_in_cents",
|
||||
version + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_ohlc: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_close_in_cents: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"close_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_high_in_cents: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"high_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_low_in_cents: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"low_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_open_in_cents: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"open_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_ohlc_in_cents: EagerVec::forced_import(
|
||||
&fetched_path,
|
||||
fetched_file,
|
||||
"ohlc_in_cents",
|
||||
version + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_ohlc: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_close_in_cents: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"close_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_high_in_cents: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"high_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_low_in_cents: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"low_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_open_in_cents: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"open_in_cents",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
timeindexes_to_open: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"open",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -182,7 +179,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_first(),
|
||||
)?,
|
||||
timeindexes_to_high: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"high",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -192,7 +189,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_max(),
|
||||
)?,
|
||||
timeindexes_to_low: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"low",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -202,7 +199,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_min(),
|
||||
)?,
|
||||
timeindexes_to_close: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"close",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -212,7 +209,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"open_in_sats",
|
||||
Source::Compute,
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
@@ -222,7 +219,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_first(),
|
||||
)?,
|
||||
timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"high_in_sats",
|
||||
Source::Compute,
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
@@ -232,7 +229,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_max(),
|
||||
)?,
|
||||
timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"low_in_sats",
|
||||
Source::Compute,
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
@@ -242,7 +239,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_min(),
|
||||
)?,
|
||||
timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"close_in_sats",
|
||||
Source::Compute,
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
@@ -252,143 +249,143 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
file,
|
||||
"open",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
VecBuilderOptions::default().add_first(),
|
||||
)?,
|
||||
chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
file,
|
||||
"high",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
VecBuilderOptions::default().add_max(),
|
||||
)?,
|
||||
chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
file,
|
||||
"low",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
VecBuilderOptions::default().add_min(),
|
||||
)?,
|
||||
chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
file,
|
||||
"close",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
file,
|
||||
"open_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
VecBuilderOptions::default().add_first(),
|
||||
)?,
|
||||
chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
file,
|
||||
"high_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
VecBuilderOptions::default().add_max(),
|
||||
)?,
|
||||
chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
file,
|
||||
"low_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
VecBuilderOptions::default().add_min(),
|
||||
)?,
|
||||
chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import(
|
||||
path,
|
||||
file,
|
||||
"close_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
weekindex_to_ohlc: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
weekindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
difficultyepoch_to_ohlc: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
difficultyepoch_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
monthindex_to_ohlc: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
monthindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
quarterindex_to_ohlc: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
quarterindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
semesterindex_to_ohlc: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
semesterindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
yearindex_to_ohlc: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
yearindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
// halvingepoch_to_ohlc: StorableVec::forced_import(path,
|
||||
// halvingepoch_to_ohlc: StorableVec::forced_import(file,
|
||||
// "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?,
|
||||
decadeindex_to_ohlc: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
decadeindex_to_ohlc_in_sats: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"ohlc_in_sats",
|
||||
version + VERSION + VERSION_IN_SATS + Version::ZERO,
|
||||
format,
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{FromCoarserIndex, Result, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_vec::{
|
||||
use brk_vecs::{
|
||||
AnyCollectableVec, AnyIterableVec, BoxedAnyIterableVec, CloneableAnyIterableVec, Computation,
|
||||
ComputedVec, ComputedVecFrom2, Format, StoredIndex,
|
||||
ComputedVec, ComputedVecFrom2, File, Format, StoredIndex,
|
||||
};
|
||||
|
||||
use crate::grouped::{EagerVecBuilder, VecBuilderOptions};
|
||||
@@ -39,7 +39,7 @@ where
|
||||
{
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
version: Version,
|
||||
format: Format,
|
||||
@@ -66,7 +66,7 @@ where
|
||||
Box::new(
|
||||
ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("first"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -91,7 +91,7 @@ where
|
||||
Box::new(
|
||||
ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -100,7 +100,7 @@ where
|
||||
source
|
||||
.as_ref()
|
||||
.unwrap_or_else(|| {
|
||||
dbg!(path, name, I::to_string());
|
||||
dbg!(file, name, I::to_string());
|
||||
panic!()
|
||||
})
|
||||
.clone()
|
||||
@@ -124,7 +124,7 @@ where
|
||||
Box::new(
|
||||
ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("min"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -149,7 +149,7 @@ where
|
||||
Box::new(
|
||||
ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("max"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -174,7 +174,7 @@ where
|
||||
Box::new(
|
||||
ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("average"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -206,7 +206,7 @@ where
|
||||
Box::new(
|
||||
ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
&(if !options.last && !options.average && !options.min && !options.max {
|
||||
name.to_string()
|
||||
} else {
|
||||
@@ -241,7 +241,7 @@ where
|
||||
Box::new(
|
||||
ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
&suffix("cumulative"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{CheckedSub, Result, StoredUsize, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, EagerVec, Format, StoredIndex, StoredType};
|
||||
use brk_vecs::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, EagerVec, File, Format, StoredIndex, StoredType,
|
||||
};
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
|
||||
use crate::utils::get_percentile;
|
||||
@@ -37,7 +39,7 @@ where
|
||||
T: ComputedType,
|
||||
{
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
version: Version,
|
||||
format: Format,
|
||||
@@ -59,7 +61,7 @@ where
|
||||
first: options.first.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("first"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -69,13 +71,13 @@ where
|
||||
}),
|
||||
last: options.last.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(path, name, version + Version::ZERO, format).unwrap(),
|
||||
EagerVec::forced_import(file, name, version + Version::ZERO, format).unwrap(),
|
||||
)
|
||||
}),
|
||||
min: options.min.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("min"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -86,7 +88,7 @@ where
|
||||
max: options.max.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("max"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -97,7 +99,7 @@ where
|
||||
median: options.median.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("median"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -108,7 +110,7 @@ where
|
||||
average: options.average.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("average"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -119,7 +121,7 @@ where
|
||||
sum: options.sum.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&(if !options.last && !options.average && !options.min && !options.max {
|
||||
name.to_string()
|
||||
} else {
|
||||
@@ -134,7 +136,7 @@ where
|
||||
cumulative: options.cumulative.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("cumulative"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -145,7 +147,7 @@ where
|
||||
_90p: options._90p.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("90p"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -156,7 +158,7 @@ where
|
||||
_75p: options._75p.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("75p"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -167,7 +169,7 @@ where
|
||||
_25p: options._25p.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("25p"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -178,7 +180,7 @@ where
|
||||
_10p: options._10p.then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&maybe_suffix("10p"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -309,7 +311,7 @@ where
|
||||
.inspect_err(|_| {
|
||||
dbg!(
|
||||
&values,
|
||||
max.path(),
|
||||
max.name(),
|
||||
first_indexes.name(),
|
||||
first_index,
|
||||
count_indexes.name(),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{
|
||||
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, Result, SemesterIndex, Version, WeekIndex,
|
||||
@@ -6,8 +6,8 @@ use brk_core::{
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format,
|
||||
use brk_vecs::{
|
||||
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, File, Format,
|
||||
};
|
||||
|
||||
use crate::{Indexes, grouped::ComputedVecBuilder, indexes};
|
||||
@@ -37,7 +37,7 @@ where
|
||||
{
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
source: Source<DateIndex, T>,
|
||||
version: Version,
|
||||
@@ -47,11 +47,11 @@ where
|
||||
options: VecBuilderOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let dateindex = source.is_compute().then(|| {
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
|
||||
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap()
|
||||
});
|
||||
|
||||
let dateindex_extra = EagerVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -64,7 +64,7 @@ where
|
||||
|
||||
Ok(Self {
|
||||
weekindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -75,7 +75,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
monthindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -86,7 +86,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
quarterindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -97,7 +97,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
semesterindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -108,7 +108,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
yearindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -119,7 +119,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
decadeindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{
|
||||
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, Result,
|
||||
@@ -6,8 +6,8 @@ use brk_core::{
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, Format,
|
||||
use brk_vecs::{
|
||||
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, EagerVec, File, Format,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
@@ -45,7 +45,7 @@ where
|
||||
{
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
source: Source<Height, T>,
|
||||
version: Version,
|
||||
@@ -55,11 +55,11 @@ where
|
||||
options: VecBuilderOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let height = source.is_compute().then(|| {
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format).unwrap()
|
||||
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap()
|
||||
});
|
||||
|
||||
let height_extra = EagerVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -67,7 +67,7 @@ where
|
||||
)?;
|
||||
|
||||
let dateindex = EagerVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -78,7 +78,7 @@ where
|
||||
|
||||
Ok(Self {
|
||||
weekindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -89,7 +89,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
monthindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -100,7 +100,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
quarterindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -111,7 +111,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
semesterindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -122,7 +122,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
yearindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -133,7 +133,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
decadeindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -143,12 +143,12 @@ where
|
||||
indexes.decadeindex_to_decadeindex.boxed_clone(),
|
||||
options.into(),
|
||||
)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
|
||||
height,
|
||||
height_extra,
|
||||
dateindex,
|
||||
difficultyepoch: EagerVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{DifficultyEpoch, Height, Result, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, EagerVec, Format};
|
||||
use brk_vecs::{AnyCollectableVec, EagerVec, File, Format};
|
||||
|
||||
use crate::{Indexes, indexes};
|
||||
|
||||
@@ -28,17 +28,17 @@ where
|
||||
f64: From<T>,
|
||||
{
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
version: Version,
|
||||
format: Format,
|
||||
options: VecBuilderOptions,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let height =
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)?;
|
||||
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)?;
|
||||
|
||||
let height_extra = EagerVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -51,13 +51,13 @@ where
|
||||
height,
|
||||
height_extra,
|
||||
difficultyepoch: EagerVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{
|
||||
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
|
||||
@@ -6,9 +6,9 @@ use brk_core::{
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
use brk_vecs::{
|
||||
AnyCollectableVec, AnyVec, CloneableAnyIterableVec, CollectableVec, Computation, EagerVec,
|
||||
Format, StoredIndex, VecIterator,
|
||||
File, Format, StoredIndex, VecIterator,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
@@ -46,7 +46,7 @@ where
|
||||
{
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
source: Source<TxIndex, T>,
|
||||
version: Version,
|
||||
@@ -57,13 +57,13 @@ where
|
||||
) -> color_eyre::Result<Self> {
|
||||
let txindex = source.is_compute().then(|| {
|
||||
Box::new(
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
|
||||
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let height = EagerVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -73,7 +73,7 @@ where
|
||||
let options = options.remove_percentiles();
|
||||
|
||||
let dateindex = EagerVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -82,7 +82,7 @@ where
|
||||
|
||||
Ok(Self {
|
||||
weekindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -93,7 +93,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
monthindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -104,7 +104,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
quarterindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -115,7 +115,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
semesterindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -126,7 +126,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
yearindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -137,7 +137,7 @@ where
|
||||
options.into(),
|
||||
)?,
|
||||
decadeindex: ComputedVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -152,13 +152,13 @@ where
|
||||
height,
|
||||
dateindex,
|
||||
difficultyepoch: EagerVecBuilder::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
options,
|
||||
)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(path, name, version + VERSION + Version::ZERO, format, options)?,
|
||||
// halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use std::{f32, path::Path};
|
||||
use std::{f32, sync::Arc};
|
||||
|
||||
use brk_core::{Date, DateIndex, Dollars, Result, StoredF32, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Computation, EagerVec, Format,
|
||||
use brk_vecs::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, CollectableVec, Computation, EagerVec, File, Format,
|
||||
StoredIndex, VecIterator,
|
||||
};
|
||||
|
||||
@@ -60,7 +60,7 @@ const VERSION: Version = Version::ZERO;
|
||||
impl ComputedRatioVecsFromDateIndex {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
source: Source<DateIndex, Dollars>,
|
||||
version: Version,
|
||||
@@ -74,7 +74,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
Ok(Self {
|
||||
price: source.is_compute().then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
Source::Compute,
|
||||
version + VERSION,
|
||||
@@ -86,7 +86,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
.unwrap()
|
||||
}),
|
||||
ratio: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -97,7 +97,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
)?,
|
||||
ratio_sma: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_sma"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -110,7 +110,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_1w_sma: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_1w_sma"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -123,7 +123,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_1m_sma: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_1m_sma"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -136,7 +136,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_1y_sma: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_1y_sma"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -149,7 +149,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_4y_sma: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_4y_sma"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -162,7 +162,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_1y_sma_momentum_oscillator: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_1y_sma_momentum_oscillator"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -175,7 +175,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_sd: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_sd"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -188,7 +188,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_4y_sd: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_4y_sd"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -201,7 +201,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_1y_sd: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_1y_sd"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -214,7 +214,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p99_9: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p99_9"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -227,7 +227,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p99_5: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p99_5"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -240,7 +240,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p99: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p99"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -253,7 +253,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p1: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p1"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -266,7 +266,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p0_5: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p0_5"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -279,7 +279,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p0_1: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p0_1"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -292,7 +292,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p1sd: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p1sd"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -305,7 +305,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p2sd: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p2sd"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -318,7 +318,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p3sd: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p3sd"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -331,7 +331,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_m1sd: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_m1sd"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -344,7 +344,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_m2sd: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_m2sd"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -357,7 +357,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_m3sd: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_m3sd"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -370,7 +370,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p99_9_as_price: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p99_9_as_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -383,7 +383,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p99_5_as_price: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p99_5_as_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -396,7 +396,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p99_as_price: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p99_as_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -409,7 +409,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p1_as_price: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p1_as_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -422,7 +422,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p0_5_as_price: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p0_5_as_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -435,7 +435,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p0_1_as_price: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p0_1_as_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -448,7 +448,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p1sd_as_price: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p1sd_as_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -461,7 +461,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p2sd_as_price: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p2sd_as_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -474,7 +474,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_p3sd_as_price: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_p3sd_as_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -487,7 +487,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_m1sd_as_price: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_m1sd_as_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -500,7 +500,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_m2sd_as_price: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_m2sd_as_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -513,7 +513,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_m3sd_as_price: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_m3sd_as_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -526,7 +526,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_zscore: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_zscore"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -539,7 +539,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_4y_zscore: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_4y_zscore"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -552,7 +552,7 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
}),
|
||||
ratio_1y_zscore: extended.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_ratio_1y_zscore"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -1069,6 +1069,10 @@ impl ComputedRatioVecsFromDateIndex {
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
drop(sma_iter);
|
||||
drop(_4y_sma_iter);
|
||||
drop(_1y_sma_iter);
|
||||
|
||||
self.mut_ratio_vecs()
|
||||
.into_iter()
|
||||
.try_for_each(|v| v.safe_flush(exit))?;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use brk_vec::BoxedAnyIterableVec;
|
||||
use brk_vecs::BoxedAnyIterableVec;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum Source<I, T> {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::ops::{Add, AddAssign, Div};
|
||||
|
||||
use brk_vec::StoredType;
|
||||
use brk_vecs::StoredType;
|
||||
|
||||
pub trait ComputedType
|
||||
where
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{Bitcoin, DateIndex, Dollars, Result, Sats, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec};
|
||||
use brk_vecs::{AnyCollectableVec, CollectableVec, Computation, EagerVec, File, Format, StoredVec};
|
||||
|
||||
use crate::{Indexes, fetched, grouped::ComputedVecsFromDateIndex, indexes};
|
||||
|
||||
@@ -21,7 +21,7 @@ const VERSION: Version = Version::ZERO;
|
||||
impl ComputedValueVecsFromDateIndex {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
source: Source<DateIndex, Sats>,
|
||||
version: Version,
|
||||
@@ -33,7 +33,7 @@ impl ComputedValueVecsFromDateIndex {
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
sats: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
source,
|
||||
version + VERSION,
|
||||
@@ -43,7 +43,7 @@ impl ComputedValueVecsFromDateIndex {
|
||||
options,
|
||||
)?,
|
||||
bitcoin: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_in_btc"),
|
||||
Source::Compute,
|
||||
version + VERSION,
|
||||
@@ -54,7 +54,7 @@ impl ComputedValueVecsFromDateIndex {
|
||||
)?,
|
||||
dollars: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_in_usd"),
|
||||
Source::Compute,
|
||||
version + VERSION,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, CollectableVec, Computation, EagerVec, Format, StoredVec};
|
||||
use brk_vecs::{AnyCollectableVec, CollectableVec, Computation, EagerVec, File, Format, StoredVec};
|
||||
|
||||
use crate::{Indexes, fetched, grouped::Source, indexes};
|
||||
|
||||
@@ -21,7 +21,7 @@ const VERSION: Version = Version::ZERO;
|
||||
impl ComputedValueVecsFromHeight {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
source: Source<Height, Sats>,
|
||||
version: Version,
|
||||
@@ -33,7 +33,7 @@ impl ComputedValueVecsFromHeight {
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
sats: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
source,
|
||||
version + VERSION,
|
||||
@@ -43,7 +43,7 @@ impl ComputedValueVecsFromHeight {
|
||||
options,
|
||||
)?,
|
||||
bitcoin: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_in_btc"),
|
||||
Source::Compute,
|
||||
version + VERSION,
|
||||
@@ -54,7 +54,7 @@ impl ComputedValueVecsFromHeight {
|
||||
)?,
|
||||
dollars: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_in_usd"),
|
||||
Source::Compute,
|
||||
version + VERSION,
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
use brk_vecs::{
|
||||
AnyCollectableVec, CloneableAnyIterableVec, CollectableVec, Computation, ComputedVecFrom3,
|
||||
Format, LazyVecFrom1, StoredIndex, StoredVec,
|
||||
File, Format, LazyVecFrom1, StoredIndex, StoredVec,
|
||||
};
|
||||
|
||||
use crate::{Indexes, fetched, grouped::Source, indexes};
|
||||
@@ -38,7 +38,7 @@ const VERSION: Version = Version::ZERO;
|
||||
impl ComputedValueVecsFromTxindex {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
indexes: &indexes::Vecs,
|
||||
source: Source<TxIndex, Sats>,
|
||||
@@ -54,7 +54,7 @@ impl ComputedValueVecsFromTxindex {
|
||||
let name_in_usd = format!("{name}_in_usd");
|
||||
|
||||
let sats = ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
file,
|
||||
name,
|
||||
source.clone(),
|
||||
version + VERSION,
|
||||
@@ -79,7 +79,7 @@ impl ComputedValueVecsFromTxindex {
|
||||
);
|
||||
|
||||
let bitcoin = ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&name_in_btc,
|
||||
Source::None,
|
||||
version + VERSION,
|
||||
@@ -92,7 +92,7 @@ impl ComputedValueVecsFromTxindex {
|
||||
let dollars_txindex = fetched.map(|fetched| {
|
||||
ComputedVecFrom3::forced_import_or_init_from_3(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
&name_in_usd,
|
||||
version + VERSION,
|
||||
format,
|
||||
@@ -127,7 +127,7 @@ impl ComputedValueVecsFromTxindex {
|
||||
dollars_txindex,
|
||||
dollars: compute_dollars.then(|| {
|
||||
ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&name_in_usd,
|
||||
Source::None,
|
||||
version + VERSION,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{Bitcoin, Dollars, Height, Result, Sats, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, CollectableVec, EagerVec, Format, StoredVec};
|
||||
use brk_vecs::{AnyCollectableVec, CollectableVec, EagerVec, File, Format, StoredVec};
|
||||
|
||||
use crate::{Indexes, fetched, grouped::Source, indexes};
|
||||
|
||||
@@ -18,7 +18,7 @@ const VERSION: Version = Version::ZERO;
|
||||
|
||||
impl ComputedHeightValueVecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
source: Source<Height, Sats>,
|
||||
version: Version,
|
||||
@@ -27,18 +27,18 @@ impl ComputedHeightValueVecs {
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
sats: source.is_compute().then(|| {
|
||||
EagerVec::forced_import(path, name, version + VERSION + Version::ZERO, format)
|
||||
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)
|
||||
.unwrap()
|
||||
}),
|
||||
bitcoin: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_in_btc"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dollars: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&format!("{name}_in_usd"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::{ops::Deref, path::Path};
|
||||
use std::{ops::Deref, sync::Arc};
|
||||
|
||||
use brk_core::{
|
||||
Date, DateIndex, DecadeIndex, DifficultyEpoch, EmptyOutputIndex, HalvingEpoch, Height,
|
||||
@@ -10,9 +10,9 @@ use brk_core::{
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
use brk_vecs::{
|
||||
AnyCollectableVec, CloneableAnyIterableVec, Computation, ComputedVec, ComputedVecFrom1,
|
||||
ComputedVecFrom2, EagerVec, Format, StoredIndex, VecIterator,
|
||||
ComputedVecFrom2, EagerVec, File, Format, StoredIndex, VecIterator,
|
||||
};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
@@ -97,7 +97,7 @@ pub struct Vecs {
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
version: Version,
|
||||
indexer: &Indexer,
|
||||
computation: Computation,
|
||||
@@ -105,7 +105,7 @@ impl Vecs {
|
||||
) -> color_eyre::Result<Self> {
|
||||
let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"outputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -115,7 +115,7 @@ impl Vecs {
|
||||
|
||||
let inputindex_to_inputindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"inputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -125,7 +125,7 @@ impl Vecs {
|
||||
|
||||
let txindex_to_txindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"txindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -135,7 +135,7 @@ impl Vecs {
|
||||
|
||||
let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"input_count",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -158,7 +158,7 @@ impl Vecs {
|
||||
|
||||
let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"output_count",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -181,7 +181,7 @@ impl Vecs {
|
||||
|
||||
let p2pk33addressindex_to_p2pk33addressindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"p2pk33addressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -190,7 +190,7 @@ impl Vecs {
|
||||
)?;
|
||||
let p2pk65addressindex_to_p2pk65addressindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"p2pk65addressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -199,7 +199,7 @@ impl Vecs {
|
||||
)?;
|
||||
let p2pkhaddressindex_to_p2pkhaddressindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"p2pkhaddressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -208,7 +208,7 @@ impl Vecs {
|
||||
)?;
|
||||
let p2shaddressindex_to_p2shaddressindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"p2shaddressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -217,7 +217,7 @@ impl Vecs {
|
||||
)?;
|
||||
let p2traddressindex_to_p2traddressindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"p2traddressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -226,7 +226,7 @@ impl Vecs {
|
||||
)?;
|
||||
let p2wpkhaddressindex_to_p2wpkhaddressindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"p2wpkhaddressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -235,7 +235,7 @@ impl Vecs {
|
||||
)?;
|
||||
let p2wshaddressindex_to_p2wshaddressindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"p2wshaddressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -244,7 +244,7 @@ impl Vecs {
|
||||
)?;
|
||||
let p2aaddressindex_to_p2aaddressindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"p2aaddressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -253,7 +253,7 @@ impl Vecs {
|
||||
)?;
|
||||
let p2msoutputindex_to_p2msoutputindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"p2msoutputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -262,7 +262,7 @@ impl Vecs {
|
||||
)?;
|
||||
let emptyoutputindex_to_emptyoutputindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"emptyoutputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -271,7 +271,7 @@ impl Vecs {
|
||||
)?;
|
||||
let unknownoutputindex_to_unknownoutputindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"unknownoutputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -280,7 +280,7 @@ impl Vecs {
|
||||
)?;
|
||||
let opreturnindex_to_opreturnindex = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"opreturnindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -308,259 +308,259 @@ impl Vecs {
|
||||
unknownoutputindex_to_unknownoutputindex,
|
||||
|
||||
dateindex_to_date: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"date",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_dateindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"dateindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_first_height: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"first_height",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_monthindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"monthindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_weekindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"weekindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
decadeindex_to_decadeindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"decadeindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
decadeindex_to_first_yearindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"first_yearindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
difficultyepoch_to_difficultyepoch: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"difficultyepoch",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
difficultyepoch_to_first_height: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"first_height",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
halvingepoch_to_first_height: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"first_height",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
halvingepoch_to_halvingepoch: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"halvingepoch",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_date: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"date",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_difficultyepoch: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"difficultyepoch",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_halvingepoch: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"halvingepoch",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_height: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"height",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
monthindex_to_first_dateindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"first_dateindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
monthindex_to_monthindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"monthindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
monthindex_to_quarterindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"quarterindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
monthindex_to_semesterindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"semesterindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
monthindex_to_yearindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"yearindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
quarterindex_to_first_monthindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"first_monthindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
semesterindex_to_first_monthindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"first_monthindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
weekindex_to_first_dateindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"first_dateindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
yearindex_to_first_monthindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"first_monthindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
quarterindex_to_quarterindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"quarterindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
semesterindex_to_semesterindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"semesterindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
weekindex_to_weekindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"weekindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
yearindex_to_decadeindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"decadeindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
yearindex_to_yearindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"yearindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_date_fixed: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"date_fixed",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_dateindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"dateindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
txindex_to_height: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"height",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_timestamp_fixed: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"timestamp_fixed",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_txindex_count: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"txindex_count",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
dateindex_to_height_count: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"height_count",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
weekindex_to_dateindex_count: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"dateindex_count",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
difficultyepoch_to_height_count: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"height_count",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
monthindex_to_dateindex_count: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"dateindex_count",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
quarterindex_to_monthindex_count: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"monthindex_count",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
semesterindex_to_monthindex_count: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"monthindex_count",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
yearindex_to_monthindex_count: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"monthindex_count",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
decadeindex_to_yearindex_count: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"yearindex_count",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
outputindex_to_txindex: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
"txindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
|
||||
@@ -3,13 +3,13 @@
|
||||
#![doc = include_str!("../examples/main.rs")]
|
||||
#![doc = "```"]
|
||||
|
||||
use std::path::Path;
|
||||
use std::{path::Path, sync::Arc};
|
||||
|
||||
use brk_core::Version;
|
||||
use brk_exit::Exit;
|
||||
use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{Computation, Format};
|
||||
use brk_vecs::{Computation, File, Format, PAGE_SIZE};
|
||||
use log::info;
|
||||
|
||||
mod all;
|
||||
@@ -32,6 +32,7 @@ use states::*;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Computer {
|
||||
file: Arc<File>,
|
||||
fetcher: Option<Fetcher>,
|
||||
pub vecs: all::Vecs,
|
||||
}
|
||||
@@ -47,17 +48,28 @@ impl Computer {
|
||||
fetcher: Option<Fetcher>,
|
||||
format: Format,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let computed_path = outputs_dir.join("computed");
|
||||
let states_path = computed_path.join("states");
|
||||
|
||||
let file = Arc::new(File::open(&computed_path.join("vecs"))?);
|
||||
file.set_min_len(PAGE_SIZE * 100_000_000)?;
|
||||
file.set_min_regions(50_000)?;
|
||||
|
||||
let file_fetched = Arc::new(File::open(&outputs_dir.join("fetched/vecs"))?);
|
||||
|
||||
Ok(Self {
|
||||
vecs: all::Vecs::import(
|
||||
// TODO: Give self.path, join inside import
|
||||
&outputs_dir.join("vecs/computed"),
|
||||
&file,
|
||||
VERSION + Version::ZERO,
|
||||
indexer,
|
||||
fetcher.is_some(),
|
||||
computation,
|
||||
format,
|
||||
&file_fetched,
|
||||
&states_path,
|
||||
)?,
|
||||
fetcher,
|
||||
file,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -71,6 +83,9 @@ impl Computer {
|
||||
) -> color_eyre::Result<()> {
|
||||
info!("Computing...");
|
||||
self.vecs
|
||||
.compute(indexer, starting_indexes, self.fetcher.as_mut(), exit)
|
||||
.compute(indexer, starting_indexes, self.fetcher.as_mut(), exit)?;
|
||||
self.file.flush()?;
|
||||
self.file.punch_holes()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
+137
-137
File diff suppressed because it is too large
Load Diff
@@ -1,9 +1,9 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, Computation, Format, VecIterator};
|
||||
use brk_vecs::{AnyCollectableVec, Computation, File, Format, VecIterator};
|
||||
|
||||
use crate::grouped::Source;
|
||||
|
||||
@@ -24,7 +24,7 @@ pub struct Vecs {
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
version: Version,
|
||||
computation: Computation,
|
||||
format: Format,
|
||||
@@ -32,7 +32,7 @@ impl Vecs {
|
||||
) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
indexes_to_difficulty: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"difficulty",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -42,7 +42,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"difficultyepoch",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -52,7 +52,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"halvingepoch",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use std::{ops::Deref, path::Path};
|
||||
use std::{ops::Deref, path::Path, sync::Arc};
|
||||
|
||||
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, StoredUsize, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, Format, VecIterator,
|
||||
use brk_vecs::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, Computation, EagerVec, File, Format, VecIterator,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
@@ -35,7 +35,7 @@ pub struct Vecs {
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
cohort_name: Option<&str>,
|
||||
computation: Computation,
|
||||
format: Format,
|
||||
@@ -57,13 +57,13 @@ impl Vecs {
|
||||
compute_dollars,
|
||||
)?,
|
||||
height_to_address_count: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("address_count"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
indexes_to_address_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("address_count"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -73,7 +73,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_last(),
|
||||
)?,
|
||||
inner: common::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
cohort_name,
|
||||
computation,
|
||||
format,
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
use std::path::Path;
|
||||
use std::{path::Path, sync::Arc};
|
||||
|
||||
use brk_core::{
|
||||
AddressGroups, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, GroupFilter, Height,
|
||||
Result, Version,
|
||||
AddressGroups, Bitcoin, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, DateIndex,
|
||||
Dollars, GroupFilter, Height, Result, Version,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_vec::{Computation, Format};
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vecs::{AnyIterableVec, Computation, File, Format};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::{
|
||||
Indexes, fetched, indexes,
|
||||
Indexes, fetched, indexes, market,
|
||||
stateful::{
|
||||
address_cohort,
|
||||
r#trait::{CohortVecs, DynCohortVecs},
|
||||
@@ -24,7 +24,7 @@ pub struct Vecs(AddressGroups<(GroupFilter, address_cohort::Vecs)>);
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
version: Version,
|
||||
_computation: Computation,
|
||||
format: Format,
|
||||
@@ -36,7 +36,7 @@ impl Vecs {
|
||||
AddressGroups {
|
||||
amount_range: ByAmountRange {
|
||||
_0sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_with_0sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -47,7 +47,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1sat_to_10sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_1sat_under_10sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -58,7 +58,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10sats_to_100sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_10sats_under_100sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -69,7 +69,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_100sats_to_1k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_100sats_under_1k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -80,7 +80,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1k_sats_to_10k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_1k_sats_under_10k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -91,7 +91,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10k_sats_to_100k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_10k_sats_under_100k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -102,7 +102,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_100k_sats_to_1m_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_100k_sats_under_1m_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -113,7 +113,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1m_sats_to_10m_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_1m_sats_under_10m_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -124,7 +124,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10m_sats_to_1btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_10m_sats_under_1btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -135,7 +135,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1btc_to_10btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_1btc_under_10btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -146,7 +146,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10btc_to_100btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_10btc_under_100btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -157,7 +157,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_100btc_to_1k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_100btc_under_1k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -168,7 +168,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1k_btc_to_10k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_1k_btc_under_10k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -179,7 +179,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10k_btc_to_100k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_10k_btc_under_100k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -190,7 +190,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_100k_btc_or_more: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_100k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -203,7 +203,7 @@ impl Vecs {
|
||||
},
|
||||
lt_amount: ByLowerThanAmount {
|
||||
_10sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_10sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -214,7 +214,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_100sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_100sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -225,7 +225,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_1k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -236,7 +236,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_10k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -247,7 +247,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_100k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_100k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -258,7 +258,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1m_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_1m_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -269,7 +269,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10m_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_10m_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -280,7 +280,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_1btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -291,7 +291,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_10btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -302,7 +302,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_100btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_100btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -313,7 +313,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_1k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -324,7 +324,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_10k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -335,7 +335,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_100k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_under_100k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -348,7 +348,7 @@ impl Vecs {
|
||||
},
|
||||
ge_amount: ByGreatEqualAmount {
|
||||
_1sat: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_1sat"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -359,7 +359,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_10sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -370,7 +370,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_100sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_100sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -381,7 +381,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_1k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -392,7 +392,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_10k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -403,7 +403,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_100k_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_100k_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -414,7 +414,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1m_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_1m_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -425,7 +425,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10m_sats: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_10m_sats"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -436,7 +436,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_1btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -447,7 +447,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_10btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -458,7 +458,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_100btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_100btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -469,7 +469,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_1k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_1k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -480,7 +480,7 @@ impl Vecs {
|
||||
true,
|
||||
)?,
|
||||
_10k_btc: address_cohort::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
Some("addrs_above_10k_btc"),
|
||||
_computation,
|
||||
format,
|
||||
@@ -535,16 +535,61 @@ impl Vecs {
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
]
|
||||
.into_par_iter()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.try_for_each(|(vecs, stateful)| {
|
||||
vecs.compute_from_stateful(starting_indexes, &stateful, exit)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_rest_part1(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
self.as_mut_vecs().into_iter().try_for_each(|(_, v)| {
|
||||
v.compute_rest_part1(indexer, indexes, fetched, starting_indexes, exit)
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute_rest_part2(
|
||||
&mut self,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
fetched: Option<&fetched::Vecs>,
|
||||
starting_indexes: &Indexes,
|
||||
market: &market::Vecs,
|
||||
height_to_supply: &impl AnyIterableVec<Height, Bitcoin>,
|
||||
dateindex_to_supply: &impl AnyIterableVec<DateIndex, Bitcoin>,
|
||||
height_to_realized_cap: Option<&impl AnyIterableVec<Height, Dollars>>,
|
||||
dateindex_to_realized_cap: Option<&impl AnyIterableVec<DateIndex, Dollars>>,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
self.0.as_boxed_mut_vecs().into_iter().try_for_each(|v| {
|
||||
v.into_iter().try_for_each(|(_, v)| {
|
||||
v.compute_rest_part2(
|
||||
indexer,
|
||||
indexes,
|
||||
fetched,
|
||||
starting_indexes,
|
||||
market,
|
||||
height_to_supply,
|
||||
dateindex_to_supply,
|
||||
height_to_realized_cap,
|
||||
dateindex_to_realized_cap,
|
||||
exit,
|
||||
)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
|
||||
self.as_mut_separate_vecs()
|
||||
.par_iter_mut()
|
||||
.into_iter()
|
||||
.try_for_each(|(_, v)| v.safe_flush_stateful_vecs(height, exit))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use brk_core::{ByAddressType, Height};
|
||||
use brk_vec::VecIterator;
|
||||
use brk_vecs::VecIterator;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use brk_core::{ByAddressType, Height, Result, StoredUsize};
|
||||
use brk_exit::Exit;
|
||||
use brk_vec::EagerVec;
|
||||
use brk_vecs::EagerVec;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use brk_core::{ByAddressType, StoredUsize};
|
||||
use brk_exit::Exit;
|
||||
use brk_vec::AnyCollectableVec;
|
||||
use brk_vecs::AnyCollectableVec;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::{
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{
|
||||
Bitcoin, DateIndex, Dollars, Height, Result, Sats, StoredF32, StoredF64, StoredUsize, Version,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
use brk_vecs::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, CloneableAnyIterableVec, Computation, EagerVec,
|
||||
Format, VecIterator,
|
||||
File, Format, VecIterator,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
@@ -128,7 +128,7 @@ pub struct Vecs {
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
cohort_name: Option<&str>,
|
||||
computation: Computation,
|
||||
format: Format,
|
||||
@@ -146,7 +146,7 @@ impl Vecs {
|
||||
|
||||
let dateindex_to_supply_in_profit = compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_profit"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -156,7 +156,7 @@ impl Vecs {
|
||||
|
||||
let dateindex_to_supply_even = compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_even"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -166,7 +166,7 @@ impl Vecs {
|
||||
|
||||
let dateindex_to_supply_in_loss = compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_loss"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -176,7 +176,7 @@ impl Vecs {
|
||||
|
||||
let dateindex_to_unrealized_profit = compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("unrealized_profit"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -186,7 +186,7 @@ impl Vecs {
|
||||
|
||||
let dateindex_to_unrealized_loss = compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("unrealized_loss"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -197,7 +197,7 @@ impl Vecs {
|
||||
Ok(Self {
|
||||
height_to_supply_in_profit: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_profit"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -206,7 +206,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_supply_in_profit: compute_dollars.then(|| {
|
||||
ComputedValueVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_profit"),
|
||||
dateindex_to_supply_in_profit.as_ref().map(|v | v.boxed_clone()).into(),
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -221,7 +221,7 @@ impl Vecs {
|
||||
dateindex_to_supply_in_profit,
|
||||
height_to_supply_even: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_even"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -230,7 +230,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_supply_even: compute_dollars.then(|| {
|
||||
ComputedValueVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_even"),
|
||||
dateindex_to_supply_even.as_ref().map(|v | v.boxed_clone()).into(),
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -245,7 +245,7 @@ impl Vecs {
|
||||
dateindex_to_supply_even,
|
||||
height_to_supply_in_loss: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_loss"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -254,7 +254,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_supply_in_loss: compute_dollars.then(|| {
|
||||
ComputedValueVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_loss"),
|
||||
dateindex_to_supply_in_loss.as_ref().map(|v | v.boxed_clone()).into(),
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -269,7 +269,7 @@ impl Vecs {
|
||||
dateindex_to_supply_in_loss,
|
||||
height_to_unrealized_profit: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("unrealized_profit"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -278,7 +278,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_unrealized_profit: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("unrealized_profit"),
|
||||
dateindex_to_unrealized_profit.as_ref().map(|v | v.boxed_clone()).into(),
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -292,7 +292,7 @@ impl Vecs {
|
||||
dateindex_to_unrealized_profit,
|
||||
height_to_unrealized_loss: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("unrealized_loss"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -301,7 +301,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_min_price_paid: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("min_price_paid"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -310,7 +310,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_max_price_paid: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("max_price_paid"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -319,7 +319,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_unrealized_loss: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("unrealized_loss"),
|
||||
dateindex_to_unrealized_loss.as_ref().map(|v | v.boxed_clone()).into(),
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -333,7 +333,7 @@ impl Vecs {
|
||||
dateindex_to_unrealized_loss,
|
||||
height_to_realized_cap: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("realized_cap"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -342,7 +342,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_realized_cap: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("realized_cap"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -355,7 +355,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_min_price_paid: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("min_price_paid"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -368,7 +368,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_max_price_paid: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("max_price_paid"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -380,13 +380,13 @@ impl Vecs {
|
||||
.unwrap()
|
||||
}),
|
||||
height_to_supply: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_supply_value: ComputedHeightValueVecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -394,7 +394,7 @@ impl Vecs {
|
||||
compute_dollars,
|
||||
)?,
|
||||
indexes_to_supply: ComputedValueVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -405,13 +405,13 @@ impl Vecs {
|
||||
indexes,
|
||||
)?,
|
||||
height_to_utxo_count: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("utxo_count"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
indexes_to_utxo_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("utxo_count"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -422,7 +422,7 @@ impl Vecs {
|
||||
)?,
|
||||
indexes_to_realized_price: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("realized_price"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -435,7 +435,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_realized_price_extra: compute_dollars.then(|| {
|
||||
ComputedRatioVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("realized_price"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -448,7 +448,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_realized_profit: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("realized_profit"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -457,7 +457,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_realized_profit: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("realized_profit"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -472,7 +472,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_realized_loss: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("realized_loss"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -481,7 +481,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_realized_loss: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("realized_loss"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -496,7 +496,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_negative_realized_loss: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("negative_realized_loss"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -509,7 +509,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_value_created: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("value_created"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -518,7 +518,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_value_created: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("value_created"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -531,7 +531,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_realized_value: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("realized_value"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -544,7 +544,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_adjusted_value_created: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("adjusted_value_created"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -553,7 +553,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_adjusted_value_created: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("adjusted_value_created"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -566,7 +566,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_value_destroyed: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("value_destroyed"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -575,7 +575,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_value_destroyed: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("value_destroyed"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -588,7 +588,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_adjusted_value_destroyed: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("adjusted_value_destroyed"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -597,7 +597,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_adjusted_value_destroyed: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("adjusted_value_destroyed"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -610,7 +610,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_realized_cap_30d_change: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("realized_cap_30d_change"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -623,7 +623,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_net_realized_profit_and_loss: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("net_realized_profit_and_loss"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -638,7 +638,7 @@ impl Vecs {
|
||||
}),
|
||||
dateindex_to_sell_side_risk_ratio: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("sell_side_risk_ratio"),
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
@@ -647,7 +647,7 @@ impl Vecs {
|
||||
}),
|
||||
dateindex_to_spent_output_profit_ratio: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("spent_output_profit_ratio"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -656,7 +656,7 @@ impl Vecs {
|
||||
}),
|
||||
dateindex_to_adjusted_spent_output_profit_ratio: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("adjusted_spent_output_profit_ratio"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -664,7 +664,7 @@ impl Vecs {
|
||||
.unwrap()
|
||||
}),
|
||||
height_to_halved_supply_value: ComputedHeightValueVecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("halved_supply"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -672,7 +672,7 @@ impl Vecs {
|
||||
compute_dollars,
|
||||
)?,
|
||||
indexes_to_halved_supply: ComputedValueVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("halved_supply"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -684,7 +684,7 @@ impl Vecs {
|
||||
)?,
|
||||
height_to_negative_unrealized_loss: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("negative_unrealized_loss"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -693,7 +693,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_negative_unrealized_loss: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("negative_unrealized_loss"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -706,7 +706,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_net_unrealized_profit_and_loss: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("net_unrealized_profit_and_loss"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -715,7 +715,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_net_unrealized_profit_and_loss: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("net_unrealized_profit_and_loss"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -729,7 +729,7 @@ impl Vecs {
|
||||
height_to_net_unrealized_profit_and_loss_relative_to_market_cap: compute_dollars.then(
|
||||
|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("net_unrealized_profit_and_loss_relative_to_market_cap"),
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
@@ -740,7 +740,7 @@ impl Vecs {
|
||||
indexes_to_net_unrealized_profit_and_loss_relative_to_market_cap: compute_dollars.then(
|
||||
|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("net_unrealized_profit_and_loss_relative_to_market_cap"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -754,7 +754,7 @@ impl Vecs {
|
||||
),
|
||||
indexes_to_realized_profit_relative_to_realized_cap: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("realized_profit_relative_to_realized_cap"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -767,7 +767,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_realized_loss_relative_to_realized_cap: compute_dollars.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("realized_loss_relative_to_realized_cap"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -781,7 +781,7 @@ impl Vecs {
|
||||
indexes_to_net_realized_profit_and_loss_relative_to_realized_cap: compute_dollars.then(
|
||||
|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("net_realized_profit_and_loss_relative_to_realized_cap"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -795,7 +795,7 @@ impl Vecs {
|
||||
),
|
||||
height_to_supply_even_value: compute_dollars.then(|| {
|
||||
ComputedHeightValueVecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_even"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -806,7 +806,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_supply_in_loss_value: compute_dollars.then(|| {
|
||||
ComputedHeightValueVecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_loss"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -817,7 +817,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_supply_in_profit_value: compute_dollars.then(|| {
|
||||
ComputedHeightValueVecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_profit"),
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -828,7 +828,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_supply_even_relative_to_own_supply: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_even_relative_to_own_supply"),
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
@@ -837,7 +837,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_supply_in_loss_relative_to_own_supply: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_loss_relative_to_own_supply"),
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
@@ -846,7 +846,7 @@ impl Vecs {
|
||||
}),
|
||||
height_to_supply_in_profit_relative_to_own_supply: compute_dollars.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_profit_relative_to_own_supply"),
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
@@ -855,7 +855,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_supply_even_relative_to_own_supply: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_even_relative_to_own_supply"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -868,7 +868,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_supply_in_loss_relative_to_own_supply: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_loss_relative_to_own_supply"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -881,7 +881,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_supply_in_profit_relative_to_own_supply: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_profit_relative_to_own_supply"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -894,7 +894,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_supply_relative_to_circulating_supply: compute_relative_to_all.then(|| {
|
||||
ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_relative_to_circulating_supply"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -909,7 +909,7 @@ impl Vecs {
|
||||
&& compute_dollars)
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_even_relative_to_circulating_supply"),
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
@@ -920,7 +920,7 @@ impl Vecs {
|
||||
&& compute_dollars)
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_loss_relative_to_circulating_supply"),
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
@@ -931,7 +931,7 @@ impl Vecs {
|
||||
&& compute_dollars)
|
||||
.then(|| {
|
||||
EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_profit_relative_to_circulating_supply"),
|
||||
version + VERSION + Version::ONE,
|
||||
format,
|
||||
@@ -942,7 +942,7 @@ impl Vecs {
|
||||
&& compute_dollars)
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_even_relative_to_circulating_supply"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -957,7 +957,7 @@ impl Vecs {
|
||||
&& compute_dollars)
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_loss_relative_to_circulating_supply"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -972,7 +972,7 @@ impl Vecs {
|
||||
&& compute_dollars)
|
||||
.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("supply_in_profit_relative_to_circulating_supply"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ONE,
|
||||
@@ -984,19 +984,19 @@ impl Vecs {
|
||||
.unwrap()
|
||||
}),
|
||||
height_to_satblocks_destroyed: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("satblocks_destroyed"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
height_to_satdays_destroyed: EagerVec::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("satdays_destroyed"),
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
)?,
|
||||
indexes_to_coinblocks_destroyed: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("coinblocks_destroyed"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::TWO,
|
||||
@@ -1006,7 +1006,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
indexes_to_coindays_destroyed: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("coindays_destroyed"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::TWO,
|
||||
@@ -1017,7 +1017,7 @@ impl Vecs {
|
||||
)?,
|
||||
indexes_to_net_realized_profit_and_loss_cumulative_30d_change: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("net_realized_profit_and_loss_cumulative_30d_change"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::new(3),
|
||||
@@ -1030,7 +1030,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_realized_cap"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::new(3),
|
||||
@@ -1043,7 +1043,7 @@ impl Vecs {
|
||||
}),
|
||||
indexes_to_net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap: compute_dollars.then(|| {
|
||||
ComputedVecsFromDateIndex::forced_import(
|
||||
path,
|
||||
file,
|
||||
&suffix("net_realized_profit_and_loss_cumulative_30d_change_relative_to_market_cap"),
|
||||
Source::Compute,
|
||||
version + VERSION + Version::new(3),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use brk_vec::{IndexedVec, StoredIndex, StoredType};
|
||||
use brk_vecs::{StampedVec, StoredIndex, StoredType};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct RangeMap<I, T>(BTreeMap<I, T>);
|
||||
@@ -20,12 +20,12 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<I, T> From<&IndexedVec<I, T>> for RangeMap<T, I>
|
||||
impl<I, T> From<&StampedVec<I, T>> for RangeMap<T, I>
|
||||
where
|
||||
I: StoredIndex,
|
||||
T: StoredIndex + StoredType,
|
||||
{
|
||||
fn from(vec: &IndexedVec<I, T>) -> Self {
|
||||
fn from(vec: &StampedVec<I, T>) -> Self {
|
||||
Self(
|
||||
vec.into_iter()
|
||||
.map(|(i, v)| (v.into_owned(), i))
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec};
|
||||
use brk_vecs::{AnyCollectableVec, AnyIterableVec};
|
||||
|
||||
use crate::{Indexes, fetched, indexes, market};
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use std::{ops::Deref, path::Path};
|
||||
use std::{ops::Deref, path::Path, sync::Arc};
|
||||
|
||||
use brk_core::{Bitcoin, DateIndex, Dollars, Height, Result, Version};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyIterableVec, Computation, Format};
|
||||
use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, File, Format};
|
||||
|
||||
use crate::{
|
||||
Indexes, UTXOCohortState, fetched, indexes, market,
|
||||
@@ -25,7 +25,7 @@ pub struct Vecs {
|
||||
impl Vecs {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
cohort_name: Option<&str>,
|
||||
computation: Computation,
|
||||
format: Format,
|
||||
@@ -48,7 +48,7 @@ impl Vecs {
|
||||
)?,
|
||||
|
||||
inner: common::Vecs::forced_import(
|
||||
path,
|
||||
file,
|
||||
cohort_name,
|
||||
computation,
|
||||
format,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -87,12 +87,12 @@ impl PriceToAmount {
|
||||
self.height = Some(height);
|
||||
height.write(&self.path_height())?;
|
||||
|
||||
let config = config::standard();
|
||||
let file = File::create(self.path_state()).inspect_err(|_| {
|
||||
dbg!(self.path_state());
|
||||
})?;
|
||||
let mut writer = BufWriter::new(file);
|
||||
encode_into_std_write(&self.state, &mut writer, config)?;
|
||||
encode_into_std_write(&self.state, &mut writer, config::standard())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -1,641 +0,0 @@
|
||||
use std::{path::Path, thread};
|
||||
|
||||
use brk_core::{
|
||||
AddressData, ByAddressType, EmptyAddressData, Height, OutputType, P2AAddressIndex,
|
||||
P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex,
|
||||
P2WPKHAddressIndex, P2WSHAddressIndex, Result, TypeIndex, Version,
|
||||
};
|
||||
use brk_store::{AnyStore, Store};
|
||||
use fjall::{PersistMode, TransactionalKeyspace};
|
||||
use log::info;
|
||||
|
||||
use crate::stateful::{AddressTypeToTypeIndexTree, WithAddressDataSource};
|
||||
|
||||
const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Stores {
|
||||
keyspace: TransactionalKeyspace,
|
||||
|
||||
pub p2aaddressindex_to_addressdata: Store<P2AAddressIndex, AddressData>,
|
||||
pub p2aaddressindex_to_emptyaddressdata: Store<P2AAddressIndex, EmptyAddressData>,
|
||||
pub p2pk33addressindex_to_addressdata: Store<P2PK33AddressIndex, AddressData>,
|
||||
pub p2pk33addressindex_to_emptyaddressdata: Store<P2PK33AddressIndex, EmptyAddressData>,
|
||||
pub p2pk65addressindex_to_addressdata: Store<P2PK65AddressIndex, AddressData>,
|
||||
pub p2pk65addressindex_to_emptyaddressdata: Store<P2PK65AddressIndex, EmptyAddressData>,
|
||||
pub p2pkhaddressindex_to_addressdata: Store<P2PKHAddressIndex, AddressData>,
|
||||
pub p2pkhaddressindex_to_emptyaddressdata: Store<P2PKHAddressIndex, EmptyAddressData>,
|
||||
pub p2shaddressindex_to_addressdata: Store<P2SHAddressIndex, AddressData>,
|
||||
pub p2shaddressindex_to_emptyaddressdata: Store<P2SHAddressIndex, EmptyAddressData>,
|
||||
pub p2traddressindex_to_addressdata: Store<P2TRAddressIndex, AddressData>,
|
||||
pub p2traddressindex_to_emptyaddressdata: Store<P2TRAddressIndex, EmptyAddressData>,
|
||||
pub p2wpkhaddressindex_to_addressdata: Store<P2WPKHAddressIndex, AddressData>,
|
||||
pub p2wpkhaddressindex_to_emptyaddressdata: Store<P2WPKHAddressIndex, EmptyAddressData>,
|
||||
pub p2wshaddressindex_to_addressdata: Store<P2WSHAddressIndex, AddressData>,
|
||||
pub p2wshaddressindex_to_emptyaddressdata: Store<P2WSHAddressIndex, EmptyAddressData>,
|
||||
}
|
||||
|
||||
impl Stores {
|
||||
pub fn import(
|
||||
path: &Path,
|
||||
version: Version,
|
||||
keyspace: &TransactionalKeyspace,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let (
|
||||
(p2aaddressindex_to_addressdata, p2aaddressindex_to_emptyaddressdata),
|
||||
(p2pk33addressindex_to_addressdata, p2pk33addressindex_to_emptyaddressdata),
|
||||
(p2pk65addressindex_to_addressdata, p2pk65addressindex_to_emptyaddressdata),
|
||||
(p2pkhaddressindex_to_addressdata, p2pkhaddressindex_to_emptyaddressdata),
|
||||
(p2shaddressindex_to_addressdata, p2shaddressindex_to_emptyaddressdata),
|
||||
(p2traddressindex_to_addressdata, p2traddressindex_to_emptyaddressdata),
|
||||
(p2wpkhaddressindex_to_addressdata, p2wpkhaddressindex_to_emptyaddressdata),
|
||||
(p2wshaddressindex_to_addressdata, p2wshaddressindex_to_emptyaddressdata),
|
||||
) = thread::scope(|scope| {
|
||||
let p2a = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2aaddressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2aaddressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2pk33 = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2pk33addressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2pk33addressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2pk65 = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2pk65addressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2pk65addressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2pkh = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2pkhaddressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2pkhaddressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2sh = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2shaddressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2shaddressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2tr = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2traddressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2traddressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2wpkh = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2wpkhaddressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2wpkhaddressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let p2wsh = scope.spawn(|| {
|
||||
(
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2wshaddressindex_to_addressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
Store::import(
|
||||
keyspace,
|
||||
path,
|
||||
"p2wshaddressindex_to_emptyaddressdata",
|
||||
version + VERSION + Version::ZERO,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
(
|
||||
p2a.join().unwrap(),
|
||||
p2pk33.join().unwrap(),
|
||||
p2pk65.join().unwrap(),
|
||||
p2pkh.join().unwrap(),
|
||||
p2sh.join().unwrap(),
|
||||
p2tr.join().unwrap(),
|
||||
p2wpkh.join().unwrap(),
|
||||
p2wsh.join().unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
keyspace: keyspace.clone(),
|
||||
|
||||
p2aaddressindex_to_addressdata,
|
||||
p2aaddressindex_to_emptyaddressdata,
|
||||
|
||||
p2pk33addressindex_to_addressdata,
|
||||
p2pk33addressindex_to_emptyaddressdata,
|
||||
|
||||
p2pk65addressindex_to_addressdata,
|
||||
p2pk65addressindex_to_emptyaddressdata,
|
||||
|
||||
p2pkhaddressindex_to_addressdata,
|
||||
p2pkhaddressindex_to_emptyaddressdata,
|
||||
|
||||
p2shaddressindex_to_addressdata,
|
||||
p2shaddressindex_to_emptyaddressdata,
|
||||
|
||||
p2traddressindex_to_addressdata,
|
||||
p2traddressindex_to_emptyaddressdata,
|
||||
|
||||
p2wpkhaddressindex_to_addressdata,
|
||||
p2wpkhaddressindex_to_emptyaddressdata,
|
||||
|
||||
p2wshaddressindex_to_addressdata,
|
||||
p2wshaddressindex_to_emptyaddressdata,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn starting_height(&self) -> Height {
|
||||
self.as_slice()
|
||||
.into_iter()
|
||||
.map(|store| store.height().map(Height::incremented).unwrap_or_default())
|
||||
.min()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub fn reset(&mut self) -> Result<()> {
|
||||
info!("Resetting stores...");
|
||||
info!("> If it gets stuck here, stop the program and start it again");
|
||||
|
||||
self.as_mut_slice()
|
||||
.into_iter()
|
||||
.try_for_each(|store| store.reset())?;
|
||||
|
||||
self.keyspace
|
||||
.persist(PersistMode::SyncAll)
|
||||
.map_err(|e| e.into())
|
||||
}
|
||||
|
||||
pub fn get_addressdata(
|
||||
&self,
|
||||
address_type: OutputType,
|
||||
type_index: TypeIndex,
|
||||
) -> Result<Option<AddressData>> {
|
||||
Ok(match address_type {
|
||||
OutputType::P2A => self
|
||||
.p2aaddressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2PK33 => self
|
||||
.p2pk33addressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2PK65 => self
|
||||
.p2pk65addressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2PKH => self
|
||||
.p2pkhaddressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2SH => self
|
||||
.p2shaddressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2TR => self
|
||||
.p2traddressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2WPKH => self
|
||||
.p2wpkhaddressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2WSH => self
|
||||
.p2wshaddressindex_to_addressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
_ => unreachable!(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_emptyaddressdata(
|
||||
&self,
|
||||
address_type: OutputType,
|
||||
type_index: TypeIndex,
|
||||
) -> Result<Option<EmptyAddressData>> {
|
||||
Ok(match address_type {
|
||||
OutputType::P2A => self
|
||||
.p2aaddressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2PK33 => self
|
||||
.p2pk33addressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2PK65 => self
|
||||
.p2pk65addressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2PKH => self
|
||||
.p2pkhaddressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2SH => self
|
||||
.p2shaddressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2TR => self
|
||||
.p2traddressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2WPKH => self
|
||||
.p2wpkhaddressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
OutputType::P2WSH => self
|
||||
.p2wshaddressindex_to_emptyaddressdata
|
||||
.get(&type_index.into())?
|
||||
.map(|c| c.into_owned()),
|
||||
_ => unreachable!(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn commit(
|
||||
&mut self,
|
||||
height: Height,
|
||||
addresstype_to_typeindex_to_addressdata: AddressTypeToTypeIndexTree<
|
||||
WithAddressDataSource<AddressData>,
|
||||
>,
|
||||
addresstype_to_typeindex_to_emptyaddressdata: AddressTypeToTypeIndexTree<
|
||||
WithAddressDataSource<EmptyAddressData>,
|
||||
>,
|
||||
) -> Result<()> {
|
||||
let ByAddressType {
|
||||
p2pk65,
|
||||
p2pk33,
|
||||
p2pkh,
|
||||
p2sh,
|
||||
p2wpkh,
|
||||
p2wsh,
|
||||
p2tr,
|
||||
p2a,
|
||||
} = addresstype_to_typeindex_to_addressdata.unwrap();
|
||||
|
||||
let ByAddressType {
|
||||
p2pk65: empty_p2pk65,
|
||||
p2pk33: empty_p2pk33,
|
||||
p2pkh: empty_p2pkh,
|
||||
p2sh: empty_p2sh,
|
||||
p2wpkh: empty_p2wpkh,
|
||||
p2wsh: empty_p2wsh,
|
||||
p2tr: empty_p2tr,
|
||||
p2a: empty_p2a,
|
||||
} = addresstype_to_typeindex_to_emptyaddressdata.unwrap();
|
||||
|
||||
thread::scope(|s| {
|
||||
s.spawn(|| {
|
||||
self.p2aaddressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2a
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2a.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2pk33addressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2pk33
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2pk33.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2pk65addressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2pk65
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2pk65.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2pkhaddressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2pkh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2pkh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2shaddressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2sh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2sh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2traddressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2tr
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2tr.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2wpkhaddressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2wpkh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2wpkh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
s.spawn(|| {
|
||||
self.p2wshaddressindex_to_addressdata.commit_(
|
||||
height,
|
||||
empty_p2wsh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_addressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
p2wsh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
});
|
||||
|
||||
thread::scope(|scope| {
|
||||
scope.spawn(|| {
|
||||
self.p2aaddressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2a.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2a.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2pk33addressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2pk33
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2pk33.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2pk65addressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2pk65
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2pk65.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2pkhaddressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2pkh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2pkh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2shaddressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2sh.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2sh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2traddressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2tr.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2tr.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2wpkhaddressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2wpkh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2wpkh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
scope.spawn(|| {
|
||||
self.p2wshaddressindex_to_emptyaddressdata.commit_(
|
||||
height,
|
||||
p2wsh
|
||||
.iter()
|
||||
.filter(|(_, addressdata)| addressdata.is_from_emptyaddressdata())
|
||||
.map(|(typeindex, _)| (*typeindex).into()),
|
||||
empty_p2wsh.iter().map(|(typeindex, addressdata)| {
|
||||
((*typeindex).into(), addressdata.deref().clone())
|
||||
}),
|
||||
)
|
||||
});
|
||||
});
|
||||
|
||||
self.keyspace
|
||||
.persist(PersistMode::SyncAll)
|
||||
.map_err(|e| e.into())
|
||||
}
|
||||
|
||||
pub fn as_slice(&self) -> [&(dyn AnyStore + Send + Sync); 16] {
|
||||
[
|
||||
&self.p2aaddressindex_to_addressdata,
|
||||
&self.p2aaddressindex_to_emptyaddressdata,
|
||||
&self.p2pk33addressindex_to_addressdata,
|
||||
&self.p2pk33addressindex_to_emptyaddressdata,
|
||||
&self.p2pk65addressindex_to_addressdata,
|
||||
&self.p2pk65addressindex_to_emptyaddressdata,
|
||||
&self.p2pkhaddressindex_to_addressdata,
|
||||
&self.p2pkhaddressindex_to_emptyaddressdata,
|
||||
&self.p2shaddressindex_to_addressdata,
|
||||
&self.p2shaddressindex_to_emptyaddressdata,
|
||||
&self.p2traddressindex_to_addressdata,
|
||||
&self.p2traddressindex_to_emptyaddressdata,
|
||||
&self.p2wpkhaddressindex_to_addressdata,
|
||||
&self.p2wpkhaddressindex_to_emptyaddressdata,
|
||||
&self.p2wshaddressindex_to_addressdata,
|
||||
&self.p2wshaddressindex_to_emptyaddressdata,
|
||||
]
|
||||
}
|
||||
|
||||
fn as_mut_slice(&mut self) -> [&mut (dyn AnyStore + Send + Sync); 16] {
|
||||
[
|
||||
&mut self.p2aaddressindex_to_addressdata,
|
||||
&mut self.p2aaddressindex_to_emptyaddressdata,
|
||||
&mut self.p2pk33addressindex_to_addressdata,
|
||||
&mut self.p2pk33addressindex_to_emptyaddressdata,
|
||||
&mut self.p2pk65addressindex_to_addressdata,
|
||||
&mut self.p2pk65addressindex_to_emptyaddressdata,
|
||||
&mut self.p2pkhaddressindex_to_addressdata,
|
||||
&mut self.p2pkhaddressindex_to_emptyaddressdata,
|
||||
&mut self.p2shaddressindex_to_addressdata,
|
||||
&mut self.p2shaddressindex_to_emptyaddressdata,
|
||||
&mut self.p2traddressindex_to_addressdata,
|
||||
&mut self.p2traddressindex_to_emptyaddressdata,
|
||||
&mut self.p2wpkhaddressindex_to_addressdata,
|
||||
&mut self.p2wpkhaddressindex_to_emptyaddressdata,
|
||||
&mut self.p2wshaddressindex_to_addressdata,
|
||||
&mut self.p2wshaddressindex_to_emptyaddressdata,
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{
|
||||
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredU32,
|
||||
@@ -6,9 +6,9 @@ use brk_core::{
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{
|
||||
use brk_vecs::{
|
||||
AnyCollectableVec, AnyIterableVec, CloneableAnyIterableVec, Computation, ComputedVec,
|
||||
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Format, StoredIndex, VecIterator,
|
||||
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, File, Format, StoredIndex, VecIterator,
|
||||
};
|
||||
|
||||
use crate::grouped::{
|
||||
@@ -86,7 +86,7 @@ pub struct Vecs {
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
version: Version,
|
||||
indexer: &Indexer,
|
||||
indexes: &indexes::Vecs,
|
||||
@@ -98,7 +98,7 @@ impl Vecs {
|
||||
|
||||
let inputindex_to_value = ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"value",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -125,7 +125,7 @@ impl Vecs {
|
||||
|
||||
let txindex_to_weight = ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"weight",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -153,7 +153,7 @@ impl Vecs {
|
||||
|
||||
let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"vsize",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -170,7 +170,7 @@ impl Vecs {
|
||||
|
||||
let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"is_coinbase",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -194,7 +194,7 @@ impl Vecs {
|
||||
|
||||
let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"input_value",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -230,7 +230,7 @@ impl Vecs {
|
||||
|
||||
// let indexes_to_input_value: ComputedVecsFromTxindex<Sats> =
|
||||
// ComputedVecsFromTxindex::forced_import(
|
||||
// path,
|
||||
// file,
|
||||
// "input_value",
|
||||
// true,
|
||||
// version + VERSION + Version::ZERO,
|
||||
@@ -244,7 +244,7 @@ impl Vecs {
|
||||
|
||||
let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"output_value",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -280,7 +280,7 @@ impl Vecs {
|
||||
|
||||
// let indexes_to_output_value: ComputedVecsFromTxindex<Sats> =
|
||||
// ComputedVecsFromTxindex::forced_import(
|
||||
// path,
|
||||
// file,
|
||||
// "output_value",
|
||||
// true,
|
||||
// version + VERSION + Version::ZERO,
|
||||
@@ -294,7 +294,7 @@ impl Vecs {
|
||||
|
||||
let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"fee",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -317,7 +317,7 @@ impl Vecs {
|
||||
|
||||
let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2(
|
||||
computation,
|
||||
path,
|
||||
file,
|
||||
"feerate",
|
||||
version + VERSION + Version::ZERO,
|
||||
format,
|
||||
@@ -337,7 +337,7 @@ impl Vecs {
|
||||
|
||||
Ok(Self {
|
||||
indexes_to_tx_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"tx_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -352,7 +352,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_input_count: ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"input_count",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -367,7 +367,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_output_count: ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"output_count",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -382,7 +382,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"tx_v1",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -392,7 +392,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
indexes_to_tx_v2: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"tx_v2",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -402,7 +402,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
indexes_to_tx_v3: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"tx_v3",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -412,7 +412,7 @@ impl Vecs {
|
||||
VecBuilderOptions::default().add_sum().add_cumulative(),
|
||||
)?,
|
||||
indexes_to_fee: ComputedValueVecsFromTxindex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"fee",
|
||||
indexes,
|
||||
Source::Vec(txindex_to_fee.boxed_clone()),
|
||||
@@ -428,7 +428,7 @@ impl Vecs {
|
||||
.add_average(),
|
||||
)?,
|
||||
indexes_to_feerate: ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"feerate",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -441,7 +441,7 @@ impl Vecs {
|
||||
.add_average(),
|
||||
)?,
|
||||
indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"tx_vsize",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -454,7 +454,7 @@ impl Vecs {
|
||||
.add_average(),
|
||||
)?,
|
||||
indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import(
|
||||
path,
|
||||
file,
|
||||
"tx_weight",
|
||||
Source::None,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -467,7 +467,7 @@ impl Vecs {
|
||||
.add_average(),
|
||||
)?,
|
||||
indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"subsidy",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -483,7 +483,7 @@ impl Vecs {
|
||||
indexes,
|
||||
)?,
|
||||
indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"coinbase",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -499,7 +499,7 @@ impl Vecs {
|
||||
indexes,
|
||||
)?,
|
||||
indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"unclaimed_rewards",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -510,7 +510,7 @@ impl Vecs {
|
||||
indexes,
|
||||
)?,
|
||||
indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"p2a_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -525,7 +525,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"p2ms_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -540,7 +540,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"p2pk33_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -555,7 +555,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"p2pk65_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -570,7 +570,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"p2pkh_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -585,7 +585,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"p2sh_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -600,7 +600,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"p2tr_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -615,7 +615,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"p2wpkh_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -630,7 +630,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"p2wsh_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -645,7 +645,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"opreturn_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -660,7 +660,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"unknownoutput_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -675,7 +675,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"emptyoutput_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
@@ -690,7 +690,7 @@ impl Vecs {
|
||||
.add_cumulative(),
|
||||
)?,
|
||||
indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import(
|
||||
path,
|
||||
file,
|
||||
"exact_utxo_count",
|
||||
Source::Compute,
|
||||
version + VERSION + Version::ZERO,
|
||||
|
||||
@@ -16,7 +16,7 @@ derive_deref = { workspace = true }
|
||||
fjall = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
log = { workspace = true }
|
||||
rapidhash = "1.4.0"
|
||||
rapidhash = "2.0.2"
|
||||
rlimit = "0.10.2"
|
||||
serde = { workspace = true }
|
||||
serde_bytes = { workspace = true }
|
||||
|
||||
@@ -35,7 +35,8 @@ pub enum Error {
|
||||
WrongAddressType,
|
||||
UnindexableDate,
|
||||
|
||||
String(&'static str),
|
||||
Str(&'static str),
|
||||
String(String),
|
||||
}
|
||||
|
||||
impl From<time::SystemTimeError> for Error {
|
||||
@@ -134,6 +135,7 @@ impl fmt::Display for Error {
|
||||
"Date cannot be indexed, must be 2009-01-03, 2009-01-09 or greater"
|
||||
),
|
||||
|
||||
Error::Str(s) => write!(f, "{s}"),
|
||||
Error::String(s) => write!(f, "{s}"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,14 @@ pub struct AddressGroups<T> {
|
||||
}
|
||||
|
||||
impl<T> AddressGroups<T> {
|
||||
pub fn as_boxed_mut_vecs(&mut self) -> Vec<Box<[&mut T]>> {
|
||||
vec![
|
||||
Box::new(self.ge_amount.as_mut_vec()),
|
||||
Box::new(self.amount_range.as_mut_vec()),
|
||||
Box::new(self.lt_amount.as_mut_vec()),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn as_mut_vecs(&mut self) -> Vec<&mut T> {
|
||||
self.ge_amount
|
||||
.as_mut_vec()
|
||||
|
||||
@@ -171,3 +171,11 @@ where
|
||||
self.p2a += rhs.p2a;
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ByAddressType<Option<T>> {
|
||||
pub fn take(&mut self) {
|
||||
self.as_mut_vec().into_iter().for_each(|opt| {
|
||||
opt.take();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,3 +3,10 @@ pub struct ByAnyAddress<T> {
|
||||
pub loaded: T,
|
||||
pub empty: T,
|
||||
}
|
||||
|
||||
impl<T> ByAnyAddress<Option<T>> {
|
||||
pub fn take(&mut self) {
|
||||
self.loaded.take();
|
||||
self.empty.take();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,21 @@ pub struct UTXOGroups<T> {
|
||||
}
|
||||
|
||||
impl<T> UTXOGroups<T> {
|
||||
pub fn as_boxed_mut_vecs(&mut self) -> Vec<Box<[&mut T]>> {
|
||||
vec![
|
||||
Box::new([&mut self.all]),
|
||||
Box::new(self.term.as_mut_vec()),
|
||||
Box::new(self.max_age.as_mut_vec()),
|
||||
Box::new(self.min_age.as_mut_vec()),
|
||||
Box::new(self.ge_amount.as_mut_vec()),
|
||||
Box::new(self.age_range.as_mut_vec()),
|
||||
Box::new(self.epoch.as_mut_vec()),
|
||||
Box::new(self.amount_range.as_mut_vec()),
|
||||
Box::new(self.lt_amount.as_mut_vec()),
|
||||
Box::new(self._type.as_mut_vec()),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn as_mut_vecs(&mut self) -> Vec<&mut T> {
|
||||
[&mut self.all]
|
||||
.into_iter()
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
use std::hash::Hasher;
|
||||
|
||||
use byteview::ByteView;
|
||||
use derive_deref::Deref;
|
||||
use zerocopy::{FromBytes, IntoBytes};
|
||||
@@ -25,9 +23,7 @@ pub struct AddressBytesHash([u8; 8]);
|
||||
|
||||
impl From<(&AddressBytes, OutputType)> for AddressBytesHash {
|
||||
fn from((address_bytes, outputtype): (&AddressBytes, OutputType)) -> Self {
|
||||
let mut hasher = rapidhash::RapidHasher::default();
|
||||
hasher.write(address_bytes.as_slice());
|
||||
let mut slice = hasher.finish().to_le_bytes();
|
||||
let mut slice = rapidhash::v3::rapidhash_v3(address_bytes.as_slice()).to_le_bytes();
|
||||
slice[0] = slice[0].wrapping_add(outputtype as u8);
|
||||
Self(slice)
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ impl Height {
|
||||
pub const ZERO: Self = Self(0);
|
||||
pub const MAX: Self = Self(u32::MAX);
|
||||
|
||||
pub fn new(height: u32) -> Self {
|
||||
pub const fn new(height: u32) -> Self {
|
||||
Self(height)
|
||||
}
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ impl LoadedAddressData {
|
||||
|
||||
pub fn send(&mut self, amount: Sats, previous_price: Option<Dollars>) -> Result<()> {
|
||||
if self.amount() < amount {
|
||||
return Err(Error::String("Previous_amount smaller than sent amount"));
|
||||
return Err(Error::Str("Previous_amount smaller than sent amount"));
|
||||
}
|
||||
self.sent += amount;
|
||||
self.outputs_len -= 1;
|
||||
|
||||
@@ -5,7 +5,7 @@ pub fn copy_first_4bytes(slice: &[u8]) -> Result<[u8; 4]> {
|
||||
let mut buf: [u8; 4] = [0; 4];
|
||||
let buf_len = buf.len();
|
||||
if slice.len() < buf_len {
|
||||
return Err(Error::String("Buffer is too small to convert to 8 bytes"));
|
||||
return Err(Error::Str("Buffer is too small to convert to 8 bytes"));
|
||||
}
|
||||
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
|
||||
buf[i] = *r;
|
||||
@@ -18,7 +18,7 @@ pub fn copy_first_8bytes(slice: &[u8]) -> Result<[u8; 8]> {
|
||||
let mut buf: [u8; 8] = [0; 8];
|
||||
let buf_len = buf.len();
|
||||
if slice.len() < buf_len {
|
||||
return Err(Error::String("Buffer is too small to convert to 8 bytes"));
|
||||
return Err(Error::Str("Buffer is too small to convert to 8 bytes"));
|
||||
}
|
||||
slice.iter().take(buf_len).enumerate().for_each(|(i, r)| {
|
||||
buf[i] = *r;
|
||||
|
||||
@@ -11,3 +11,4 @@ repository.workspace = true
|
||||
brk_logger = { workspace = true }
|
||||
ctrlc = { version = "3.4.7", features = ["termination"] }
|
||||
log = { workspace = true }
|
||||
parking_lot = { workspace = true }
|
||||
|
||||
@@ -8,7 +8,7 @@ fn main() {
|
||||
|
||||
brk_logger::init(Some(Path::new(".log")));
|
||||
|
||||
exit.block();
|
||||
let lock = exit.lock();
|
||||
|
||||
let mut i = 0;
|
||||
while i < 21 {
|
||||
@@ -17,7 +17,7 @@ fn main() {
|
||||
i += 1;
|
||||
}
|
||||
|
||||
exit.release();
|
||||
drop(lock);
|
||||
|
||||
let mut j = 0;
|
||||
while j < 10 {
|
||||
|
||||
+11
-46
@@ -3,69 +3,34 @@
|
||||
#![doc = include_str!("../examples/main.rs")]
|
||||
#![doc = "```"]
|
||||
|
||||
use std::{
|
||||
process::exit,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
thread::sleep,
|
||||
time::Duration,
|
||||
};
|
||||
use std::{process::exit, sync::Arc};
|
||||
|
||||
use log::info;
|
||||
use parking_lot::{RwLock, RwLockReadGuard};
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct Exit {
|
||||
blocking: Arc<AtomicBool>,
|
||||
triggered: Arc<AtomicBool>,
|
||||
}
|
||||
pub struct Exit(Arc<RwLock<()>>);
|
||||
|
||||
impl Exit {
|
||||
pub fn new() -> Self {
|
||||
let s = Self {
|
||||
triggered: Arc::new(AtomicBool::new(false)),
|
||||
blocking: Arc::new(AtomicBool::new(false)),
|
||||
};
|
||||
let arc = Arc::new(RwLock::new(()));
|
||||
|
||||
let triggered = s.triggered.clone();
|
||||
|
||||
let blocking = s.blocking.clone();
|
||||
let is_blocking = move || blocking.load(Ordering::SeqCst);
|
||||
let copy = arc.clone();
|
||||
|
||||
ctrlc::set_handler(move || {
|
||||
info!("Exitting...");
|
||||
|
||||
triggered.store(true, Ordering::SeqCst);
|
||||
|
||||
if is_blocking() {
|
||||
if copy.is_locked() {
|
||||
info!("Waiting to exit safely...");
|
||||
|
||||
while is_blocking() {
|
||||
sleep(Duration::from_millis(50));
|
||||
}
|
||||
}
|
||||
|
||||
let _lock = copy.write();
|
||||
info!("Exiting...");
|
||||
exit(0);
|
||||
})
|
||||
.expect("Error setting Ctrl-C handler");
|
||||
|
||||
s
|
||||
Self(arc)
|
||||
}
|
||||
|
||||
pub fn block(&self) {
|
||||
self.blocking.store(true, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn blocked(&self) -> bool {
|
||||
self.blocking.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub fn release(&self) {
|
||||
self.blocking.store(false, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn triggered(&self) -> bool {
|
||||
self.triggered.load(Ordering::SeqCst)
|
||||
pub fn lock(&self) -> RwLockReadGuard<'_, ()> {
|
||||
self.0.read()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ brk_exit = { workspace = true }
|
||||
brk_logger = { workspace = true }
|
||||
brk_parser = { workspace = true }
|
||||
brk_store = { workspace = true }
|
||||
brk_vec = { workspace = true }
|
||||
brk_vecs = { workspace = true }
|
||||
color-eyre = { workspace = true }
|
||||
fjall = { workspace = true }
|
||||
log = { workspace = true }
|
||||
|
||||
@@ -15,9 +15,9 @@ fn main() -> color_eyre::Result<()> {
|
||||
brk_logger::init(Some(Path::new(".log")));
|
||||
|
||||
// let bitcoin_dir = brk_core::default_bitcoin_path();
|
||||
let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
|
||||
let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
|
||||
// let outputs_dir = brk_core::default_brk_path().join("outputs");
|
||||
let outputs_dir = Path::new("/Volumes/WD_BLACK/brk/outputs");
|
||||
let outputs_dir = Path::new("/Volumes/WD_BLACK1/brk");
|
||||
|
||||
let rpc = Box::leak(Box::new(bitcoincore_rpc::Client::new(
|
||||
"http://localhost:8332",
|
||||
@@ -33,7 +33,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
|
||||
loop {
|
||||
let i = Instant::now();
|
||||
indexer.index(&parser, rpc, &exit, false)?;
|
||||
indexer.index(&parser, rpc, &exit, true)?;
|
||||
dbg!(i.elapsed());
|
||||
|
||||
sleep(Duration::from_secs(5 * 60));
|
||||
@@ -6,7 +6,7 @@ use brk_core::{
|
||||
Result, TxIndex, TypeIndex, UnknownOutputIndex,
|
||||
};
|
||||
use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
|
||||
use brk_vec::{AnyIndexedVec, AnyIterableVec, AnyVec, IndexedVec, StoredIndex, StoredType};
|
||||
use brk_vecs::{AnyIterableVec, AnyStampedVec, AnyVec, StampedVec, StoredIndex, StoredType};
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
|
||||
use crate::{Stores, Vecs};
|
||||
@@ -89,8 +89,10 @@ impl Indexes {
|
||||
impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from((vecs, stores, rpc): (&mut Vecs, &Stores, &Client)) -> color_eyre::Result<Self> {
|
||||
// Height at which we wanna start: min last saved + 1 or 0
|
||||
let starting_height = vecs.starting_height().min(stores.starting_height());
|
||||
// Height at which we want to start: min last saved + 1 or 0
|
||||
let vecs_starting_height = vecs.starting_height();
|
||||
let stores_starting_height = stores.starting_height();
|
||||
let starting_height = vecs_starting_height.min(stores_starting_height);
|
||||
|
||||
let range = u32::from(
|
||||
starting_height
|
||||
@@ -208,18 +210,18 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
|
||||
}
|
||||
|
||||
pub fn starting_index<I, T>(
|
||||
height_to_index: &IndexedVec<Height, I>,
|
||||
index_to_else: &IndexedVec<I, T>,
|
||||
height_to_index: &StampedVec<Height, I>,
|
||||
index_to_else: &StampedVec<I, T>,
|
||||
starting_height: Height,
|
||||
) -> Option<I>
|
||||
where
|
||||
I: StoredType + StoredIndex + From<usize>,
|
||||
T: StoredType,
|
||||
{
|
||||
let h = height_to_index.height();
|
||||
let h = Height::from(u64::from(height_to_index.stamp()));
|
||||
if h.is_zero() {
|
||||
None
|
||||
} else if height_to_index.height() + 1_u32 == starting_height {
|
||||
} else if h + 1_u32 == starting_height {
|
||||
Some(I::from(index_to_else.len()))
|
||||
} else {
|
||||
height_to_index.iter().get_inner(starting_height)
|
||||
|
||||
+137
-100
@@ -1,21 +1,20 @@
|
||||
#![doc = include_str!("../README.md")]
|
||||
#![doc = "\n## Example\n\n```rust"]
|
||||
#![doc = include_str!("../examples/main.rs")]
|
||||
#![doc = include_str!("../examples/indexer.rs")]
|
||||
#![doc = "```"]
|
||||
|
||||
use std::{collections::BTreeMap, path::Path, str::FromStr, thread};
|
||||
|
||||
use brk_core::{
|
||||
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
|
||||
OutputType, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex, TypeIndexWithOutputindex,
|
||||
Unit, Version, Vin, Vout, setrlimit,
|
||||
};
|
||||
use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc, thread, time::Instant};
|
||||
|
||||
use bitcoin::{Transaction, TxIn, TxOut};
|
||||
use brk_core::{
|
||||
AddressBytes, AddressBytesHash, BlockHash, BlockHashPrefix, Height, InputIndex, OutputIndex,
|
||||
OutputType, Result, Sats, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex,
|
||||
TypeIndexWithOutputindex, Unit, Version, Vin, Vout, setrlimit,
|
||||
};
|
||||
use brk_exit::Exit;
|
||||
use brk_parser::Parser;
|
||||
use brk_store::AnyStore;
|
||||
use brk_vec::{AnyVec, Mmap, VecIterator};
|
||||
use brk_vecs::{AnyVec, File, PAGE_SIZE, Reader, VecIterator};
|
||||
use color_eyre::eyre::{ContextCompat, eyre};
|
||||
use log::{error, info};
|
||||
use rayon::prelude::*;
|
||||
@@ -28,11 +27,12 @@ pub use stores::*;
|
||||
pub use vecs::*;
|
||||
|
||||
const SNAPSHOT_BLOCK_RANGE: usize = 1000;
|
||||
const COLLISIONS_CHECKED_UP_TO: u32 = 893_000;
|
||||
const COLLISIONS_CHECKED_UP_TO: Height = Height::new(907_000);
|
||||
const VERSION: Version = Version::ONE;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Indexer {
|
||||
pub file: Arc<File>,
|
||||
pub vecs: Vecs,
|
||||
pub stores: Stores,
|
||||
}
|
||||
@@ -41,9 +41,19 @@ impl Indexer {
|
||||
pub fn forced_import(outputs_dir: &Path) -> color_eyre::Result<Self> {
|
||||
setrlimit()?;
|
||||
|
||||
let file = Arc::new(File::open(&outputs_dir.join("indexed/vecs"))?);
|
||||
|
||||
let vecs = Vecs::forced_import(&file, VERSION + Version::ZERO)?;
|
||||
|
||||
file.set_min_len(PAGE_SIZE * 50_000_000)?;
|
||||
|
||||
Ok(Self {
|
||||
vecs: Vecs::forced_import(&outputs_dir.join("vecs/indexed"), VERSION + Version::ZERO)?,
|
||||
stores: Stores::forced_import(&outputs_dir.join("stores"), VERSION + Version::ZERO)?,
|
||||
vecs,
|
||||
stores: Stores::forced_import(
|
||||
&outputs_dir.join("indexed/stores"),
|
||||
VERSION + Version::ZERO,
|
||||
)?,
|
||||
file,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -54,17 +64,19 @@ impl Indexer {
|
||||
exit: &Exit,
|
||||
check_collisions: bool,
|
||||
) -> color_eyre::Result<Indexes> {
|
||||
let file = self.file.clone();
|
||||
|
||||
let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc))
|
||||
.unwrap_or_else(|_report| Indexes::default());
|
||||
|
||||
// dbg!(starting_indexes);
|
||||
// panic!();
|
||||
|
||||
exit.block();
|
||||
let lock = exit.lock();
|
||||
self.stores
|
||||
.rollback_if_needed(&mut self.vecs, &starting_indexes)?;
|
||||
self.vecs.rollback_if_needed(&starting_indexes)?;
|
||||
exit.release();
|
||||
drop(lock);
|
||||
|
||||
let vecs = &mut self.vecs;
|
||||
let stores = &mut self.stores;
|
||||
@@ -84,85 +96,84 @@ impl Indexer {
|
||||
|
||||
info!("Started indexing...");
|
||||
|
||||
let export_if_needed = |stores: &mut Stores,
|
||||
vecs: &mut Vecs,
|
||||
height: Height,
|
||||
rem: bool,
|
||||
exit: &Exit|
|
||||
-> color_eyre::Result<bool> {
|
||||
if height == 0 || (height % SNAPSHOT_BLOCK_RANGE != 0) != rem || exit.triggered() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
info!("Exporting...");
|
||||
exit.block();
|
||||
stores.commit(height)?;
|
||||
vecs.flush(height)?;
|
||||
exit.release();
|
||||
Ok(true)
|
||||
let should_export = |height: Height, rem: bool| -> bool {
|
||||
height != 0 && (height % SNAPSHOT_BLOCK_RANGE == 0) != rem
|
||||
};
|
||||
|
||||
let mut txindex_to_first_outputindex_mmap_opt = None;
|
||||
let mut p2pk65addressindex_to_p2pk65bytes_mmap_opt = None;
|
||||
let mut p2pk33addressindex_to_p2pk33bytes_mmap_opt = None;
|
||||
let mut p2pkhaddressindex_to_p2pkhbytes_mmap_opt = None;
|
||||
let mut p2shaddressindex_to_p2shbytes_mmap_opt = None;
|
||||
let mut p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt = None;
|
||||
let mut p2wshaddressindex_to_p2wshbytes_mmap_opt = None;
|
||||
let mut p2traddressindex_to_p2trbytes_mmap_opt = None;
|
||||
let mut p2aaddressindex_to_p2abytes_mmap_opt = None;
|
||||
let export =
|
||||
|stores: &mut Stores, vecs: &mut Vecs, height: Height, exit: &Exit| -> Result<()> {
|
||||
info!("Exporting...");
|
||||
let _lock = exit.lock();
|
||||
let i = Instant::now();
|
||||
stores.commit(height).unwrap();
|
||||
info!("Commited stores in {}s", i.elapsed().as_secs());
|
||||
let i = Instant::now();
|
||||
vecs.flush(height)?;
|
||||
info!("Flushed vecs in {}s", i.elapsed().as_secs());
|
||||
let i = Instant::now();
|
||||
file.flush()?;
|
||||
info!("Flushed file in {}s", i.elapsed().as_secs());
|
||||
Ok(())
|
||||
};
|
||||
|
||||
let mut txindex_to_first_outputindex_reader_opt = None;
|
||||
let mut p2pk65addressindex_to_p2pk65bytes_reader_opt = None;
|
||||
let mut p2pk33addressindex_to_p2pk33bytes_reader_opt = None;
|
||||
let mut p2pkhaddressindex_to_p2pkhbytes_reader_opt = None;
|
||||
let mut p2shaddressindex_to_p2shbytes_reader_opt = None;
|
||||
let mut p2wpkhaddressindex_to_p2wpkhbytes_reader_opt = None;
|
||||
let mut p2wshaddressindex_to_p2wshbytes_reader_opt = None;
|
||||
let mut p2traddressindex_to_p2trbytes_reader_opt = None;
|
||||
let mut p2aaddressindex_to_p2abytes_reader_opt = None;
|
||||
|
||||
let reset_mmaps_options =
|
||||
|vecs: &mut Vecs,
|
||||
txindex_to_first_outputindex_mmap_opt: &mut Option<Mmap>,
|
||||
p2pk65addressindex_to_p2pk65bytes_mmap_opt: &mut Option<Mmap>,
|
||||
p2pk33addressindex_to_p2pk33bytes_mmap_opt: &mut Option<Mmap>,
|
||||
p2pkhaddressindex_to_p2pkhbytes_mmap_opt: &mut Option<Mmap>,
|
||||
p2shaddressindex_to_p2shbytes_mmap_opt: &mut Option<Mmap>,
|
||||
p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt: &mut Option<Mmap>,
|
||||
p2wshaddressindex_to_p2wshbytes_mmap_opt: &mut Option<Mmap>,
|
||||
p2traddressindex_to_p2trbytes_mmap_opt: &mut Option<Mmap>,
|
||||
p2aaddressindex_to_p2abytes_mmap_opt: &mut Option<Mmap>| {
|
||||
txindex_to_first_outputindex_mmap_opt
|
||||
.replace(vecs.txindex_to_first_outputindex.create_mmap().unwrap());
|
||||
p2pk65addressindex_to_p2pk65bytes_mmap_opt.replace(
|
||||
txindex_to_first_outputindex_reader_opt: &mut Option<Reader<'static>>,
|
||||
p2pk65addressindex_to_p2pk65bytes_reader_opt: &mut Option<Reader<'static>>,
|
||||
p2pk33addressindex_to_p2pk33bytes_reader_opt: &mut Option<Reader<'static>>,
|
||||
p2pkhaddressindex_to_p2pkhbytes_reader_opt: &mut Option<Reader<'static>>,
|
||||
p2shaddressindex_to_p2shbytes_reader_opt: &mut Option<Reader<'static>>,
|
||||
p2wpkhaddressindex_to_p2wpkhbytes_reader_opt: &mut Option<Reader<'static>>,
|
||||
p2wshaddressindex_to_p2wshbytes_reader_opt: &mut Option<Reader<'static>>,
|
||||
p2traddressindex_to_p2trbytes_reader_opt: &mut Option<Reader<'static>>,
|
||||
p2aaddressindex_to_p2abytes_reader_opt: &mut Option<Reader<'static>>| {
|
||||
txindex_to_first_outputindex_reader_opt
|
||||
.replace(vecs.txindex_to_first_outputindex.create_static_reader());
|
||||
p2pk65addressindex_to_p2pk65bytes_reader_opt.replace(
|
||||
vecs.p2pk65addressindex_to_p2pk65bytes
|
||||
.create_mmap()
|
||||
.unwrap(),
|
||||
.create_static_reader(),
|
||||
);
|
||||
p2pk33addressindex_to_p2pk33bytes_mmap_opt.replace(
|
||||
p2pk33addressindex_to_p2pk33bytes_reader_opt.replace(
|
||||
vecs.p2pk33addressindex_to_p2pk33bytes
|
||||
.create_mmap()
|
||||
.unwrap(),
|
||||
.create_static_reader(),
|
||||
);
|
||||
p2pkhaddressindex_to_p2pkhbytes_mmap_opt
|
||||
.replace(vecs.p2pkhaddressindex_to_p2pkhbytes.create_mmap().unwrap());
|
||||
p2shaddressindex_to_p2shbytes_mmap_opt
|
||||
.replace(vecs.p2shaddressindex_to_p2shbytes.create_mmap().unwrap());
|
||||
p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt.replace(
|
||||
p2pkhaddressindex_to_p2pkhbytes_reader_opt
|
||||
.replace(vecs.p2pkhaddressindex_to_p2pkhbytes.create_static_reader());
|
||||
p2shaddressindex_to_p2shbytes_reader_opt
|
||||
.replace(vecs.p2shaddressindex_to_p2shbytes.create_static_reader());
|
||||
p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.replace(
|
||||
vecs.p2wpkhaddressindex_to_p2wpkhbytes
|
||||
.create_mmap()
|
||||
.unwrap(),
|
||||
.create_static_reader(),
|
||||
);
|
||||
p2wshaddressindex_to_p2wshbytes_mmap_opt
|
||||
.replace(vecs.p2wshaddressindex_to_p2wshbytes.create_mmap().unwrap());
|
||||
p2traddressindex_to_p2trbytes_mmap_opt
|
||||
.replace(vecs.p2traddressindex_to_p2trbytes.create_mmap().unwrap());
|
||||
p2aaddressindex_to_p2abytes_mmap_opt
|
||||
.replace(vecs.p2aaddressindex_to_p2abytes.create_mmap().unwrap());
|
||||
p2wshaddressindex_to_p2wshbytes_reader_opt
|
||||
.replace(vecs.p2wshaddressindex_to_p2wshbytes.create_static_reader());
|
||||
p2traddressindex_to_p2trbytes_reader_opt
|
||||
.replace(vecs.p2traddressindex_to_p2trbytes.create_static_reader());
|
||||
p2aaddressindex_to_p2abytes_reader_opt
|
||||
.replace(vecs.p2aaddressindex_to_p2abytes.create_static_reader());
|
||||
};
|
||||
|
||||
reset_mmaps_options(
|
||||
vecs,
|
||||
&mut txindex_to_first_outputindex_mmap_opt,
|
||||
&mut p2pk65addressindex_to_p2pk65bytes_mmap_opt,
|
||||
&mut p2pk33addressindex_to_p2pk33bytes_mmap_opt,
|
||||
&mut p2pkhaddressindex_to_p2pkhbytes_mmap_opt,
|
||||
&mut p2shaddressindex_to_p2shbytes_mmap_opt,
|
||||
&mut p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt,
|
||||
&mut p2wshaddressindex_to_p2wshbytes_mmap_opt,
|
||||
&mut p2traddressindex_to_p2trbytes_mmap_opt,
|
||||
&mut p2aaddressindex_to_p2abytes_mmap_opt,
|
||||
&mut txindex_to_first_outputindex_reader_opt,
|
||||
&mut p2pk65addressindex_to_p2pk65bytes_reader_opt,
|
||||
&mut p2pk33addressindex_to_p2pk33bytes_reader_opt,
|
||||
&mut p2pkhaddressindex_to_p2pkhbytes_reader_opt,
|
||||
&mut p2shaddressindex_to_p2shbytes_reader_opt,
|
||||
&mut p2wpkhaddressindex_to_p2wpkhbytes_reader_opt,
|
||||
&mut p2wshaddressindex_to_p2wshbytes_reader_opt,
|
||||
&mut p2traddressindex_to_p2trbytes_reader_opt,
|
||||
&mut p2aaddressindex_to_p2abytes_reader_opt,
|
||||
);
|
||||
|
||||
parser.parse(start, end).iter().try_for_each(
|
||||
@@ -171,18 +182,18 @@ impl Indexer {
|
||||
|
||||
idxs.height = height;
|
||||
|
||||
let txindex_to_first_outputindex_mmap = txindex_to_first_outputindex_mmap_opt.as_ref().unwrap();
|
||||
let p2pk65addressindex_to_p2pk65bytes_mmap = p2pk65addressindex_to_p2pk65bytes_mmap_opt.as_ref().unwrap();
|
||||
let p2pk33addressindex_to_p2pk33bytes_mmap = p2pk33addressindex_to_p2pk33bytes_mmap_opt.as_ref().unwrap();
|
||||
let p2pkhaddressindex_to_p2pkhbytes_mmap = p2pkhaddressindex_to_p2pkhbytes_mmap_opt.as_ref().unwrap();
|
||||
let p2shaddressindex_to_p2shbytes_mmap = p2shaddressindex_to_p2shbytes_mmap_opt.as_ref().unwrap();
|
||||
let p2wpkhaddressindex_to_p2wpkhbytes_mmap = p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt.as_ref().unwrap();
|
||||
let p2wshaddressindex_to_p2wshbytes_mmap = p2wshaddressindex_to_p2wshbytes_mmap_opt.as_ref().unwrap();
|
||||
let p2traddressindex_to_p2trbytes_mmap = p2traddressindex_to_p2trbytes_mmap_opt.as_ref().unwrap();
|
||||
let p2aaddressindex_to_p2abytes_mmap = p2aaddressindex_to_p2abytes_mmap_opt.as_ref().unwrap();
|
||||
let txindex_to_first_outputindex_mmap = txindex_to_first_outputindex_reader_opt.as_ref().unwrap();
|
||||
let p2pk65addressindex_to_p2pk65bytes_mmap = p2pk65addressindex_to_p2pk65bytes_reader_opt.as_ref().unwrap();
|
||||
let p2pk33addressindex_to_p2pk33bytes_mmap = p2pk33addressindex_to_p2pk33bytes_reader_opt.as_ref().unwrap();
|
||||
let p2pkhaddressindex_to_p2pkhbytes_mmap = p2pkhaddressindex_to_p2pkhbytes_reader_opt.as_ref().unwrap();
|
||||
let p2shaddressindex_to_p2shbytes_mmap = p2shaddressindex_to_p2shbytes_reader_opt.as_ref().unwrap();
|
||||
let p2wpkhaddressindex_to_p2wpkhbytes_mmap = p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.as_ref().unwrap();
|
||||
let p2wshaddressindex_to_p2wshbytes_mmap = p2wshaddressindex_to_p2wshbytes_reader_opt.as_ref().unwrap();
|
||||
let p2traddressindex_to_p2trbytes_mmap = p2traddressindex_to_p2trbytes_reader_opt.as_ref().unwrap();
|
||||
let p2aaddressindex_to_p2abytes_mmap = p2aaddressindex_to_p2abytes_reader_opt.as_ref().unwrap();
|
||||
|
||||
// Used to check rapidhash collisions
|
||||
let check_collisions = check_collisions && height > Height::new(COLLISIONS_CHECKED_UP_TO);
|
||||
let check_collisions = check_collisions && height > COLLISIONS_CHECKED_UP_TO ;
|
||||
|
||||
let blockhash = BlockHash::from(blockhash);
|
||||
let blockhash_prefix = BlockHashPrefix::from(&blockhash);
|
||||
@@ -722,6 +733,8 @@ impl Indexer {
|
||||
},
|
||||
)?;
|
||||
|
||||
drop(txindex_to_txid_iter);
|
||||
|
||||
txindex_to_tx_and_txid
|
||||
.into_iter()
|
||||
.try_for_each(|(txindex, (tx, txid))| -> color_eyre::Result<()> {
|
||||
@@ -738,20 +751,29 @@ impl Indexer {
|
||||
idxs.inputindex += InputIndex::from(inputs_len);
|
||||
idxs.outputindex += OutputIndex::from(outputs_len);
|
||||
|
||||
let exported = export_if_needed(stores, vecs, height, false, exit)?;
|
||||
|
||||
if exported {
|
||||
if should_export(height, false) {
|
||||
txindex_to_first_outputindex_reader_opt.take();
|
||||
p2pk65addressindex_to_p2pk65bytes_reader_opt.take();
|
||||
p2pk33addressindex_to_p2pk33bytes_reader_opt.take();
|
||||
p2pkhaddressindex_to_p2pkhbytes_reader_opt.take();
|
||||
p2shaddressindex_to_p2shbytes_reader_opt.take();
|
||||
p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take();
|
||||
p2wshaddressindex_to_p2wshbytes_reader_opt.take();
|
||||
p2traddressindex_to_p2trbytes_reader_opt.take();
|
||||
p2aaddressindex_to_p2abytes_reader_opt.take();
|
||||
export(stores, vecs, height, exit)?;
|
||||
reset_mmaps_options(
|
||||
vecs,
|
||||
&mut txindex_to_first_outputindex_mmap_opt,
|
||||
&mut p2pk65addressindex_to_p2pk65bytes_mmap_opt,
|
||||
&mut p2pk33addressindex_to_p2pk33bytes_mmap_opt,
|
||||
&mut p2pkhaddressindex_to_p2pkhbytes_mmap_opt,
|
||||
&mut p2shaddressindex_to_p2shbytes_mmap_opt,
|
||||
&mut p2wpkhaddressindex_to_p2wpkhbytes_mmap_opt,
|
||||
&mut p2wshaddressindex_to_p2wshbytes_mmap_opt,
|
||||
&mut p2traddressindex_to_p2trbytes_mmap_opt,
|
||||
&mut p2aaddressindex_to_p2abytes_mmap_opt,
|
||||
&mut txindex_to_first_outputindex_reader_opt,
|
||||
&mut p2pk65addressindex_to_p2pk65bytes_reader_opt,
|
||||
&mut p2pk33addressindex_to_p2pk33bytes_reader_opt,
|
||||
&mut p2pkhaddressindex_to_p2pkhbytes_reader_opt,
|
||||
&mut p2shaddressindex_to_p2shbytes_reader_opt,
|
||||
&mut p2wpkhaddressindex_to_p2wpkhbytes_reader_opt,
|
||||
&mut p2wshaddressindex_to_p2wshbytes_reader_opt,
|
||||
&mut p2traddressindex_to_p2trbytes_reader_opt,
|
||||
&mut p2aaddressindex_to_p2abytes_reader_opt,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -759,7 +781,22 @@ impl Indexer {
|
||||
},
|
||||
)?;
|
||||
|
||||
export_if_needed(stores, vecs, idxs.height, true, exit)?;
|
||||
if should_export(idxs.height, true) {
|
||||
txindex_to_first_outputindex_reader_opt.take();
|
||||
p2pk65addressindex_to_p2pk65bytes_reader_opt.take();
|
||||
p2pk33addressindex_to_p2pk33bytes_reader_opt.take();
|
||||
p2pkhaddressindex_to_p2pkhbytes_reader_opt.take();
|
||||
p2shaddressindex_to_p2shbytes_reader_opt.take();
|
||||
p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take();
|
||||
p2wshaddressindex_to_p2wshbytes_reader_opt.take();
|
||||
p2traddressindex_to_p2trbytes_reader_opt.take();
|
||||
p2aaddressindex_to_p2abytes_reader_opt.take();
|
||||
export(stores, vecs, idxs.height, exit)?;
|
||||
}
|
||||
|
||||
let i = Instant::now();
|
||||
file.punch_holes()?;
|
||||
info!("Punched holes in file in {}s", i.elapsed().as_secs());
|
||||
|
||||
Ok(starting_indexes)
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ use brk_core::{
|
||||
OutputType, Result, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version,
|
||||
};
|
||||
use brk_store::{AnyStore, Store};
|
||||
use brk_vec::{AnyIterableVec, VecIterator};
|
||||
use brk_vecs::{AnyIterableVec, VecIterator};
|
||||
use fjall::{PersistMode, TransactionalKeyspace};
|
||||
use rayon::prelude::*;
|
||||
|
||||
|
||||
+184
-186
@@ -1,4 +1,4 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use brk_core::{
|
||||
AddressBytes, BlockHash, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
|
||||
@@ -8,7 +8,7 @@ use brk_core::{
|
||||
RawLockTime, Result, Sats, StoredF64, StoredU32, StoredUsize, Timestamp, TxIndex, TxVersion,
|
||||
Txid, TypeIndex, UnknownOutputIndex, Version, Weight,
|
||||
};
|
||||
use brk_vec::{AnyCollectableVec, AnyIndexedVec, Format, IndexedVec};
|
||||
use brk_vecs::{AnyCollectableVec, AnyStampedVec, File, Format, StampedVec};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::Indexes;
|
||||
@@ -17,317 +17,317 @@ const VERSION: Version = Version::ZERO;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Vecs {
|
||||
pub emptyoutputindex_to_txindex: IndexedVec<EmptyOutputIndex, TxIndex>,
|
||||
pub height_to_blockhash: IndexedVec<Height, BlockHash>,
|
||||
pub height_to_difficulty: IndexedVec<Height, StoredF64>,
|
||||
pub height_to_first_emptyoutputindex: IndexedVec<Height, EmptyOutputIndex>,
|
||||
pub height_to_first_inputindex: IndexedVec<Height, InputIndex>,
|
||||
pub height_to_first_opreturnindex: IndexedVec<Height, OpReturnIndex>,
|
||||
pub height_to_first_outputindex: IndexedVec<Height, OutputIndex>,
|
||||
pub height_to_first_p2aaddressindex: IndexedVec<Height, P2AAddressIndex>,
|
||||
pub height_to_first_p2msoutputindex: IndexedVec<Height, P2MSOutputIndex>,
|
||||
pub height_to_first_p2pk33addressindex: IndexedVec<Height, P2PK33AddressIndex>,
|
||||
pub height_to_first_p2pk65addressindex: IndexedVec<Height, P2PK65AddressIndex>,
|
||||
pub height_to_first_p2pkhaddressindex: IndexedVec<Height, P2PKHAddressIndex>,
|
||||
pub height_to_first_p2shaddressindex: IndexedVec<Height, P2SHAddressIndex>,
|
||||
pub height_to_first_p2traddressindex: IndexedVec<Height, P2TRAddressIndex>,
|
||||
pub height_to_first_p2wpkhaddressindex: IndexedVec<Height, P2WPKHAddressIndex>,
|
||||
pub height_to_first_p2wshaddressindex: IndexedVec<Height, P2WSHAddressIndex>,
|
||||
pub height_to_first_txindex: IndexedVec<Height, TxIndex>,
|
||||
pub height_to_first_unknownoutputindex: IndexedVec<Height, UnknownOutputIndex>,
|
||||
pub emptyoutputindex_to_txindex: StampedVec<EmptyOutputIndex, TxIndex>,
|
||||
pub height_to_blockhash: StampedVec<Height, BlockHash>,
|
||||
pub height_to_difficulty: StampedVec<Height, StoredF64>,
|
||||
pub height_to_first_emptyoutputindex: StampedVec<Height, EmptyOutputIndex>,
|
||||
pub height_to_first_inputindex: StampedVec<Height, InputIndex>,
|
||||
pub height_to_first_opreturnindex: StampedVec<Height, OpReturnIndex>,
|
||||
pub height_to_first_outputindex: StampedVec<Height, OutputIndex>,
|
||||
pub height_to_first_p2aaddressindex: StampedVec<Height, P2AAddressIndex>,
|
||||
pub height_to_first_p2msoutputindex: StampedVec<Height, P2MSOutputIndex>,
|
||||
pub height_to_first_p2pk33addressindex: StampedVec<Height, P2PK33AddressIndex>,
|
||||
pub height_to_first_p2pk65addressindex: StampedVec<Height, P2PK65AddressIndex>,
|
||||
pub height_to_first_p2pkhaddressindex: StampedVec<Height, P2PKHAddressIndex>,
|
||||
pub height_to_first_p2shaddressindex: StampedVec<Height, P2SHAddressIndex>,
|
||||
pub height_to_first_p2traddressindex: StampedVec<Height, P2TRAddressIndex>,
|
||||
pub height_to_first_p2wpkhaddressindex: StampedVec<Height, P2WPKHAddressIndex>,
|
||||
pub height_to_first_p2wshaddressindex: StampedVec<Height, P2WSHAddressIndex>,
|
||||
pub height_to_first_txindex: StampedVec<Height, TxIndex>,
|
||||
pub height_to_first_unknownoutputindex: StampedVec<Height, UnknownOutputIndex>,
|
||||
/// Doesn't guarantee continuity due to possible reorgs
|
||||
pub height_to_timestamp: IndexedVec<Height, Timestamp>,
|
||||
pub height_to_total_size: IndexedVec<Height, StoredUsize>,
|
||||
pub height_to_weight: IndexedVec<Height, Weight>,
|
||||
pub height_to_timestamp: StampedVec<Height, Timestamp>,
|
||||
pub height_to_total_size: StampedVec<Height, StoredUsize>,
|
||||
pub height_to_weight: StampedVec<Height, Weight>,
|
||||
/// If outputindex == Outputindex::MAX then it's coinbase
|
||||
pub inputindex_to_outputindex: IndexedVec<InputIndex, OutputIndex>,
|
||||
pub opreturnindex_to_txindex: IndexedVec<OpReturnIndex, TxIndex>,
|
||||
pub outputindex_to_outputtype: IndexedVec<OutputIndex, OutputType>,
|
||||
pub outputindex_to_typeindex: IndexedVec<OutputIndex, TypeIndex>,
|
||||
pub outputindex_to_value: IndexedVec<OutputIndex, Sats>,
|
||||
pub p2aaddressindex_to_p2abytes: IndexedVec<P2AAddressIndex, P2ABytes>,
|
||||
pub p2msoutputindex_to_txindex: IndexedVec<P2MSOutputIndex, TxIndex>,
|
||||
pub p2pk33addressindex_to_p2pk33bytes: IndexedVec<P2PK33AddressIndex, P2PK33Bytes>,
|
||||
pub p2pk65addressindex_to_p2pk65bytes: IndexedVec<P2PK65AddressIndex, P2PK65Bytes>,
|
||||
pub p2pkhaddressindex_to_p2pkhbytes: IndexedVec<P2PKHAddressIndex, P2PKHBytes>,
|
||||
pub p2shaddressindex_to_p2shbytes: IndexedVec<P2SHAddressIndex, P2SHBytes>,
|
||||
pub p2traddressindex_to_p2trbytes: IndexedVec<P2TRAddressIndex, P2TRBytes>,
|
||||
pub p2wpkhaddressindex_to_p2wpkhbytes: IndexedVec<P2WPKHAddressIndex, P2WPKHBytes>,
|
||||
pub p2wshaddressindex_to_p2wshbytes: IndexedVec<P2WSHAddressIndex, P2WSHBytes>,
|
||||
pub txindex_to_base_size: IndexedVec<TxIndex, StoredU32>,
|
||||
pub txindex_to_first_inputindex: IndexedVec<TxIndex, InputIndex>,
|
||||
pub txindex_to_first_outputindex: IndexedVec<TxIndex, OutputIndex>,
|
||||
pub txindex_to_is_explicitly_rbf: IndexedVec<TxIndex, bool>,
|
||||
pub txindex_to_rawlocktime: IndexedVec<TxIndex, RawLockTime>,
|
||||
pub txindex_to_total_size: IndexedVec<TxIndex, StoredU32>,
|
||||
pub txindex_to_txid: IndexedVec<TxIndex, Txid>,
|
||||
pub txindex_to_txversion: IndexedVec<TxIndex, TxVersion>,
|
||||
pub unknownoutputindex_to_txindex: IndexedVec<UnknownOutputIndex, TxIndex>,
|
||||
pub inputindex_to_outputindex: StampedVec<InputIndex, OutputIndex>,
|
||||
pub opreturnindex_to_txindex: StampedVec<OpReturnIndex, TxIndex>,
|
||||
pub outputindex_to_outputtype: StampedVec<OutputIndex, OutputType>,
|
||||
pub outputindex_to_typeindex: StampedVec<OutputIndex, TypeIndex>,
|
||||
pub outputindex_to_value: StampedVec<OutputIndex, Sats>,
|
||||
pub p2aaddressindex_to_p2abytes: StampedVec<P2AAddressIndex, P2ABytes>,
|
||||
pub p2msoutputindex_to_txindex: StampedVec<P2MSOutputIndex, TxIndex>,
|
||||
pub p2pk33addressindex_to_p2pk33bytes: StampedVec<P2PK33AddressIndex, P2PK33Bytes>,
|
||||
pub p2pk65addressindex_to_p2pk65bytes: StampedVec<P2PK65AddressIndex, P2PK65Bytes>,
|
||||
pub p2pkhaddressindex_to_p2pkhbytes: StampedVec<P2PKHAddressIndex, P2PKHBytes>,
|
||||
pub p2shaddressindex_to_p2shbytes: StampedVec<P2SHAddressIndex, P2SHBytes>,
|
||||
pub p2traddressindex_to_p2trbytes: StampedVec<P2TRAddressIndex, P2TRBytes>,
|
||||
pub p2wpkhaddressindex_to_p2wpkhbytes: StampedVec<P2WPKHAddressIndex, P2WPKHBytes>,
|
||||
pub p2wshaddressindex_to_p2wshbytes: StampedVec<P2WSHAddressIndex, P2WSHBytes>,
|
||||
pub txindex_to_base_size: StampedVec<TxIndex, StoredU32>,
|
||||
pub txindex_to_first_inputindex: StampedVec<TxIndex, InputIndex>,
|
||||
pub txindex_to_first_outputindex: StampedVec<TxIndex, OutputIndex>,
|
||||
pub txindex_to_is_explicitly_rbf: StampedVec<TxIndex, bool>,
|
||||
pub txindex_to_rawlocktime: StampedVec<TxIndex, RawLockTime>,
|
||||
pub txindex_to_total_size: StampedVec<TxIndex, StoredU32>,
|
||||
pub txindex_to_txid: StampedVec<TxIndex, Txid>,
|
||||
pub txindex_to_txversion: StampedVec<TxIndex, TxVersion>,
|
||||
pub unknownoutputindex_to_txindex: StampedVec<UnknownOutputIndex, TxIndex>,
|
||||
}
|
||||
|
||||
impl Vecs {
|
||||
pub fn forced_import(path: &Path, version: Version) -> color_eyre::Result<Self> {
|
||||
pub fn forced_import(file: &Arc<File>, version: Version) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
emptyoutputindex_to_txindex: IndexedVec::forced_import(
|
||||
path,
|
||||
emptyoutputindex_to_txindex: StampedVec::forced_import(
|
||||
file,
|
||||
"txindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_blockhash: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_blockhash: StampedVec::forced_import(
|
||||
file,
|
||||
"blockhash",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_difficulty: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_difficulty: StampedVec::forced_import(
|
||||
file,
|
||||
"difficulty",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_emptyoutputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_emptyoutputindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_emptyoutputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_inputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_inputindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_inputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_opreturnindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_opreturnindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_opreturnindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_outputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_outputindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_outputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2aaddressindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_p2aaddressindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_p2aaddressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2msoutputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_p2msoutputindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_p2msoutputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2pk33addressindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_p2pk33addressindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_p2pk33addressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2pk65addressindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_p2pk65addressindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_p2pk65addressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2pkhaddressindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_p2pkhaddressindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_p2pkhaddressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2shaddressindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_p2shaddressindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_p2shaddressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2traddressindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_p2traddressindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_p2traddressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2wpkhaddressindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_p2wpkhaddressindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_p2wpkhaddressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_p2wshaddressindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_p2wshaddressindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_p2wshaddressindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_txindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_txindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_txindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_first_unknownoutputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_first_unknownoutputindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_unknownoutputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_timestamp: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_timestamp: StampedVec::forced_import(
|
||||
file,
|
||||
"timestamp",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_total_size: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_total_size: StampedVec::forced_import(
|
||||
file,
|
||||
"total_size",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
height_to_weight: IndexedVec::forced_import(
|
||||
path,
|
||||
height_to_weight: StampedVec::forced_import(
|
||||
file,
|
||||
"weight",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
inputindex_to_outputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
inputindex_to_outputindex: StampedVec::forced_import(
|
||||
file,
|
||||
"outputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
opreturnindex_to_txindex: IndexedVec::forced_import(
|
||||
path,
|
||||
opreturnindex_to_txindex: StampedVec::forced_import(
|
||||
file,
|
||||
"txindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
outputindex_to_outputtype: IndexedVec::forced_import(
|
||||
path,
|
||||
outputindex_to_outputtype: StampedVec::forced_import(
|
||||
file,
|
||||
"outputtype",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
outputindex_to_typeindex: IndexedVec::forced_import(
|
||||
path,
|
||||
outputindex_to_typeindex: StampedVec::forced_import(
|
||||
file,
|
||||
"typeindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
outputindex_to_value: IndexedVec::forced_import(
|
||||
path,
|
||||
outputindex_to_value: StampedVec::forced_import(
|
||||
file,
|
||||
"value",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
p2aaddressindex_to_p2abytes: IndexedVec::forced_import(
|
||||
path,
|
||||
p2aaddressindex_to_p2abytes: StampedVec::forced_import(
|
||||
file,
|
||||
"p2abytes",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
p2msoutputindex_to_txindex: IndexedVec::forced_import(
|
||||
path,
|
||||
p2msoutputindex_to_txindex: StampedVec::forced_import(
|
||||
file,
|
||||
"txindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
p2pk33addressindex_to_p2pk33bytes: IndexedVec::forced_import(
|
||||
path,
|
||||
p2pk33addressindex_to_p2pk33bytes: StampedVec::forced_import(
|
||||
file,
|
||||
"p2pk33bytes",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
p2pk65addressindex_to_p2pk65bytes: IndexedVec::forced_import(
|
||||
path,
|
||||
p2pk65addressindex_to_p2pk65bytes: StampedVec::forced_import(
|
||||
file,
|
||||
"p2pk65bytes",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
p2pkhaddressindex_to_p2pkhbytes: IndexedVec::forced_import(
|
||||
path,
|
||||
p2pkhaddressindex_to_p2pkhbytes: StampedVec::forced_import(
|
||||
file,
|
||||
"p2pkhbytes",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
p2shaddressindex_to_p2shbytes: IndexedVec::forced_import(
|
||||
path,
|
||||
p2shaddressindex_to_p2shbytes: StampedVec::forced_import(
|
||||
file,
|
||||
"p2shbytes",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
p2traddressindex_to_p2trbytes: IndexedVec::forced_import(
|
||||
path,
|
||||
p2traddressindex_to_p2trbytes: StampedVec::forced_import(
|
||||
file,
|
||||
"p2trbytes",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
p2wpkhaddressindex_to_p2wpkhbytes: IndexedVec::forced_import(
|
||||
path,
|
||||
p2wpkhaddressindex_to_p2wpkhbytes: StampedVec::forced_import(
|
||||
file,
|
||||
"p2wpkhbytes",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
p2wshaddressindex_to_p2wshbytes: IndexedVec::forced_import(
|
||||
path,
|
||||
p2wshaddressindex_to_p2wshbytes: StampedVec::forced_import(
|
||||
file,
|
||||
"p2wshbytes",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_base_size: IndexedVec::forced_import(
|
||||
path,
|
||||
txindex_to_base_size: StampedVec::forced_import(
|
||||
file,
|
||||
"base_size",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_first_inputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
txindex_to_first_inputindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_inputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_first_outputindex: IndexedVec::forced_import(
|
||||
path,
|
||||
txindex_to_first_outputindex: StampedVec::forced_import(
|
||||
file,
|
||||
"first_outputindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_is_explicitly_rbf: IndexedVec::forced_import(
|
||||
path,
|
||||
txindex_to_is_explicitly_rbf: StampedVec::forced_import(
|
||||
file,
|
||||
"is_explicitly_rbf",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_rawlocktime: IndexedVec::forced_import(
|
||||
path,
|
||||
txindex_to_rawlocktime: StampedVec::forced_import(
|
||||
file,
|
||||
"rawlocktime",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_total_size: IndexedVec::forced_import(
|
||||
path,
|
||||
txindex_to_total_size: StampedVec::forced_import(
|
||||
file,
|
||||
"total_size",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_txid: IndexedVec::forced_import(
|
||||
path,
|
||||
txindex_to_txid: StampedVec::forced_import(
|
||||
file,
|
||||
"txid",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
txindex_to_txversion: IndexedVec::forced_import(
|
||||
path,
|
||||
txindex_to_txversion: StampedVec::forced_import(
|
||||
file,
|
||||
"txversion",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
)?,
|
||||
unknownoutputindex_to_txindex: IndexedVec::forced_import(
|
||||
path,
|
||||
unknownoutputindex_to_txindex: StampedVec::forced_import(
|
||||
file,
|
||||
"txindex",
|
||||
version + VERSION + Version::ZERO,
|
||||
Format::Raw,
|
||||
@@ -357,94 +357,92 @@ impl Vecs {
|
||||
unknownoutputindex,
|
||||
} = starting_indexes;
|
||||
|
||||
let stamp = u64::from(saved_height).into();
|
||||
|
||||
self.emptyoutputindex_to_txindex
|
||||
.truncate_if_needed(emptyoutputindex, saved_height)?;
|
||||
self.height_to_blockhash
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(emptyoutputindex, stamp)?;
|
||||
self.height_to_blockhash.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_difficulty
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_emptyoutputindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_inputindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_opreturnindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_outputindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_p2aaddressindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_p2msoutputindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_p2pk33addressindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_p2pk65addressindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_p2pkhaddressindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_p2shaddressindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_p2traddressindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_p2wpkhaddressindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_p2wshaddressindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_txindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_first_unknownoutputindex
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
self.height_to_timestamp
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_timestamp.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_total_size
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
self.height_to_weight
|
||||
.truncate_if_needed(height, saved_height)?;
|
||||
.truncate_if_needed(height, stamp)?;
|
||||
self.height_to_weight.truncate_if_needed(height, stamp)?;
|
||||
self.inputindex_to_outputindex
|
||||
.truncate_if_needed(inputindex, saved_height)?;
|
||||
.truncate_if_needed(inputindex, stamp)?;
|
||||
self.opreturnindex_to_txindex
|
||||
.truncate_if_needed(opreturnindex, saved_height)?;
|
||||
.truncate_if_needed(opreturnindex, stamp)?;
|
||||
self.outputindex_to_outputtype
|
||||
.truncate_if_needed(outputindex, saved_height)?;
|
||||
.truncate_if_needed(outputindex, stamp)?;
|
||||
self.outputindex_to_typeindex
|
||||
.truncate_if_needed(outputindex, saved_height)?;
|
||||
.truncate_if_needed(outputindex, stamp)?;
|
||||
self.outputindex_to_value
|
||||
.truncate_if_needed(outputindex, saved_height)?;
|
||||
.truncate_if_needed(outputindex, stamp)?;
|
||||
self.p2aaddressindex_to_p2abytes
|
||||
.truncate_if_needed(p2aaddressindex, saved_height)?;
|
||||
.truncate_if_needed(p2aaddressindex, stamp)?;
|
||||
self.p2msoutputindex_to_txindex
|
||||
.truncate_if_needed(p2msoutputindex, saved_height)?;
|
||||
.truncate_if_needed(p2msoutputindex, stamp)?;
|
||||
self.p2pk33addressindex_to_p2pk33bytes
|
||||
.truncate_if_needed(p2pk33addressindex, saved_height)?;
|
||||
.truncate_if_needed(p2pk33addressindex, stamp)?;
|
||||
self.p2pk65addressindex_to_p2pk65bytes
|
||||
.truncate_if_needed(p2pk65addressindex, saved_height)?;
|
||||
.truncate_if_needed(p2pk65addressindex, stamp)?;
|
||||
self.p2pkhaddressindex_to_p2pkhbytes
|
||||
.truncate_if_needed(p2pkhaddressindex, saved_height)?;
|
||||
.truncate_if_needed(p2pkhaddressindex, stamp)?;
|
||||
self.p2shaddressindex_to_p2shbytes
|
||||
.truncate_if_needed(p2shaddressindex, saved_height)?;
|
||||
.truncate_if_needed(p2shaddressindex, stamp)?;
|
||||
self.p2traddressindex_to_p2trbytes
|
||||
.truncate_if_needed(p2traddressindex, saved_height)?;
|
||||
.truncate_if_needed(p2traddressindex, stamp)?;
|
||||
self.p2wpkhaddressindex_to_p2wpkhbytes
|
||||
.truncate_if_needed(p2wpkhaddressindex, saved_height)?;
|
||||
.truncate_if_needed(p2wpkhaddressindex, stamp)?;
|
||||
self.p2wshaddressindex_to_p2wshbytes
|
||||
.truncate_if_needed(p2wshaddressindex, saved_height)?;
|
||||
.truncate_if_needed(p2wshaddressindex, stamp)?;
|
||||
self.txindex_to_base_size
|
||||
.truncate_if_needed(txindex, saved_height)?;
|
||||
.truncate_if_needed(txindex, stamp)?;
|
||||
self.txindex_to_first_inputindex
|
||||
.truncate_if_needed(txindex, saved_height)?;
|
||||
.truncate_if_needed(txindex, stamp)?;
|
||||
self.txindex_to_first_outputindex
|
||||
.truncate_if_needed(txindex, saved_height)?;
|
||||
.truncate_if_needed(txindex, stamp)?;
|
||||
self.txindex_to_is_explicitly_rbf
|
||||
.truncate_if_needed(txindex, saved_height)?;
|
||||
.truncate_if_needed(txindex, stamp)?;
|
||||
self.txindex_to_rawlocktime
|
||||
.truncate_if_needed(txindex, saved_height)?;
|
||||
.truncate_if_needed(txindex, stamp)?;
|
||||
self.txindex_to_total_size
|
||||
.truncate_if_needed(txindex, saved_height)?;
|
||||
self.txindex_to_txid
|
||||
.truncate_if_needed(txindex, saved_height)?;
|
||||
.truncate_if_needed(txindex, stamp)?;
|
||||
self.txindex_to_txid.truncate_if_needed(txindex, stamp)?;
|
||||
self.txindex_to_txversion
|
||||
.truncate_if_needed(txindex, saved_height)?;
|
||||
.truncate_if_needed(txindex, stamp)?;
|
||||
self.unknownoutputindex_to_txindex
|
||||
.truncate_if_needed(unknownoutputindex, saved_height)?;
|
||||
.truncate_if_needed(unknownoutputindex, stamp)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -481,14 +479,14 @@ impl Vecs {
|
||||
pub fn flush(&mut self, height: Height) -> Result<()> {
|
||||
self.mut_vecs()
|
||||
.into_par_iter()
|
||||
.try_for_each(|vec| vec.flush(height))
|
||||
.try_for_each(|vec| vec.flush(u64::from(height).into()))
|
||||
}
|
||||
|
||||
pub fn starting_height(&mut self) -> Height {
|
||||
self.mut_vecs()
|
||||
.into_iter()
|
||||
.map(|vec| {
|
||||
let h = vec.height();
|
||||
let h = Height::from(u64::from(vec.stamp()));
|
||||
if h > Height::ZERO { h.incremented() } else { h }
|
||||
})
|
||||
.min()
|
||||
@@ -544,7 +542,7 @@ impl Vecs {
|
||||
]
|
||||
}
|
||||
|
||||
fn mut_vecs(&mut self) -> Vec<&mut dyn AnyIndexedVec> {
|
||||
fn mut_vecs(&mut self) -> Vec<&mut dyn AnyStampedVec> {
|
||||
vec![
|
||||
&mut self.emptyoutputindex_to_txindex,
|
||||
&mut self.height_to_blockhash,
|
||||
|
||||
@@ -11,7 +11,7 @@ repository.workspace = true
|
||||
brk_core = { workspace = true }
|
||||
brk_computer = { workspace = true }
|
||||
brk_indexer = { workspace = true }
|
||||
brk_vec = { workspace = true }
|
||||
brk_vecs = { workspace = true }
|
||||
color-eyre = { workspace = true }
|
||||
derive_deref = { workspace = true }
|
||||
rmcp = { workspace = true }
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::path::Path;
|
||||
use brk_computer::Computer;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_interface::{Index, Interface, Params, ParamsOpt};
|
||||
use brk_vec::{Computation, Format};
|
||||
use brk_vecs::{Computation, Format};
|
||||
|
||||
pub fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
@@ -8,7 +8,7 @@ use std::collections::BTreeMap;
|
||||
use brk_computer::Computer;
|
||||
use brk_core::{Height, Result};
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::{AnyCollectableVec, AnyIndexedVec};
|
||||
use brk_vecs::{AnyCollectableVec, AnyStampedVec};
|
||||
use tabled::settings::Style;
|
||||
|
||||
mod deser;
|
||||
@@ -47,7 +47,7 @@ impl<'a> Interface<'a> {
|
||||
}
|
||||
|
||||
pub fn get_height(&self) -> Height {
|
||||
self._indexer.vecs.height_to_blockhash.height()
|
||||
Height::from(u64::from(self._indexer.vecs.height_to_blockhash.stamp()))
|
||||
}
|
||||
|
||||
pub fn search(&self, params: &Params) -> Vec<(String, &&dyn AnyCollectableVec)> {
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
|
||||
|
||||
use brk_computer::Computer;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_vec::AnyCollectableVec;
|
||||
use brk_vecs::AnyCollectableVec;
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use crate::pagination::{PaginatedIndexParam, PaginationParam};
|
||||
|
||||
@@ -20,7 +20,7 @@ brk_interface = { workspace = true }
|
||||
brk_logger = { workspace = true }
|
||||
brk_mcp = { workspace = true }
|
||||
brk_parser = { workspace = true }
|
||||
brk_vec = { workspace = true }
|
||||
brk_vecs = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
clap_derive = { workspace = true }
|
||||
color-eyre = { workspace = true }
|
||||
|
||||
@@ -8,7 +8,7 @@ use brk_fetcher::Fetcher;
|
||||
use brk_indexer::Indexer;
|
||||
use brk_parser::Parser;
|
||||
use brk_server::{Server, Website};
|
||||
use brk_vec::{Computation, Format};
|
||||
use brk_vecs::{Computation, Format};
|
||||
|
||||
pub fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
/vec
|
||||
_lib.rs
|
||||
Generated
-25
@@ -1,25 +0,0 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.169"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
|
||||
|
||||
[[package]]
|
||||
name = "memmap2"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "storable_vec"
|
||||
version = "0.1.2"
|
||||
dependencies = [
|
||||
"memmap2",
|
||||
]
|
||||
@@ -1,38 +0,0 @@
|
||||
# BRK Vec
|
||||
|
||||
<p align="left">
|
||||
<a href="https://github.com/bitcoinresearchkit/brk">
|
||||
<img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/bitcoinresearchkit/brk?style=social">
|
||||
</a>
|
||||
<a href="https://github.com/bitcoinresearchkit/brk/blob/main/LICENSE.md">
|
||||
<img src="https://img.shields.io/crates/l/brk" alt="License" />
|
||||
</a>
|
||||
<a href="https://crates.io/crates/brk_vec">
|
||||
<img src="https://img.shields.io/crates/v/brk_vec" alt="Version" />
|
||||
</a>
|
||||
<a href="https://docs.rs/brk_vec">
|
||||
<img src="https://img.shields.io/docsrs/brk_vec" alt="Documentation" />
|
||||
</a>
|
||||
<img src="https://img.shields.io/crates/size/brk_vec" alt="Size" />
|
||||
<a href="https://deps.rs/crate/brk_vec">
|
||||
<img src="https://deps.rs/crate/brk_vec/latest/status.svg" alt="Dependency status">
|
||||
</a>
|
||||
<a href="https://discord.gg/HaR3wpH3nr">
|
||||
<img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
|
||||
</a>
|
||||
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
|
||||
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
|
||||
</a>
|
||||
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
|
||||
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
|
||||
</a>
|
||||
<a href="https://x.com/brkdotorg">
|
||||
<img src="https://img.shields.io/badge/x.com-black" alt="X" />
|
||||
</a>
|
||||
</p>
|
||||
|
||||
A `Vec` (an array) that is stored on disk and thus which can be much larger than the available RAM.
|
||||
|
||||
Compared to a key/value store, the data stored is raw byte interpretation of the Vec's values without any overhead which is very efficient. Additionally it uses close to no RAM when caching isn't active and up to 100 MB when it is.
|
||||
|
||||
Compression is also available and built on top [`zstd`](https://crates.io/crates/zstd) to save even more space (from 0 to 75%). The tradeoff being slower reading speeds, especially random reading speeds. This is due to the data being stored in compressed pages of 16 KB, which means that if you to read even one value in that page you have to uncompress the whole page.
|
||||
@@ -1,13 +0,0 @@
|
||||
#![doc = include_str!("../README.md")]
|
||||
#![doc = "\n## Example\n\n```rust"]
|
||||
#![doc = include_str!("../examples/main.rs")]
|
||||
#![doc = "```"]
|
||||
|
||||
mod structs;
|
||||
mod traits;
|
||||
mod variants;
|
||||
|
||||
pub use memmap2::Mmap;
|
||||
pub use structs::*;
|
||||
pub use traits::*;
|
||||
pub use variants::*;
|
||||
@@ -1,71 +0,0 @@
|
||||
use std::{
|
||||
fs,
|
||||
io::{self, Read},
|
||||
ops::{AddAssign, Deref, DerefMut},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
use brk_core::{Error, Result};
|
||||
use zerocopy::{FromBytes, IntoBytes};
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
#[derive(
|
||||
Debug,
|
||||
Default,
|
||||
Clone,
|
||||
Copy,
|
||||
PartialEq,
|
||||
Eq,
|
||||
PartialOrd,
|
||||
Ord,
|
||||
FromBytes,
|
||||
IntoBytes,
|
||||
Immutable,
|
||||
KnownLayout,
|
||||
)]
|
||||
pub struct Length(usize);
|
||||
|
||||
impl Length {
|
||||
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
|
||||
fs::write(path, self.as_bytes())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<usize> for Length {
|
||||
fn from(value: usize) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for Length {
|
||||
type Target = usize;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for Length {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<&Path> for Length {
|
||||
type Error = Error;
|
||||
fn try_from(value: &Path) -> Result<Self, Self::Error> {
|
||||
let mut buf = [0; 8];
|
||||
if let Ok(bytes) = fs::read(value) {
|
||||
bytes.as_slice().read_exact(&mut buf)?;
|
||||
Ok(*(Self::ref_from_bytes(&buf)?))
|
||||
} else {
|
||||
Ok(Self::default())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AddAssign<usize> for Length {
|
||||
fn add_assign(&mut self, rhs: usize) {
|
||||
self.0 += rhs;
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
mod compressed_page_meta;
|
||||
mod compressed_pages_meta;
|
||||
mod format;
|
||||
mod header;
|
||||
// mod length;
|
||||
mod unsafe_slice;
|
||||
|
||||
pub use compressed_page_meta::*;
|
||||
pub use compressed_pages_meta::*;
|
||||
pub use format::*;
|
||||
pub use header::*;
|
||||
// pub use length::*;
|
||||
pub use unsafe_slice::*;
|
||||
@@ -0,0 +1,2 @@
|
||||
/vecs
|
||||
/raw
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "brk_vec"
|
||||
description = "A storeable vec"
|
||||
name = "brk_vecs"
|
||||
description = "A simple index/value store"
|
||||
keywords = ["vec", "disk", "data"]
|
||||
categories = ["database"]
|
||||
version.workspace = true
|
||||
@@ -10,13 +10,14 @@ homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
arc-swap = { workspace = true }
|
||||
brk_core = { workspace = true }
|
||||
brk_exit = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
clap_derive = { workspace = true }
|
||||
libc = "0.2.174"
|
||||
log = { workspace = true }
|
||||
memmap2 = "0.9.7"
|
||||
parking_lot = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
@@ -0,0 +1,423 @@
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use brk_core::Result;
|
||||
use brk_vecs::{File, PAGE_SIZE};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let _ = fs::remove_dir_all("vecs");
|
||||
|
||||
let file = File::open(Path::new("vecs"))?;
|
||||
|
||||
// let file_min_len = PAGE_SIZE * 1_000_000;
|
||||
// let min_regions = 20_000;
|
||||
|
||||
// file.set_min_len(file_min_len)?;
|
||||
// file.set_min_regions(min_regions)?;
|
||||
|
||||
let (region1_i, _) = file.create_region_if_needed("region1")?;
|
||||
|
||||
{
|
||||
let layout = file.layout();
|
||||
assert!(layout.start_to_index().len() == 1);
|
||||
assert!(layout.start_to_index().first_key_value() == Some((&0, &0)));
|
||||
assert!(layout.start_to_hole().is_empty());
|
||||
|
||||
let regions = file.regions();
|
||||
assert!(
|
||||
regions
|
||||
.get_region_index_from_id("region1")
|
||||
.is_some_and(|i| i == region1_i)
|
||||
);
|
||||
|
||||
let region = file.get_region(region1_i.into())?;
|
||||
assert!(region.start() == 0);
|
||||
assert!(region.len() == 0);
|
||||
assert!(region.reserved() == PAGE_SIZE);
|
||||
}
|
||||
|
||||
file.write_all_to_region(region1_i.into(), &[0, 1, 2, 3, 4])?;
|
||||
|
||||
{
|
||||
let region = file.get_region(region1_i.into())?;
|
||||
assert!(region.start() == 0);
|
||||
assert!(region.len() == 5);
|
||||
assert!(region.reserved() == PAGE_SIZE);
|
||||
|
||||
assert!(file.mmap()[0..10] == [0, 1, 2, 3, 4, 0, 0, 0, 0, 0]);
|
||||
}
|
||||
|
||||
file.write_all_to_region(region1_i.into(), &[5, 6, 7, 8, 9])?;
|
||||
|
||||
{
|
||||
let region = file.get_region(region1_i.into())?;
|
||||
assert!(region.start() == 0);
|
||||
assert!(region.len() == 10);
|
||||
assert!(region.reserved() == PAGE_SIZE);
|
||||
|
||||
assert!(file.mmap()[0..10] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
|
||||
}
|
||||
|
||||
file.write_all_to_region_at(region1_i.into(), &[1, 2], 0)?;
|
||||
|
||||
{
|
||||
let region = file.get_region(region1_i.into())?;
|
||||
assert!(region.start() == 0);
|
||||
assert!(region.len() == 10);
|
||||
assert!(region.reserved() == PAGE_SIZE);
|
||||
|
||||
assert!(file.mmap()[0..10] == [1, 2, 2, 3, 4, 5, 6, 7, 8, 9]);
|
||||
}
|
||||
|
||||
file.write_all_to_region_at(region1_i.into(), &[10, 11, 12, 13, 14, 15, 16, 17, 18], 4)?;
|
||||
|
||||
{
|
||||
let region = file.get_region(region1_i.into())?;
|
||||
assert!(region.start() == 0);
|
||||
assert!(region.len() == 13);
|
||||
assert!(region.reserved() == PAGE_SIZE);
|
||||
|
||||
assert!(
|
||||
file.mmap()[0..20]
|
||||
== [
|
||||
1, 2, 2, 3, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, 0, 0, 0, 0, 0, 0
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
file.write_all_to_region_at(region1_i.into(), &[1], 18)?;
|
||||
|
||||
{
|
||||
let region = file.get_region(region1_i.into())?;
|
||||
assert!(region.start() == 0);
|
||||
assert!(region.len() == 19);
|
||||
assert!(region.reserved() == PAGE_SIZE);
|
||||
|
||||
assert!(
|
||||
file.mmap()[0..20]
|
||||
== [
|
||||
1, 2, 2, 3, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, 0, 0, 0, 0, 1, 0
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
file.write_all_to_region_at(region1_i.into(), &[1; 8000], 0)?;
|
||||
|
||||
{
|
||||
let region = file.get_region(region1_i.into())?;
|
||||
assert!(region.start() == 0);
|
||||
assert!(region.len() == 8000);
|
||||
assert!(region.reserved() == PAGE_SIZE * 2);
|
||||
|
||||
assert!(file.mmap()[0..8000] == [1; 8000]);
|
||||
assert!(file.mmap()[8000..8001] == [0]);
|
||||
}
|
||||
|
||||
println!("Disk usage - pre sync: {}", file.disk_usage());
|
||||
file.flush()?;
|
||||
println!("Disk usage - post sync: {}", file.disk_usage());
|
||||
|
||||
file.truncate_region(region1_i.into(), 10)?;
|
||||
|
||||
{
|
||||
let region = file.get_region(region1_i.into())?;
|
||||
assert!(region.start() == 0);
|
||||
assert!(region.len() == 10);
|
||||
assert!(region.reserved() == PAGE_SIZE * 2);
|
||||
// We only punch a hole in whole pages (4096 bytes)
|
||||
// Thus the last byte of the page where the is still data wasn't overwritten when truncating
|
||||
// And the first byte of the punched page was set to 0
|
||||
assert!(file.mmap()[4095..=4096] == [1, 0]);
|
||||
}
|
||||
|
||||
file.flush()?;
|
||||
println!("Disk usage - post trunc: {}", file.disk_usage());
|
||||
|
||||
file.remove_region(region1_i.into())?;
|
||||
|
||||
file.flush()?;
|
||||
|
||||
println!("Disk usage - post remove: {}", file.disk_usage());
|
||||
|
||||
{
|
||||
let regions = file.regions();
|
||||
let index_to_region = regions.index_to_region();
|
||||
assert!(index_to_region.len() == 1);
|
||||
assert!(index_to_region[0].is_none());
|
||||
assert!(regions.id_to_index().is_empty());
|
||||
|
||||
let layout = file.layout();
|
||||
assert!(layout.start_to_index().is_empty());
|
||||
assert!(layout.start_to_hole().is_empty());
|
||||
}
|
||||
|
||||
let (region1_i, _) = file.create_region_if_needed("region1")?;
|
||||
let (region2_i, _) = file.create_region_if_needed("region2")?;
|
||||
let (region3_i, _) = file.create_region_if_needed("region3")?;
|
||||
|
||||
// dbg!(file.layout());
|
||||
|
||||
{
|
||||
let regions = file.regions();
|
||||
let index_to_region = regions.index_to_region();
|
||||
assert!(index_to_region.len() == 3);
|
||||
let region1 = file.get_region(region1_i.into())?;
|
||||
assert!(region1.start() == 0);
|
||||
assert!(region1.len() == 0);
|
||||
assert!(region1.reserved() == PAGE_SIZE);
|
||||
let region2 = file.get_region(region2_i.into())?;
|
||||
assert!(region2.start() == PAGE_SIZE);
|
||||
assert!(region2.len() == 0);
|
||||
assert!(region2.reserved() == PAGE_SIZE);
|
||||
let region3 = file.get_region(region3_i.into())?;
|
||||
assert!(region3.start() == PAGE_SIZE * 2);
|
||||
assert!(region3.len() == 0);
|
||||
assert!(region3.reserved() == PAGE_SIZE);
|
||||
let id_to_index = regions.id_to_index();
|
||||
assert!(id_to_index.len() == 3);
|
||||
assert!(id_to_index.get("region1") == Some(&0));
|
||||
assert!(id_to_index.get("region2") == Some(&1));
|
||||
assert!(id_to_index.get("region3") == Some(&2));
|
||||
|
||||
let layout = file.layout();
|
||||
let start_to_index = layout.start_to_index();
|
||||
assert!(start_to_index.len() == 3);
|
||||
assert!(start_to_index.get(&0) == Some(&0));
|
||||
assert!(start_to_index.get(&PAGE_SIZE) == Some(&1));
|
||||
assert!(start_to_index.get(&(PAGE_SIZE * 2)) == Some(&2));
|
||||
assert!(layout.start_to_hole().is_empty());
|
||||
}
|
||||
|
||||
file.remove_region(region2_i.into())?;
|
||||
|
||||
{
|
||||
let regions = file.regions();
|
||||
let index_to_region = regions.index_to_region();
|
||||
assert!(index_to_region.len() == 3);
|
||||
let region1 = file.get_region(region1_i.into())?;
|
||||
assert!(region1.start() == 0);
|
||||
assert!(region1.len() == 0);
|
||||
assert!(region1.reserved() == PAGE_SIZE);
|
||||
assert!(file.get_region(region2_i.into()).is_err());
|
||||
assert!(
|
||||
index_to_region
|
||||
.get(region2_i)
|
||||
.is_some_and(|opt| opt.is_none())
|
||||
);
|
||||
let region3 = file.get_region(region3_i.into())?;
|
||||
assert!(region3.start() == PAGE_SIZE * 2);
|
||||
assert!(region3.len() == 0);
|
||||
assert!(region3.reserved() == PAGE_SIZE);
|
||||
let id_to_index = regions.id_to_index();
|
||||
assert!(id_to_index.len() == 2);
|
||||
assert!(id_to_index.get("region1") == Some(&0));
|
||||
assert!(id_to_index.get("region2").is_none());
|
||||
assert!(id_to_index.get("region3") == Some(&2));
|
||||
|
||||
let layout = file.layout();
|
||||
let start_to_index = layout.start_to_index();
|
||||
assert!(start_to_index.len() == 2);
|
||||
assert!(start_to_index.get(&0) == Some(®ion1_i));
|
||||
assert!(start_to_index.get(&(PAGE_SIZE * 2)) == Some(®ion3_i));
|
||||
let start_to_hole = layout.start_to_hole();
|
||||
assert!(start_to_hole.len() == 1);
|
||||
assert!(start_to_hole.get(&PAGE_SIZE) == Some(&PAGE_SIZE));
|
||||
|
||||
drop(regions);
|
||||
drop(layout);
|
||||
assert!(
|
||||
file.remove_region(region2_i.into())
|
||||
.is_ok_and(|o| o.is_none())
|
||||
);
|
||||
}
|
||||
|
||||
let (region2_i, _) = file.create_region_if_needed("region2")?;
|
||||
|
||||
{
|
||||
assert!(region2_i == 1)
|
||||
}
|
||||
|
||||
file.remove_region(region2_i.into())?;
|
||||
|
||||
{
|
||||
let regions = file.regions();
|
||||
let index_to_region = regions.index_to_region();
|
||||
assert!(index_to_region.len() == 3);
|
||||
let region1 = file.get_region(region1_i.into())?;
|
||||
assert!(region1.start() == 0);
|
||||
assert!(region1.len() == 0);
|
||||
assert!(region1.reserved() == PAGE_SIZE);
|
||||
assert!(file.get_region(region2_i.into()).is_err());
|
||||
assert!(
|
||||
index_to_region
|
||||
.get(region2_i)
|
||||
.is_some_and(|opt| opt.is_none())
|
||||
);
|
||||
let region3 = file.get_region(region3_i.into())?;
|
||||
assert!(region3.start() == PAGE_SIZE * 2);
|
||||
assert!(region3.len() == 0);
|
||||
assert!(region3.reserved() == PAGE_SIZE);
|
||||
let id_to_index = regions.id_to_index();
|
||||
assert!(id_to_index.len() == 2);
|
||||
assert!(id_to_index.get("region1") == Some(&0));
|
||||
assert!(id_to_index.get("region2").is_none());
|
||||
assert!(id_to_index.get("region3") == Some(&2));
|
||||
|
||||
let layout = file.layout();
|
||||
let start_to_index = layout.start_to_index();
|
||||
assert!(start_to_index.len() == 2);
|
||||
assert!(start_to_index.get(&0) == Some(®ion1_i));
|
||||
assert!(start_to_index.get(&(PAGE_SIZE * 2)) == Some(®ion3_i));
|
||||
let start_to_hole = layout.start_to_hole();
|
||||
assert!(start_to_hole.len() == 1);
|
||||
assert!(start_to_hole.get(&PAGE_SIZE) == Some(&PAGE_SIZE));
|
||||
|
||||
drop(regions);
|
||||
drop(layout);
|
||||
assert!(
|
||||
file.remove_region(region2_i.into())
|
||||
.is_ok_and(|o| o.is_none())
|
||||
);
|
||||
}
|
||||
|
||||
file.write_all_to_region_at(region1_i.into(), &[1; 8000], 0)?;
|
||||
|
||||
{
|
||||
let regions = file.regions();
|
||||
let index_to_region = regions.index_to_region();
|
||||
assert!(index_to_region.len() == 3);
|
||||
let region1 = file.get_region(region1_i.into())?;
|
||||
assert!(region1.start() == 0);
|
||||
assert!(region1.len() == 8000);
|
||||
assert!(region1.reserved() == 2 * PAGE_SIZE);
|
||||
assert!(file.get_region(region2_i.into()).is_err());
|
||||
assert!(
|
||||
index_to_region
|
||||
.get(region2_i)
|
||||
.is_some_and(|opt| opt.is_none())
|
||||
);
|
||||
let region3 = file.get_region(region3_i.into())?;
|
||||
assert!(region3.start() == PAGE_SIZE * 2);
|
||||
assert!(region3.len() == 0);
|
||||
assert!(region3.reserved() == PAGE_SIZE);
|
||||
let id_to_index = regions.id_to_index();
|
||||
assert!(id_to_index.len() == 2);
|
||||
assert!(id_to_index.get("region1") == Some(&0));
|
||||
assert!(id_to_index.get("region2").is_none());
|
||||
assert!(id_to_index.get("region3") == Some(&2));
|
||||
|
||||
let layout = file.layout();
|
||||
let start_to_index = layout.start_to_index();
|
||||
assert!(start_to_index.len() == 2);
|
||||
assert!(start_to_index.get(&0) == Some(®ion1_i));
|
||||
assert!(start_to_index.get(&(PAGE_SIZE * 2)) == Some(®ion3_i));
|
||||
let start_to_hole = layout.start_to_hole();
|
||||
assert!(start_to_hole.is_empty());
|
||||
}
|
||||
|
||||
let (region2_i, _) = file.create_region_if_needed("region2")?;
|
||||
|
||||
{
|
||||
let regions = file.regions();
|
||||
let index_to_region = regions.index_to_region();
|
||||
assert!(index_to_region.len() == 3);
|
||||
let region1 = file.get_region(region1_i.into())?;
|
||||
assert!(region1.start() == 0);
|
||||
assert!(region1.len() == 8000);
|
||||
assert!(region1.reserved() == 2 * PAGE_SIZE);
|
||||
let region2 = file.get_region(region2_i.into())?;
|
||||
assert!(region2.start() == PAGE_SIZE * 3);
|
||||
assert!(region2.len() == 0);
|
||||
assert!(region2.reserved() == PAGE_SIZE);
|
||||
let region3 = file.get_region(region3_i.into())?;
|
||||
assert!(region3.start() == PAGE_SIZE * 2);
|
||||
assert!(region3.len() == 0);
|
||||
assert!(region3.reserved() == PAGE_SIZE);
|
||||
let id_to_index = regions.id_to_index();
|
||||
assert!(id_to_index.len() == 3);
|
||||
assert!(id_to_index.get("region1") == Some(&0));
|
||||
assert!(id_to_index.get("region2") == Some(&1));
|
||||
assert!(id_to_index.get("region3") == Some(&2));
|
||||
|
||||
let layout = file.layout();
|
||||
let start_to_index = layout.start_to_index();
|
||||
assert!(start_to_index.len() == 3);
|
||||
assert!(start_to_index.get(&0) == Some(®ion1_i));
|
||||
assert!(start_to_index.get(&(PAGE_SIZE * 2)) == Some(®ion3_i));
|
||||
assert!(start_to_index.get(&(PAGE_SIZE * 3)) == Some(®ion2_i));
|
||||
let start_to_hole = layout.start_to_hole();
|
||||
assert!(start_to_hole.is_empty());
|
||||
}
|
||||
|
||||
file.remove_region(region3_i.into())?;
|
||||
|
||||
{
|
||||
let regions = file.regions();
|
||||
let index_to_region = regions.index_to_region();
|
||||
assert!(index_to_region.len() == 3);
|
||||
let region1 = file.get_region(region1_i.into())?;
|
||||
assert!(region1.start() == 0);
|
||||
assert!(region1.len() == 8000);
|
||||
assert!(region1.reserved() == 2 * PAGE_SIZE);
|
||||
let region2 = file.get_region(region2_i.into())?;
|
||||
assert!(region2.start() == PAGE_SIZE * 3);
|
||||
assert!(region2.len() == 0);
|
||||
assert!(region2.reserved() == PAGE_SIZE);
|
||||
assert!(file.get_region(region3_i.into()).is_err());
|
||||
let id_to_index = regions.id_to_index();
|
||||
assert!(id_to_index.len() == 2);
|
||||
assert!(id_to_index.get("region1") == Some(&0));
|
||||
assert!(id_to_index.get("region2") == Some(&1));
|
||||
assert!(id_to_index.get("region3").is_none());
|
||||
|
||||
let layout = file.layout();
|
||||
let start_to_index = layout.start_to_index();
|
||||
assert!(start_to_index.len() == 2);
|
||||
assert!(start_to_index.get(&0) == Some(®ion1_i));
|
||||
assert!(start_to_index.get(&(PAGE_SIZE * 3)) == Some(®ion2_i));
|
||||
let start_to_hole = layout.start_to_hole();
|
||||
assert!(start_to_hole.get(&(PAGE_SIZE * 2)) == Some(&PAGE_SIZE));
|
||||
}
|
||||
|
||||
file.write_all_to_region(region1_i.into(), &[1; 8000])?;
|
||||
|
||||
{
|
||||
let regions = file.regions();
|
||||
let index_to_region = regions.index_to_region();
|
||||
assert!(index_to_region.len() == 3);
|
||||
let region1 = file.get_region(region1_i.into())?;
|
||||
assert!(region1.start() == PAGE_SIZE * 4);
|
||||
assert!(region1.len() == 16_000);
|
||||
assert!(region1.reserved() == 4 * PAGE_SIZE);
|
||||
let region2 = file.get_region(region2_i.into())?;
|
||||
assert!(region2.start() == PAGE_SIZE * 3);
|
||||
assert!(region2.len() == 0);
|
||||
assert!(region2.reserved() == PAGE_SIZE);
|
||||
assert!(file.get_region(region3_i.into()).is_err());
|
||||
let id_to_index = regions.id_to_index();
|
||||
assert!(id_to_index.len() == 2);
|
||||
assert!(id_to_index.get("region1") == Some(&0));
|
||||
assert!(id_to_index.get("region2") == Some(&1));
|
||||
assert!(id_to_index.get("region3").is_none());
|
||||
|
||||
let layout = file.layout();
|
||||
let start_to_index = layout.start_to_index();
|
||||
assert!(start_to_index.len() == 2);
|
||||
assert!(start_to_index.get(&(PAGE_SIZE * 4)) == Some(®ion1_i));
|
||||
assert!(start_to_index.get(&(PAGE_SIZE * 3)) == Some(®ion2_i));
|
||||
let start_to_hole = layout.start_to_hole();
|
||||
assert!(start_to_hole.get(&0) == Some(&(PAGE_SIZE * 3)));
|
||||
}
|
||||
|
||||
file.write_all_to_region(region2_i.into(), &[1; 6000])?;
|
||||
|
||||
let (region4_i, _) = file.create_region_if_needed("region4")?;
|
||||
file.remove_region(region2_i.into())?;
|
||||
file.remove_region(region4_i.into())?;
|
||||
|
||||
let regions = file.regions();
|
||||
dbg!(®ions);
|
||||
let layout = file.layout();
|
||||
dbg!(&layout);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,20 +1,22 @@
|
||||
use std::{fs, path::Path};
|
||||
use std::{fs, path::Path, sync::Arc};
|
||||
|
||||
use brk_core::{DateIndex, Height, Version};
|
||||
use brk_vec::{AnyVec, CollectableVec, Format, GenericStoredVec, StoredVec, VecIterator};
|
||||
use brk_core::{DateIndex, Version};
|
||||
use brk_vecs::{AnyVec, CollectableVec, File, GenericStoredVec, RawVec, Stamp, VecIterator};
|
||||
|
||||
type I = DateIndex;
|
||||
#[allow(clippy::upper_case_acronyms)]
|
||||
type VEC = StoredVec<I, u32>;
|
||||
type VEC = RawVec<I, u32>;
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let _ = fs::remove_dir_all("./vec");
|
||||
let _ = fs::remove_dir_all("raw");
|
||||
|
||||
let version = Version::TWO;
|
||||
let format = Format::Raw;
|
||||
// let format = Format::Raw;
|
||||
//
|
||||
let file = Arc::new(File::open(Path::new("raw"))?);
|
||||
|
||||
{
|
||||
let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
|
||||
let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?;
|
||||
|
||||
(0..21_u32).for_each(|v| {
|
||||
vec.push(v);
|
||||
@@ -26,16 +28,17 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
dbg!(iter.get(2.into()));
|
||||
dbg!(iter.get(20.into()));
|
||||
dbg!(iter.get(21.into()));
|
||||
drop(iter);
|
||||
|
||||
vec.flush()?;
|
||||
|
||||
// dbg!(vec.header());
|
||||
dbg!(vec.header());
|
||||
}
|
||||
|
||||
{
|
||||
let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
|
||||
let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?;
|
||||
|
||||
vec.mut_header().update_height(Height::new(100));
|
||||
vec.mut_header().update_stamp(Stamp::new(100));
|
||||
|
||||
let mut iter = vec.into_iter();
|
||||
dbg!(iter.get(0.into()));
|
||||
@@ -47,36 +50,38 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
dbg!(iter.get(20.into()));
|
||||
dbg!(iter.get(20.into()));
|
||||
dbg!(iter.get(0.into()));
|
||||
drop(iter);
|
||||
|
||||
vec.push(21);
|
||||
vec.push(22);
|
||||
|
||||
let mut iter = vec.into_iter();
|
||||
|
||||
dbg!(iter.get(20.into()));
|
||||
dbg!(iter.get(21.into()));
|
||||
dbg!(iter.get(22.into()));
|
||||
dbg!(iter.get(23.into()));
|
||||
drop(iter);
|
||||
|
||||
vec.flush()?;
|
||||
}
|
||||
|
||||
{
|
||||
let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
|
||||
let mut iter = vec.into_iter();
|
||||
let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?;
|
||||
|
||||
let mut iter = vec.into_iter();
|
||||
dbg!(iter.get(0.into()));
|
||||
dbg!(iter.get(20.into()));
|
||||
dbg!(iter.get(21.into()));
|
||||
dbg!(iter.get(22.into()));
|
||||
drop(iter);
|
||||
|
||||
vec.truncate_if_needed(14.into())?;
|
||||
|
||||
let mut iter = vec.into_iter();
|
||||
|
||||
iter.get(0.into());
|
||||
iter.get(5.into());
|
||||
dbg!(iter.get(0.into()));
|
||||
dbg!(iter.get(5.into()));
|
||||
dbg!(iter.get(20.into()));
|
||||
drop(iter);
|
||||
|
||||
dbg!(vec.collect_signed_range(Some(-5), None)?);
|
||||
|
||||
@@ -87,7 +92,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
}
|
||||
|
||||
{
|
||||
let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
|
||||
let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?;
|
||||
|
||||
vec.reset()?;
|
||||
|
||||
@@ -101,36 +106,43 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
dbg!(iter.get(0.into()));
|
||||
dbg!(iter.get(20.into()));
|
||||
dbg!(iter.get(21.into()));
|
||||
drop(iter);
|
||||
|
||||
let mmap = vec.create_mmap()?;
|
||||
dbg!(vec.take(10.into(), &mmap)?);
|
||||
dbg!(vec.get_or_read(10.into(), &mmap)?);
|
||||
let reader = vec.create_static_reader();
|
||||
dbg!(vec.take(10.into(), &reader)?);
|
||||
dbg!(vec.get_or_read(10.into(), &reader)?);
|
||||
dbg!(vec.holes());
|
||||
drop(reader);
|
||||
|
||||
vec.flush()?;
|
||||
dbg!(vec.holes());
|
||||
}
|
||||
|
||||
{
|
||||
let mut vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
|
||||
|
||||
let mmap = vec.create_mmap()?;
|
||||
let mut vec: VEC = RawVec::forced_import(&file, "vec", version)?;
|
||||
|
||||
dbg!(vec.holes());
|
||||
dbg!(vec.get_or_read(10.into(), &mmap)?);
|
||||
|
||||
let reader = vec.create_static_reader();
|
||||
dbg!(vec.get_or_read(10.into(), &reader)?);
|
||||
drop(reader);
|
||||
|
||||
vec.update(10.into(), 10)?;
|
||||
vec.update(0.into(), 10)?;
|
||||
|
||||
let reader = vec.create_static_reader();
|
||||
dbg!(
|
||||
vec.holes(),
|
||||
vec.get_or_read(0.into(), &mmap)?,
|
||||
vec.get_or_read(10.into(), &mmap)?
|
||||
vec.get_or_read(0.into(), &reader)?,
|
||||
vec.get_or_read(10.into(), &reader)?
|
||||
);
|
||||
drop(reader);
|
||||
|
||||
vec.flush()?;
|
||||
}
|
||||
|
||||
{
|
||||
let vec: VEC = StoredVec::forced_import(Path::new("."), "vec", version, format)?;
|
||||
let vec: VEC = RawVec::forced_import(&file, "vec", version)?;
|
||||
|
||||
dbg!(vec.collect()?);
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Identifier {
|
||||
Number(usize),
|
||||
String(String),
|
||||
}
|
||||
|
||||
impl<'a> From<&'a str> for Identifier {
|
||||
fn from(value: &'a str) -> Self {
|
||||
Self::String(value.to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for Identifier {
|
||||
fn from(value: String) -> Self {
|
||||
Self::String(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<usize> for Identifier {
|
||||
fn from(value: usize) -> Self {
|
||||
Self::Number(value)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use brk_core::Error;
|
||||
use brk_core::Result;
|
||||
|
||||
use super::{Region, Regions};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Layout {
|
||||
start_to_index: BTreeMap<u64, usize>,
|
||||
start_to_hole: BTreeMap<u64, u64>,
|
||||
}
|
||||
|
||||
impl From<&Regions> for Layout {
|
||||
fn from(value: &Regions) -> Self {
|
||||
let mut start_to_index = BTreeMap::new();
|
||||
|
||||
let index_to_region = value.index_to_region();
|
||||
|
||||
value
|
||||
.index_to_region()
|
||||
.iter()
|
||||
.enumerate()
|
||||
.flat_map(|(index, opt)| opt.as_ref().map(|region| (index, region)))
|
||||
.for_each(|(index, region)| {
|
||||
let region = region.read();
|
||||
let start = region.start();
|
||||
start_to_index.insert(start, index);
|
||||
});
|
||||
|
||||
let mut start_to_hole = BTreeMap::new();
|
||||
|
||||
let mut prev_end = 0;
|
||||
|
||||
start_to_index.iter().for_each(|(&start, &index)| {
|
||||
if prev_end != start {
|
||||
start_to_hole.insert(prev_end, start - prev_end);
|
||||
}
|
||||
let reserved = index_to_region[index].as_ref().unwrap().read().reserved();
|
||||
prev_end = start + reserved;
|
||||
});
|
||||
|
||||
Self {
|
||||
start_to_index,
|
||||
start_to_hole,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Layout {
|
||||
pub fn start_to_index(&self) -> &BTreeMap<u64, usize> {
|
||||
&self.start_to_index
|
||||
}
|
||||
|
||||
pub fn start_to_hole(&self) -> &BTreeMap<u64, u64> {
|
||||
&self.start_to_hole
|
||||
}
|
||||
|
||||
pub fn get_last_region(&self) -> Option<(u64, usize)> {
|
||||
self.start_to_index
|
||||
.last_key_value()
|
||||
.map(|(start, index)| (*start, *index))
|
||||
}
|
||||
|
||||
pub fn get_last_region_index(&self) -> Option<usize> {
|
||||
self.get_last_region().map(|(_, index)| index)
|
||||
}
|
||||
|
||||
pub fn is_last_region(&self, index: usize) -> bool {
|
||||
let last = self.get_last_region();
|
||||
let is_last = last.is_some_and(|(_, other_index)| index == other_index);
|
||||
if is_last {
|
||||
assert!(self.start_to_hole.range(last.unwrap().0..).next().is_none());
|
||||
}
|
||||
is_last
|
||||
}
|
||||
|
||||
pub fn insert_region(&mut self, start: u64, index: usize) {
|
||||
assert!(self.start_to_index.insert(start, index).is_none())
|
||||
// TODO: Other checks related to holes ?
|
||||
}
|
||||
|
||||
pub fn move_region(&mut self, new_start: u64, index: usize, region: &Region) -> Result<()> {
|
||||
self.remove_region(index, region)?;
|
||||
self.insert_region(new_start, index);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn remove_region(&mut self, index: usize, region: &Region) -> Result<()> {
|
||||
let start = region.start();
|
||||
let mut reserved = region.reserved();
|
||||
|
||||
if self
|
||||
.start_to_index
|
||||
.remove(&start)
|
||||
.is_none_or(|index_| index != index_)
|
||||
{
|
||||
return Err(Error::Str(
|
||||
"Something went wrong, indexes of removed region should be the same",
|
||||
));
|
||||
}
|
||||
|
||||
reserved += self
|
||||
.start_to_hole
|
||||
.remove(&(start + reserved))
|
||||
.unwrap_or_default();
|
||||
|
||||
if self
|
||||
.start_to_index
|
||||
.keys()
|
||||
.last()
|
||||
.is_none_or(|®ion_start| {
|
||||
self.start_to_hole
|
||||
.keys()
|
||||
.last()
|
||||
.is_some_and(|&hole_start| hole_start > region_start)
|
||||
})
|
||||
{
|
||||
self.start_to_hole.pop_last();
|
||||
} else if let Some((&hole_start, gap)) = self.start_to_hole.range_mut(..start).next_back()
|
||||
&& hole_start + *gap == start
|
||||
{
|
||||
*gap += reserved;
|
||||
} else {
|
||||
self.start_to_hole.insert(start, reserved);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_hole(&self, start: u64) -> Option<u64> {
|
||||
self.start_to_hole.get(&start).copied()
|
||||
}
|
||||
|
||||
pub fn find_smallest_adequate_hole(&self, reserved: u64) -> Option<u64> {
|
||||
self.start_to_hole
|
||||
.iter()
|
||||
.filter(|(_, gap)| **gap >= reserved)
|
||||
.map(|(start, gap)| (gap, start))
|
||||
.collect::<BTreeMap<_, _>>()
|
||||
.pop_first()
|
||||
.map(|(_, s)| *s)
|
||||
}
|
||||
|
||||
pub fn remove_or_compress_hole_to_right(&mut self, start: u64, compress_by: u64) {
|
||||
if let Some(gap) = self.start_to_hole.remove(&start)
|
||||
&& gap != compress_by
|
||||
{
|
||||
if gap > compress_by {
|
||||
self.start_to_hole
|
||||
.insert(start + compress_by, gap - compress_by);
|
||||
} else {
|
||||
panic!("Hole too small");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,517 @@
|
||||
use std::{
|
||||
fs::{self, OpenOptions},
|
||||
os::unix::io::AsRawFd,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use brk_core::{Error, Result};
|
||||
use libc::off_t;
|
||||
use log::info;
|
||||
use memmap2::{MmapMut, MmapOptions};
|
||||
use parking_lot::{RwLock, RwLockReadGuard};
|
||||
|
||||
mod identifier;
|
||||
mod layout;
|
||||
mod reader;
|
||||
mod region;
|
||||
mod regions;
|
||||
|
||||
pub use identifier::*;
|
||||
use layout::*;
|
||||
use rayon::prelude::*;
|
||||
pub use reader::*;
|
||||
pub use region::*;
|
||||
use regions::*;
|
||||
|
||||
pub const PAGE_SIZE: u64 = 4096;
|
||||
pub const PAGE_SIZE_MINUS_1: u64 = PAGE_SIZE - 1;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct File {
|
||||
path: PathBuf,
|
||||
regions: RwLock<Regions>,
|
||||
layout: RwLock<Layout>,
|
||||
file: RwLock<fs::File>,
|
||||
mmap: RwLock<MmapMut>,
|
||||
}
|
||||
|
||||
impl File {
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
let regions = Regions::open(path)?;
|
||||
|
||||
let layout = Layout::from(®ions);
|
||||
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(false)
|
||||
.open(Self::data_path_(path))?;
|
||||
|
||||
let mmap = Self::create_mmap(&file)?;
|
||||
|
||||
Ok(Self {
|
||||
path: path.to_owned(),
|
||||
file: RwLock::new(file),
|
||||
mmap: RwLock::new(mmap),
|
||||
regions: RwLock::new(regions),
|
||||
layout: RwLock::new(layout),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn file_len(&self) -> Result<u64> {
|
||||
Ok(self.file.read().metadata()?.len())
|
||||
}
|
||||
|
||||
pub fn set_min_len(&self, len: u64) -> Result<()> {
|
||||
let len = Self::ceil_number_to_page_size_multiple(len);
|
||||
|
||||
let file_len = self.file_len()?;
|
||||
if file_len < len {
|
||||
let mut mmap = self.mmap.write();
|
||||
let file = self.file.write();
|
||||
file.set_len(len)?;
|
||||
*mmap = Self::create_mmap(&file)?;
|
||||
Ok(())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_min_regions(&self, regions: usize) -> Result<()> {
|
||||
self.regions
|
||||
.write()
|
||||
.set_min_len((regions * SIZE_OF_REGION) as u64)?;
|
||||
self.set_min_len(regions as u64 * PAGE_SIZE)
|
||||
}
|
||||
|
||||
pub fn create_region_if_needed(&self, id: &str) -> Result<(usize, Arc<RwLock<Region>>)> {
|
||||
let regions = self.regions.read();
|
||||
if let Some(index) = regions.get_region_index_from_id(id) {
|
||||
return Ok((index, regions.get_region_from_index(index).unwrap()));
|
||||
}
|
||||
drop(regions);
|
||||
|
||||
let mut regions = self.regions.write();
|
||||
let mut layout = self.layout.write();
|
||||
|
||||
let start = if let Some(start) = layout.find_smallest_adequate_hole(PAGE_SIZE) {
|
||||
layout.remove_or_compress_hole_to_right(start, PAGE_SIZE);
|
||||
start
|
||||
} else {
|
||||
let start = layout
|
||||
.get_last_region_index()
|
||||
.map(|index| {
|
||||
let region_opt = regions.get_region_from_index(index);
|
||||
let region = region_opt.as_ref().unwrap().read();
|
||||
region.start() + region.reserved()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let len = start + PAGE_SIZE;
|
||||
|
||||
self.set_min_len(len)?;
|
||||
|
||||
start
|
||||
};
|
||||
|
||||
let (index, region) = regions.create_region(id.to_owned(), start)?;
|
||||
|
||||
layout.insert_region(start, index);
|
||||
|
||||
Ok((index, region))
|
||||
}
|
||||
|
||||
pub fn get_region(&self, identifier: Identifier) -> Result<RwLockReadGuard<'static, Region>> {
|
||||
let regions = self.regions.read();
|
||||
let region_opt = regions.get_region(identifier);
|
||||
let region_arc = region_opt.ok_or(Error::Str("Unknown region"))?;
|
||||
let region = region_arc.read();
|
||||
let region: RwLockReadGuard<'static, Region> = unsafe { std::mem::transmute(region) };
|
||||
Ok(region)
|
||||
}
|
||||
|
||||
pub fn create_region_reader<'a>(&'a self, identifier: Identifier) -> Result<Reader<'a>> {
|
||||
let mmap: RwLockReadGuard<'a, MmapMut> = self.mmap.read();
|
||||
let region = self.get_region(identifier)?;
|
||||
Ok(Reader::new(mmap, region))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn write_all_to_region(&self, identifier: Identifier, data: &[u8]) -> Result<()> {
|
||||
self.write_all_to_region_at_(identifier, data, None)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn write_all_to_region_at(
|
||||
&self,
|
||||
identifier: Identifier,
|
||||
data: &[u8],
|
||||
at: u64,
|
||||
) -> Result<()> {
|
||||
self.write_all_to_region_at_(identifier, data, Some(at))
|
||||
}
|
||||
|
||||
fn write_all_to_region_at_(
|
||||
&self,
|
||||
identifier: Identifier,
|
||||
data: &[u8],
|
||||
at: Option<u64>,
|
||||
) -> Result<()> {
|
||||
let regions = self.regions.read();
|
||||
let Some(region_lock) = regions.get_region(identifier.clone()) else {
|
||||
return Err(Error::Str("Unknown region"));
|
||||
};
|
||||
|
||||
let region_index = regions.identifier_to_index(identifier).unwrap();
|
||||
|
||||
let region = region_lock.read();
|
||||
let start = region.start();
|
||||
let reserved = region.reserved();
|
||||
let len = region.len();
|
||||
drop(region);
|
||||
|
||||
let data_len = data.len() as u64;
|
||||
let new_len = at.map_or(len + data_len, |at| (at + data_len).max(len));
|
||||
|
||||
if at.is_some_and(|at| at >= start + reserved) {
|
||||
return Err(Error::Str("Invalid at parameter"));
|
||||
}
|
||||
|
||||
// Write to reserved space if possible
|
||||
if new_len <= reserved {
|
||||
// println!(
|
||||
// "Write to {region_index} reserved space at {}",
|
||||
// start + at.unwrap_or(len)
|
||||
// );
|
||||
|
||||
if at.is_none() {
|
||||
self.write(start + len, data);
|
||||
}
|
||||
|
||||
let mut region = region_lock.write();
|
||||
if let Some(at) = at {
|
||||
self.write(start + at, data);
|
||||
}
|
||||
if len != new_len {
|
||||
region.set_len(new_len);
|
||||
}
|
||||
regions.write_to_mmap(®ion, region_index);
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// let layouat_lock = self.layout.read();
|
||||
|
||||
assert!(new_len > reserved);
|
||||
let mut new_reserved = reserved;
|
||||
while new_len > new_reserved {
|
||||
new_reserved *= 2;
|
||||
}
|
||||
assert!(new_len <= new_reserved);
|
||||
let added_reserve = new_reserved - reserved;
|
||||
|
||||
let mut layout = self.layout.write();
|
||||
|
||||
// If is last continue writing
|
||||
if layout.is_last_region(region_index) {
|
||||
// println!(
|
||||
// "{region_index} Append to file at {}",
|
||||
// start + at.unwrap_or(len)
|
||||
// );
|
||||
|
||||
self.set_min_len(start + new_reserved)?;
|
||||
|
||||
if at.is_none() {
|
||||
self.write(start + len, data);
|
||||
}
|
||||
let mut region = region_lock.write();
|
||||
region.set_reserved(new_reserved);
|
||||
if let Some(at) = at {
|
||||
self.write(start + at, data);
|
||||
}
|
||||
region.set_len(new_len);
|
||||
regions.write_to_mmap(®ion, region_index);
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Expand region to the right if gap is wide enough
|
||||
let hole_start = start + reserved;
|
||||
if layout
|
||||
.get_hole(hole_start)
|
||||
.is_some_and(|gap| gap >= added_reserve)
|
||||
{
|
||||
// println!("Expand {region_index} to hole");
|
||||
|
||||
layout.remove_or_compress_hole_to_right(hole_start, added_reserve);
|
||||
drop(layout);
|
||||
|
||||
if at.is_none() {
|
||||
self.write(start + len, data);
|
||||
}
|
||||
let mut region = region_lock.write();
|
||||
region.set_reserved(new_reserved);
|
||||
if let Some(at) = at {
|
||||
self.write(start + at, data);
|
||||
}
|
||||
region.set_len(new_len);
|
||||
regions.write_to_mmap(®ion, region_index);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Find hole big enough to move the region
|
||||
if let Some(hole_start) = layout.find_smallest_adequate_hole(new_reserved) {
|
||||
// println!("Move {region_index} to hole at {hole_start}");
|
||||
|
||||
layout.remove_or_compress_hole_to_right(hole_start, new_reserved);
|
||||
// drop(layout);
|
||||
|
||||
self.write(
|
||||
hole_start,
|
||||
&self.mmap.read()[start as usize..(start + len) as usize],
|
||||
);
|
||||
self.write(hole_start + len, data);
|
||||
|
||||
// let mut layout = self.layout.write();
|
||||
let mut region = region_lock.write();
|
||||
layout.move_region(hole_start, region_index, ®ion)?;
|
||||
region.set_start(hole_start);
|
||||
region.set_reserved(new_reserved);
|
||||
region.set_len(new_len);
|
||||
|
||||
drop(layout);
|
||||
|
||||
regions.write_to_mmap(®ion, region_index);
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Write at the end
|
||||
// let mut region_lock = region.write();
|
||||
let (last_region_start, last_region_index) = layout.get_last_region().unwrap();
|
||||
let new_start = last_region_start
|
||||
+ regions
|
||||
.get_region_from_index(last_region_index)
|
||||
.unwrap()
|
||||
.read()
|
||||
.reserved();
|
||||
// println!(
|
||||
// "Move {region_index} to the end, from {start}..{} to {new_start}..{}",
|
||||
// start + reserved,
|
||||
// new_start + new_reserved
|
||||
// );
|
||||
|
||||
self.set_min_len(new_start + new_reserved)?;
|
||||
|
||||
let mut region = region_lock.write();
|
||||
layout.move_region(new_start, region_index, ®ion)?;
|
||||
|
||||
self.write(
|
||||
new_start,
|
||||
&self.mmap.read()[start as usize..(start + len) as usize],
|
||||
);
|
||||
self.write(new_start + len, data);
|
||||
|
||||
// dbg!(new_start, region_index, ®ion_lock, new_reserved, new_len);
|
||||
|
||||
region.set_start(new_start);
|
||||
region.set_reserved(new_reserved);
|
||||
region.set_len(new_len);
|
||||
|
||||
drop(layout);
|
||||
|
||||
regions.write_to_mmap(®ion, region_index);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write(&self, at: u64, data: &[u8]) {
|
||||
let mmap = self.mmap.read();
|
||||
|
||||
let data_len = data.len();
|
||||
let start = at as usize;
|
||||
let end = start + data_len;
|
||||
|
||||
if end > mmap.len() {
|
||||
unreachable!("Trying to write beyond mmap")
|
||||
}
|
||||
|
||||
let slice = unsafe { std::slice::from_raw_parts_mut(mmap.as_ptr() as *mut u8, mmap.len()) };
|
||||
|
||||
slice[start..end].copy_from_slice(data);
|
||||
}
|
||||
|
||||
/// From relative to start
|
||||
pub fn truncate_region(&self, identifier: Identifier, from: u64) -> Result<()> {
|
||||
let Some(region) = self.regions.read().get_region(identifier) else {
|
||||
return Err(Error::Str("Unknown region"));
|
||||
};
|
||||
let mut region_ = region.write();
|
||||
let start = region_.start();
|
||||
let len = region_.len();
|
||||
let reserved = region_.reserved();
|
||||
|
||||
// dbg!(from, start);
|
||||
|
||||
if from == len {
|
||||
return Ok(());
|
||||
} else if from > len {
|
||||
return Err(Error::Str("Truncating further than length"));
|
||||
}
|
||||
|
||||
region_.set_len(from);
|
||||
|
||||
let end = start + reserved;
|
||||
let start = Self::ceil_number_to_page_size_multiple(start + from);
|
||||
if start > end {
|
||||
unreachable!("Should not be possible");
|
||||
} else if start < end {
|
||||
self.punch_hole(start, end - start)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn remove_region(&self, identifier: Identifier) -> Result<Option<Arc<RwLock<Region>>>> {
|
||||
let mut regions = self.regions.write();
|
||||
|
||||
let mut layout = self.layout.write();
|
||||
|
||||
let index_opt = regions.identifier_to_index(identifier.clone());
|
||||
|
||||
let Some(region) = regions.remove_region(identifier)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let index = index_opt.unwrap();
|
||||
|
||||
let region_ = region.write();
|
||||
|
||||
layout.remove_region(index, ®ion_)?;
|
||||
|
||||
self.punch_hole(region_.start(), region_.reserved())?;
|
||||
|
||||
drop(region_);
|
||||
|
||||
Ok(Some(region))
|
||||
}
|
||||
|
||||
fn create_mmap(file: &fs::File) -> Result<MmapMut> {
|
||||
Ok(unsafe { MmapOptions::new().map_mut(file)? })
|
||||
}
|
||||
|
||||
pub fn regions(&self) -> RwLockReadGuard<'_, Regions> {
|
||||
self.regions.read()
|
||||
}
|
||||
|
||||
pub fn layout(&self) -> RwLockReadGuard<'_, Layout> {
|
||||
self.layout.read()
|
||||
}
|
||||
|
||||
pub fn mmap(&self) -> RwLockReadGuard<'_, MmapMut> {
|
||||
self.mmap.read()
|
||||
}
|
||||
|
||||
fn ceil_number_to_page_size_multiple(num: u64) -> u64 {
|
||||
(num + PAGE_SIZE_MINUS_1) & !PAGE_SIZE_MINUS_1
|
||||
}
|
||||
|
||||
fn data_path(&self) -> PathBuf {
|
||||
Self::data_path_(&self.path)
|
||||
}
|
||||
fn data_path_(path: &Path) -> PathBuf {
|
||||
path.join("data")
|
||||
}
|
||||
|
||||
pub fn disk_usage(&self) -> String {
|
||||
let path = self.data_path();
|
||||
|
||||
let output = std::process::Command::new("du")
|
||||
.arg("-h")
|
||||
.arg(&path)
|
||||
.output()
|
||||
.expect("Failed to run du");
|
||||
|
||||
String::from_utf8_lossy(&output.stdout)
|
||||
.replace(path.to_str().unwrap(), " ")
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
pub fn flush(&self) -> Result<()> {
|
||||
let mmap = self.mmap.read();
|
||||
let regions = self.regions.read();
|
||||
mmap.flush()?;
|
||||
regions.flush()
|
||||
}
|
||||
|
||||
pub fn punch_holes(&self) -> Result<()> {
|
||||
let file = self.file.write();
|
||||
let mmap = self.mmap.read();
|
||||
let layout = self.layout.read();
|
||||
Self::punch_holes_(&file, &mmap, &layout)
|
||||
}
|
||||
|
||||
fn punch_holes_(file: &fs::File, mmap: &MmapMut, layout: &Layout) -> Result<()> {
|
||||
layout
|
||||
.start_to_hole()
|
||||
.par_iter()
|
||||
.try_for_each(|(&start, &hole)| -> Result<()> {
|
||||
assert!(start % PAGE_SIZE == 0);
|
||||
assert!(hole % PAGE_SIZE == 0);
|
||||
let has_old_data =
|
||||
mmap[start as usize] != 0 || mmap[(start + hole - PAGE_SIZE) as usize] != 0;
|
||||
if has_old_data {
|
||||
info!("Punching a hole of {hole} bytes at {start}...");
|
||||
Self::punch_hole_(file, start, hole)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn punch_hole(&self, start: u64, length: u64) -> Result<()> {
|
||||
let file = self.file.write();
|
||||
Self::punch_hole_macos(&file, start, length)
|
||||
}
|
||||
|
||||
fn punch_hole_(file: &fs::File, start: u64, length: u64) -> Result<()> {
|
||||
Self::punch_hole_macos(file, start, length)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn punch_hole_macos(file: &fs::File, start: u64, length: u64) -> Result<()> {
|
||||
let fpunchhole = FPunchhole {
|
||||
fp_flags: 0,
|
||||
reserved: 0,
|
||||
fp_offset: start as libc::off_t,
|
||||
fp_length: length as libc::off_t,
|
||||
};
|
||||
|
||||
let result = unsafe {
|
||||
libc::fcntl(
|
||||
file.as_raw_fd(),
|
||||
libc::F_PUNCHHOLE,
|
||||
&fpunchhole as *const FPunchhole,
|
||||
)
|
||||
};
|
||||
|
||||
if result == -1 {
|
||||
let err = std::io::Error::last_os_error();
|
||||
return Err(Error::String(format!("Failed to punch hole: {err}")));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct FPunchhole {
|
||||
fp_flags: u32,
|
||||
reserved: u32,
|
||||
fp_offset: off_t,
|
||||
fp_length: off_t,
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
use memmap2::MmapMut;
|
||||
use parking_lot::RwLockReadGuard;
|
||||
|
||||
use super::Region;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Reader<'a> {
|
||||
mmap: RwLockReadGuard<'a, MmapMut>,
|
||||
region: RwLockReadGuard<'static, Region>,
|
||||
}
|
||||
|
||||
impl<'a> Reader<'a> {
|
||||
pub fn new(
|
||||
mmap: RwLockReadGuard<'a, MmapMut>,
|
||||
region: RwLockReadGuard<'static, Region>,
|
||||
) -> Self {
|
||||
Self { mmap, region }
|
||||
}
|
||||
|
||||
pub fn read(&self, offset: u64, len: u64) -> &[u8] {
|
||||
assert!(offset + len <= self.region.len());
|
||||
let start = self.region.start() + offset;
|
||||
let end = start + len;
|
||||
&self.mmap[start as usize..end as usize]
|
||||
}
|
||||
|
||||
pub fn read_all(&self) -> &[u8] {
|
||||
self.read(0, self.region().len())
|
||||
}
|
||||
|
||||
pub fn region(&self) -> &Region {
|
||||
&self.region
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
use memmap2::MmapMut;
|
||||
use parking_lot::RwLockReadGuard;
|
||||
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use super::{File, PAGE_SIZE, Reader};
|
||||
|
||||
#[derive(Debug, Clone, FromBytes, IntoBytes, Immutable, KnownLayout)]
|
||||
#[repr(C)]
|
||||
pub struct Region {
|
||||
/// Must be multiple of 4096
|
||||
start: u64,
|
||||
len: u64,
|
||||
/// Must be multiple of 4096, greater or equal to len
|
||||
reserved: u64,
|
||||
}
|
||||
|
||||
pub const SIZE_OF_REGION: usize = size_of::<Region>();
|
||||
|
||||
impl Region {
|
||||
pub fn new(start: u64, len: u64, reserved: u64) -> Self {
|
||||
assert!(start % PAGE_SIZE == 0);
|
||||
assert!(reserved >= PAGE_SIZE);
|
||||
assert!(reserved % PAGE_SIZE == 0);
|
||||
assert!(len <= reserved);
|
||||
|
||||
Self {
|
||||
start,
|
||||
len,
|
||||
reserved,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start(&self) -> u64 {
|
||||
self.start
|
||||
}
|
||||
|
||||
pub fn set_start(&mut self, start: u64) {
|
||||
assert!(start % PAGE_SIZE == 0);
|
||||
self.start = start
|
||||
}
|
||||
|
||||
pub fn len(&self) -> u64 {
|
||||
self.len
|
||||
}
|
||||
|
||||
pub fn set_len(&mut self, len: u64) {
|
||||
assert!(len <= self.reserved());
|
||||
self.len = len
|
||||
}
|
||||
|
||||
pub fn reserved(&self) -> u64 {
|
||||
self.reserved
|
||||
}
|
||||
|
||||
pub fn set_reserved(&mut self, reserved: u64) {
|
||||
assert!(self.len() <= reserved);
|
||||
assert!(reserved >= PAGE_SIZE);
|
||||
assert!(reserved % PAGE_SIZE == 0);
|
||||
|
||||
self.reserved = reserved;
|
||||
}
|
||||
|
||||
pub fn left(&self) -> u64 {
|
||||
self.reserved - self.len
|
||||
}
|
||||
}
|
||||
|
||||
pub trait RegionReader {
|
||||
fn create_reader(self, file: &File) -> Reader;
|
||||
}
|
||||
|
||||
impl<'a> RegionReader for RwLockReadGuard<'a, Region> {
|
||||
fn create_reader(self, file: &File) -> Reader<'static> {
|
||||
let region: RwLockReadGuard<'static, Region> = unsafe { std::mem::transmute(self) };
|
||||
let mmap: RwLockReadGuard<'static, MmapMut> =
|
||||
unsafe { std::mem::transmute(file.mmap.read()) };
|
||||
Reader::new(mmap, region)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs::{self, OpenOptions},
|
||||
io::{Cursor, Read},
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use brk_core::{Error, Result};
|
||||
use memmap2::MmapMut;
|
||||
use parking_lot::{RwLock, RwLockWriteGuard};
|
||||
use zerocopy::{FromBytes, IntoBytes};
|
||||
|
||||
use super::{
|
||||
Identifier, PAGE_SIZE,
|
||||
region::{Region, SIZE_OF_REGION},
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Regions {
|
||||
id_to_index: HashMap<String, usize>,
|
||||
id_to_index_path: PathBuf,
|
||||
index_to_region: Vec<Option<Arc<RwLock<Region>>>>,
|
||||
index_to_region_file: fs::File,
|
||||
index_to_region_mmap: MmapMut,
|
||||
}
|
||||
|
||||
impl Regions {
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
let path = path.join("regions");
|
||||
|
||||
fs::create_dir_all(&path)?;
|
||||
|
||||
let id_to_index_path = path.join("id_to_index");
|
||||
|
||||
let id_to_index: HashMap<String, usize> =
|
||||
deserialize_hashmap_binary(&fs::read(&id_to_index_path).unwrap_or_default())
|
||||
.unwrap_or_default();
|
||||
|
||||
let index_to_region_file = OpenOptions::new()
|
||||
.read(true)
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(false)
|
||||
.open(path.join("index_to_region"))?;
|
||||
|
||||
let index_to_region_mmap = unsafe { MmapMut::map_mut(&index_to_region_file)? };
|
||||
|
||||
let mut index_to_region: Vec<Option<Arc<RwLock<Region>>>> = vec![];
|
||||
|
||||
id_to_index
|
||||
.iter()
|
||||
.try_for_each(|(_, &index)| -> Result<()> {
|
||||
let start = index * SIZE_OF_REGION;
|
||||
let end = start + SIZE_OF_REGION;
|
||||
let region = Region::read_from_bytes(&index_to_region_mmap[start..end])?;
|
||||
if index_to_region.len() < index + 1 {
|
||||
index_to_region.resize_with(index + 1, Default::default);
|
||||
}
|
||||
index_to_region
|
||||
.get_mut(index)
|
||||
.unwrap()
|
||||
.replace(Arc::new(RwLock::new(region)));
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
// TODO: Removes Nones from vec if needed, update map accordingly and save them
|
||||
|
||||
Ok(Self {
|
||||
id_to_index,
|
||||
id_to_index_path,
|
||||
index_to_region,
|
||||
index_to_region_file,
|
||||
index_to_region_mmap,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn set_min_len(&mut self, len: u64) -> Result<()> {
|
||||
if self.index_to_region_mmap.len() < len as usize {
|
||||
self.index_to_region_file.set_len(len)?;
|
||||
self.index_to_region_mmap = unsafe { MmapMut::map_mut(&self.index_to_region_file)? };
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn create_region(
|
||||
&mut self,
|
||||
id: String,
|
||||
start: u64,
|
||||
) -> Result<(usize, Arc<RwLock<Region>>)> {
|
||||
let index = self
|
||||
.index_to_region
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find(|(_, opt)| opt.is_none())
|
||||
.map(|(index, _)| index)
|
||||
.unwrap_or_else(|| self.index_to_region.len());
|
||||
|
||||
let region = Region::new(start, 0, PAGE_SIZE);
|
||||
|
||||
self.set_min_len(((index + 1) * SIZE_OF_REGION) as u64)?;
|
||||
|
||||
let region_lock = RwLock::new(region);
|
||||
|
||||
self.write_to_mmap(®ion_lock.write(), index);
|
||||
|
||||
let region_arc = Arc::new(region_lock);
|
||||
|
||||
let region_opt = Some(region_arc.clone());
|
||||
if index < self.index_to_region.len() {
|
||||
self.index_to_region[index] = region_opt
|
||||
} else {
|
||||
self.index_to_region.push(region_opt);
|
||||
self.index_to_region_mmap.flush()?;
|
||||
}
|
||||
|
||||
if self.id_to_index.insert(id, index).is_some() {
|
||||
return Err(Error::Str("Already exists"));
|
||||
}
|
||||
self.flush_id_to_index()?;
|
||||
|
||||
Ok((index, region_arc))
|
||||
}
|
||||
|
||||
fn flush_id_to_index(&mut self) -> Result<()> {
|
||||
fs::write(
|
||||
&self.id_to_index_path,
|
||||
serialize_hashmap_binary(&self.id_to_index),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get_region(&self, identifier: Identifier) -> Option<Arc<RwLock<Region>>> {
|
||||
match identifier {
|
||||
Identifier::Number(index) => self.get_region_from_index(index),
|
||||
Identifier::String(id) => self.get_region_from_id(&id),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get_region_from_index(&self, index: usize) -> Option<Arc<RwLock<Region>>> {
|
||||
self.index_to_region.get(index).cloned().flatten()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get_region_from_id(&self, id: &str) -> Option<Arc<RwLock<Region>>> {
|
||||
self.get_region_index_from_id(id)
|
||||
.and_then(|index| self.get_region_from_index(index))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get_region_index_from_id(&self, id: &str) -> Option<usize> {
|
||||
self.id_to_index.get(id).copied()
|
||||
}
|
||||
|
||||
fn find_id_from_index(&self, index: usize) -> Option<&String> {
|
||||
Some(
|
||||
self.id_to_index
|
||||
.iter()
|
||||
.find(|(_, v)| **v == index)
|
||||
.unwrap()
|
||||
.0,
|
||||
)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn index_to_region(&self) -> &[Option<Arc<RwLock<Region>>>] {
|
||||
&self.index_to_region
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn id_to_index(&self) -> &HashMap<String, usize> {
|
||||
&self.id_to_index
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn identifier_to_index(&self, identifier: Identifier) -> Option<usize> {
|
||||
match identifier {
|
||||
Identifier::Number(index) => Some(index),
|
||||
Identifier::String(id) => self.get_region_index_from_id(&id),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove_region(&mut self, identifier: Identifier) -> Result<Option<Arc<RwLock<Region>>>> {
|
||||
match identifier {
|
||||
Identifier::Number(index) => self.remove_region_from_index(index),
|
||||
Identifier::String(id) => self.remove_region_from_id(&id),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove_region_from_id(&mut self, id: &str) -> Result<Option<Arc<RwLock<Region>>>> {
|
||||
let Some(index) = self.get_region_index_from_id(id) else {
|
||||
return Ok(None);
|
||||
};
|
||||
self.remove_region_from_index(index)
|
||||
}
|
||||
|
||||
pub fn remove_region_from_index(
|
||||
&mut self,
|
||||
index: usize,
|
||||
) -> Result<Option<Arc<RwLock<Region>>>> {
|
||||
let Some(region) = self.index_to_region.get_mut(index).and_then(Option::take) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
self.index_to_region_mmap.flush()?;
|
||||
|
||||
self.id_to_index
|
||||
.remove(&self.find_id_from_index(index).unwrap().to_owned());
|
||||
|
||||
self.flush_id_to_index()?;
|
||||
|
||||
Ok(Some(region))
|
||||
}
|
||||
|
||||
pub fn write_to_mmap(&self, region: &RwLockWriteGuard<Region>, index: usize) {
|
||||
let mmap = &self.index_to_region_mmap;
|
||||
let start = index * SIZE_OF_REGION;
|
||||
let end = start + SIZE_OF_REGION;
|
||||
|
||||
if end > mmap.len() {
|
||||
unreachable!("Trying to write beyond mmap")
|
||||
}
|
||||
|
||||
let slice = unsafe { std::slice::from_raw_parts_mut(mmap.as_ptr() as *mut u8, mmap.len()) };
|
||||
|
||||
slice[start..end].copy_from_slice(region.as_bytes());
|
||||
}
|
||||
|
||||
pub fn flush(&self) -> Result<()> {
|
||||
self.index_to_region_mmap.flush().map_err(|e| e.into())
|
||||
}
|
||||
}
|
||||
|
||||
fn serialize_hashmap_binary(map: &HashMap<String, usize>) -> Vec<u8> {
|
||||
let mut buffer = Vec::new();
|
||||
|
||||
buffer.extend_from_slice(&map.len().to_ne_bytes());
|
||||
|
||||
for (key, value) in map {
|
||||
buffer.extend_from_slice(&key.len().to_ne_bytes());
|
||||
buffer.extend_from_slice(key.as_bytes());
|
||||
buffer.extend_from_slice(&value.to_ne_bytes());
|
||||
}
|
||||
|
||||
buffer
|
||||
}
|
||||
|
||||
fn deserialize_hashmap_binary(data: &[u8]) -> Result<HashMap<String, usize>> {
|
||||
let mut cursor = Cursor::new(data);
|
||||
let mut buffer = [0u8; 8];
|
||||
|
||||
cursor
|
||||
.read_exact(&mut buffer)
|
||||
.map_err(|_| Error::Str("Failed to read entry count"))?;
|
||||
let entry_count = usize::from_ne_bytes(buffer);
|
||||
|
||||
let mut map = HashMap::with_capacity(entry_count);
|
||||
|
||||
for _ in 0..entry_count {
|
||||
cursor
|
||||
.read_exact(&mut buffer)
|
||||
.map_err(|_| Error::Str("Failed to read key length"))?;
|
||||
let key_len = usize::from_ne_bytes(buffer);
|
||||
|
||||
let mut key_bytes = vec![0u8; key_len];
|
||||
cursor
|
||||
.read_exact(&mut key_bytes)
|
||||
.map_err(|_| Error::Str("Failed to read key"))?;
|
||||
let key = String::from_utf8(key_bytes).map_err(|_| Error::Str("Invalid UTF-8 in key"))?;
|
||||
|
||||
cursor
|
||||
.read_exact(&mut buffer)
|
||||
.map_err(|_| Error::Str("Failed to read value"))?;
|
||||
let value = usize::from_ne_bytes(buffer);
|
||||
|
||||
map.insert(key, value);
|
||||
}
|
||||
|
||||
Ok(map)
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
mod file;
|
||||
mod traits;
|
||||
mod variants;
|
||||
|
||||
use variants::*;
|
||||
|
||||
pub use file::{File, PAGE_SIZE, Reader};
|
||||
pub use traits::*;
|
||||
pub use variants::{
|
||||
AnyStampedVec, CompressedVec, Computation, ComputedVec, ComputedVecFrom1, ComputedVecFrom2,
|
||||
ComputedVecFrom3, EagerVec, Format, LazyVecFrom1, LazyVecFrom2, LazyVecFrom3, RawVec, Stamp,
|
||||
StampedVec, StoredVec,
|
||||
};
|
||||
@@ -2,15 +2,11 @@ use std::{
|
||||
borrow::Cow,
|
||||
cmp::Ordering,
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fs::{self, File, OpenOptions},
|
||||
io::{self, Seek, SeekFrom, Write},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use brk_core::{Error, Result};
|
||||
use memmap2::Mmap;
|
||||
|
||||
use crate::{AnyVec, HEADER_OFFSET, Header};
|
||||
use crate::{AnyVec, File, HEADER_OFFSET, Header, file::Reader};
|
||||
|
||||
use super::{StoredIndex, StoredType};
|
||||
|
||||
@@ -22,22 +18,46 @@ where
|
||||
{
|
||||
const SIZE_OF_T: usize = size_of::<T>();
|
||||
|
||||
#[inline]
|
||||
fn unwrap_read(&self, index: I, mmap: &Mmap) -> T {
|
||||
self.read(index, mmap).unwrap().unwrap()
|
||||
fn file(&self) -> &File;
|
||||
|
||||
fn region_index(&self) -> usize;
|
||||
|
||||
/// Be careful with deadlocks
|
||||
///
|
||||
/// You'll want to drop the reader before mutable ops
|
||||
fn create_reader(&self) -> Reader {
|
||||
self.create_static_reader()
|
||||
}
|
||||
#[inline]
|
||||
fn read(&self, index: I, mmap: &Mmap) -> Result<Option<T>> {
|
||||
self.read_(index.to_usize()?, mmap)
|
||||
|
||||
/// Be careful with deadlocks
|
||||
///
|
||||
/// You'll want to drop the reader before mutable ops
|
||||
fn create_static_reader(&self) -> Reader<'static> {
|
||||
unsafe {
|
||||
std::mem::transmute(
|
||||
self.file()
|
||||
.create_region_reader(self.region_index().into())
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
}
|
||||
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>>;
|
||||
|
||||
#[inline]
|
||||
fn get_or_read(&self, index: I, mmap: &Mmap) -> Result<Option<Cow<T>>> {
|
||||
self.get_or_read_(index.to_usize()?, mmap)
|
||||
fn unwrap_read(&self, index: I, reader: &Reader) -> T {
|
||||
self.read(index, reader).unwrap().unwrap()
|
||||
}
|
||||
#[inline]
|
||||
fn get_or_read_(&self, index: usize, mmap: &Mmap) -> Result<Option<Cow<T>>> {
|
||||
fn read(&self, index: I, reader: &Reader) -> Result<Option<T>> {
|
||||
self.read_(index.to_usize()?, reader)
|
||||
}
|
||||
fn read_(&self, index: usize, reader: &Reader) -> Result<Option<T>>;
|
||||
|
||||
#[inline]
|
||||
fn get_or_read(&self, index: I, reader: &Reader) -> Result<Option<Cow<T>>> {
|
||||
self.get_or_read_(index.to_usize()?, reader)
|
||||
}
|
||||
#[inline]
|
||||
fn get_or_read_(&self, index: usize, reader: &Reader) -> Result<Option<Cow<T>>> {
|
||||
let stored_len = self.stored_len();
|
||||
|
||||
if index >= stored_len {
|
||||
@@ -61,7 +81,7 @@ where
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
Ok(self.read_(index, mmap)?.map(Cow::Owned))
|
||||
Ok(self.read_(index, reader)?.map(Cow::Owned))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
@@ -69,10 +89,6 @@ where
|
||||
self.stored_len() + self.pushed_len()
|
||||
}
|
||||
|
||||
fn index_to_name(&self) -> String {
|
||||
format!("{}_to_{}", I::to_string(), self.name())
|
||||
}
|
||||
|
||||
fn stored_len(&self) -> usize;
|
||||
|
||||
fn pushed(&self) -> &[T];
|
||||
@@ -125,8 +141,8 @@ where
|
||||
|
||||
fn holes(&self) -> &BTreeSet<usize>;
|
||||
fn mut_holes(&mut self) -> &mut BTreeSet<usize>;
|
||||
fn take(&mut self, index: I, mmap: &Mmap) -> Result<Option<T>> {
|
||||
let opt = self.get_or_read(index, mmap)?.map(|v| v.into_owned());
|
||||
fn take(&mut self, index: I, reader: &Reader) -> Result<Option<T>> {
|
||||
let opt = self.get_or_read(index, reader)?.map(|v| v.into_owned());
|
||||
if opt.is_some() {
|
||||
self.unchecked_delete(index);
|
||||
}
|
||||
@@ -178,87 +194,13 @@ where
|
||||
fn header(&self) -> &Header;
|
||||
fn mut_header(&mut self) -> &mut Header;
|
||||
|
||||
fn parent(&self) -> &Path;
|
||||
|
||||
fn folder(&self) -> PathBuf {
|
||||
self.parent().join(self.name())
|
||||
}
|
||||
|
||||
fn folder_(parent: &Path, name: &str) -> PathBuf {
|
||||
parent.join(name)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn path(&self) -> PathBuf {
|
||||
Self::path_(self.parent(), self.name())
|
||||
}
|
||||
#[inline]
|
||||
fn path_(parent: &Path, name: &str) -> PathBuf {
|
||||
Self::folder_(parent, name).join(I::to_string())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn holes_path(&self) -> PathBuf {
|
||||
Self::holes_path_(self.parent(), self.name())
|
||||
}
|
||||
#[inline]
|
||||
fn holes_path_(parent: &Path, name: &str) -> PathBuf {
|
||||
Self::folder_(parent, name).join(format!("{}_holes", I::to_string()))
|
||||
}
|
||||
|
||||
// ---
|
||||
|
||||
fn open_file(&self) -> io::Result<File> {
|
||||
Self::open_file_(&self.path())
|
||||
}
|
||||
fn open_file_(path: &Path) -> io::Result<File> {
|
||||
let mut file = OpenOptions::new()
|
||||
.read(true)
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(false)
|
||||
.open(path)?;
|
||||
file.seek(SeekFrom::End(0))?;
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn file_set_len(&mut self, file: &mut File, len: u64) -> Result<()> {
|
||||
Self::file_set_len_(file, len)?;
|
||||
Ok(())
|
||||
}
|
||||
fn file_set_len_(file: &mut File, len: u64) -> Result<()> {
|
||||
file.set_len(len)?;
|
||||
file.seek(SeekFrom::End(0))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn file_write_all(&mut self, file: &mut File, buf: &[u8]) -> Result<()> {
|
||||
file.write_all(buf)?;
|
||||
file.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn file_truncate_and_write_all(&mut self, file: &mut File, len: u64, buf: &[u8]) -> Result<()> {
|
||||
Self::file_set_len_(file, len)?;
|
||||
self.file_write_all(file, buf)
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> Result<()>;
|
||||
|
||||
#[inline]
|
||||
fn reset_(&mut self) -> Result<()> {
|
||||
let holes_path = self.holes_path();
|
||||
if fs::exists(&holes_path)? {
|
||||
fs::remove_file(&holes_path)?;
|
||||
}
|
||||
let mut file = self.open_file()?;
|
||||
self.file_truncate_and_write_all(&mut file, HEADER_OFFSET as u64, &[])
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn create_mmap(&self) -> Result<Mmap> {
|
||||
let file = self.open_file()?;
|
||||
unsafe { Mmap::map(&file).map_err(|e| e.into()) }
|
||||
self.file().remove_region(self.holes_region_name().into())?;
|
||||
self.file()
|
||||
.truncate_region(self.region_index().into(), HEADER_OFFSET as u64)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
@@ -278,4 +220,22 @@ where
|
||||
fn flush(&mut self) -> Result<()>;
|
||||
|
||||
fn truncate_if_needed(&mut self, index: I) -> Result<()>;
|
||||
|
||||
fn index_to_name(&self) -> String {
|
||||
format!("{}_to_{}", I::to_string(), self.name())
|
||||
}
|
||||
|
||||
fn vec_region_name(&self) -> String {
|
||||
Self::vec_region_name_(self.name())
|
||||
}
|
||||
fn vec_region_name_(name: &str) -> String {
|
||||
format!("{}_to_{name}", I::to_string())
|
||||
}
|
||||
|
||||
fn holes_region_name(&self) -> String {
|
||||
Self::holes_region_name_(self.name())
|
||||
}
|
||||
fn holes_region_name_(name: &str) -> String {
|
||||
format!("{}_holes", Self::vec_region_name_(name))
|
||||
}
|
||||
}
|
||||
+137
-141
@@ -2,22 +2,27 @@ use std::{
|
||||
borrow::Cow,
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fs, mem,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use arc_swap::{ArcSwap, Guard};
|
||||
use brk_core::{Error, Result, Version};
|
||||
use memmap2::Mmap;
|
||||
use parking_lot::{RwLock, RwLockReadGuard};
|
||||
use rayon::prelude::*;
|
||||
use zstd::DEFAULT_COMPRESSION_LEVEL;
|
||||
|
||||
use crate::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedVecIterator, CollectableVec,
|
||||
CompressedPageMetadata, CompressedPagesMetadata, GenericStoredVec, HEADER_OFFSET, Header,
|
||||
RawVec, StoredIndex, StoredType, UnsafeSlice,
|
||||
File, GenericStoredVec, HEADER_OFFSET, Header, RawVec, Reader, StoredIndex, StoredType,
|
||||
UnsafeSlice,
|
||||
};
|
||||
|
||||
mod compressed_page_meta;
|
||||
mod compressed_pages_meta;
|
||||
|
||||
use compressed_page_meta::*;
|
||||
use compressed_pages_meta::*;
|
||||
|
||||
const ONE_KIB: usize = 1024;
|
||||
const ONE_MIB: usize = ONE_KIB * ONE_KIB;
|
||||
pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB;
|
||||
@@ -28,7 +33,7 @@ const VERSION: Version = Version::TWO;
|
||||
#[derive(Debug)]
|
||||
pub struct CompressedVec<I, T> {
|
||||
inner: RawVec<I, T>,
|
||||
pages_meta: Arc<ArcSwap<CompressedPagesMetadata>>,
|
||||
pages_meta: Arc<RwLock<CompressedPagesMetadata>>,
|
||||
}
|
||||
|
||||
impl<I, T> CompressedVec<I, T>
|
||||
@@ -40,57 +45,66 @@ where
|
||||
pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE_OF_T;
|
||||
pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
|
||||
|
||||
/// Same as import but will reset the folder under certain errors, so be careful !
|
||||
pub fn forced_import(parent: &Path, name: &str, mut version: Version) -> Result<Self> {
|
||||
/// Same as import but will reset the vec under certain errors, so be careful !
|
||||
pub fn forced_import(file: &Arc<File>, name: &str, mut version: Version) -> Result<Self> {
|
||||
version = version + VERSION;
|
||||
let res = Self::import(parent, name, version);
|
||||
let res = Self::import(file, name, version);
|
||||
match res {
|
||||
Err(Error::DifferentCompressionMode)
|
||||
| Err(Error::WrongEndian)
|
||||
| Err(Error::WrongLength)
|
||||
| Err(Error::DifferentVersion { .. }) => {
|
||||
let path = Self::path_(parent, name);
|
||||
fs::remove_file(path)?;
|
||||
Self::import(parent, name, version)
|
||||
todo!();
|
||||
|
||||
// let path = Self::path_(file, name);
|
||||
// fs::remove_file(path)?;
|
||||
// Self::import(file, name, version)
|
||||
}
|
||||
_ => res,
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(unreachable_code, unused_variables)]
|
||||
pub fn import(parent: &Path, name: &str, version: Version) -> Result<Self> {
|
||||
let mut inner = RawVec::import(parent, name, version)?;
|
||||
pub fn import(file: &Arc<File>, name: &str, version: Version) -> Result<Self> {
|
||||
// let mut inner = RawVec::import(file, name, version)?;
|
||||
|
||||
let pages_meta = {
|
||||
let path = inner
|
||||
.folder()
|
||||
.join(format!("{}-pages-meta", I::to_string()));
|
||||
if inner.is_empty() {
|
||||
let _ = fs::remove_file(&path);
|
||||
}
|
||||
CompressedPagesMetadata::read(&path)?
|
||||
};
|
||||
todo!();
|
||||
|
||||
inner.set_stored_len(if let Some(last) = pages_meta.last() {
|
||||
(pages_meta.len() - 1) * Self::PER_PAGE + last.values_len as usize
|
||||
} else {
|
||||
0
|
||||
});
|
||||
// let pages_meta = {
|
||||
// let path = inner
|
||||
// .folder()
|
||||
// .join(format!("{}-pages-meta", I::to_string()));
|
||||
// if inner.is_empty() {
|
||||
// let _ = fs::remove_file(&path);
|
||||
// }
|
||||
// CompressedPagesMetadata::read(&path)?
|
||||
// };
|
||||
|
||||
Ok(Self {
|
||||
inner,
|
||||
pages_meta: Arc::new(ArcSwap::new(Arc::new(pages_meta))),
|
||||
})
|
||||
// inner.set_stored_len(if let Some(last) = pages_meta.last() {
|
||||
// (pages_meta.len() - 1) * Self::PER_PAGE + last.values_len as usize
|
||||
// } else {
|
||||
// 0
|
||||
// });
|
||||
|
||||
// Ok(Self {
|
||||
// inner,
|
||||
// pages_meta: Arc::new(RwLock::new(pages_meta)),
|
||||
// })
|
||||
}
|
||||
|
||||
fn decode_page(&self, page_index: usize, mmap: &Mmap) -> Result<Vec<T>> {
|
||||
Self::decode_page_(self.stored_len(), page_index, mmap, &self.pages_meta.load())
|
||||
fn decode_page(&self, page_index: usize, reader: &Reader) -> Result<Vec<T>> {
|
||||
Self::decode_page_(
|
||||
self.stored_len(),
|
||||
page_index,
|
||||
reader,
|
||||
&self.pages_meta.read(),
|
||||
)
|
||||
}
|
||||
|
||||
fn decode_page_(
|
||||
stored_len: usize,
|
||||
page_index: usize,
|
||||
mmap: &Mmap,
|
||||
reader: &Reader,
|
||||
compressed_pages_meta: &CompressedPagesMetadata,
|
||||
) -> Result<Vec<T>> {
|
||||
if Self::page_index_to_index(page_index) >= stored_len {
|
||||
@@ -103,9 +117,11 @@ where
|
||||
let len = page.bytes_len as usize;
|
||||
let offset = page.start as usize;
|
||||
|
||||
Ok(zstd::decode_all(&mmap[offset..offset + len])
|
||||
let slice = reader.read(offset as u64, (offset + len) as u64);
|
||||
|
||||
Ok(zstd::decode_all(slice)
|
||||
.inspect_err(|_| {
|
||||
dbg!((len, offset, page_index, &mmap[..], &mmap.len()));
|
||||
dbg!((len, offset, page_index, slice));
|
||||
})?
|
||||
.chunks(Self::SIZE_OF_T)
|
||||
.map(|slice| T::try_read_from_bytes(slice).unwrap())
|
||||
@@ -162,13 +178,21 @@ where
|
||||
I: StoredIndex,
|
||||
T: StoredType,
|
||||
{
|
||||
fn file(&self) -> &File {
|
||||
self.inner.file()
|
||||
}
|
||||
|
||||
fn region_index(&self) -> usize {
|
||||
self.inner.region_index()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn read_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
|
||||
fn read_(&self, index: usize, reader: &Reader) -> Result<Option<T>> {
|
||||
let page_index = Self::index_to_page_index(index);
|
||||
let decoded_index = index % Self::PER_PAGE;
|
||||
|
||||
Ok(self
|
||||
.decode_page(page_index, mmap)?
|
||||
.decode_page(page_index, reader)?
|
||||
.get(decoded_index)
|
||||
.cloned())
|
||||
}
|
||||
@@ -181,10 +205,6 @@ where
|
||||
self.inner.mut_header()
|
||||
}
|
||||
|
||||
fn parent(&self) -> &Path {
|
||||
self.inner.parent()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn stored_len(&self) -> usize {
|
||||
self.inner.stored_len()
|
||||
@@ -204,7 +224,7 @@ where
|
||||
}
|
||||
#[inline]
|
||||
fn mut_holes(&mut self) -> &mut BTreeSet<usize> {
|
||||
self.inner.mut_holes()
|
||||
panic!("unsupported")
|
||||
}
|
||||
#[inline]
|
||||
fn updated(&self) -> &BTreeMap<usize, T> {
|
||||
@@ -212,120 +232,101 @@ where
|
||||
}
|
||||
#[inline]
|
||||
fn mut_updated(&mut self) -> &mut BTreeMap<usize, T> {
|
||||
self.inner.mut_updated()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn path(&self) -> PathBuf {
|
||||
self.inner.path()
|
||||
}
|
||||
|
||||
fn delete(&mut self, _: I) {
|
||||
panic!("unsupported")
|
||||
}
|
||||
|
||||
fn unchecked_delete(&mut self, _: I) {
|
||||
panic!("unsupported")
|
||||
}
|
||||
|
||||
fn fill_first_hole_or_push(&mut self, _: T) -> Result<I> {
|
||||
panic!("unsupported")
|
||||
}
|
||||
|
||||
fn update(&mut self, _: I, _: T) -> Result<()> {
|
||||
panic!("unsupported")
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Result<()> {
|
||||
let file_opt = self.inner.write_header_if_needed()?;
|
||||
todo!();
|
||||
|
||||
let pushed_len = self.pushed_len();
|
||||
// let file_opt = self.inner.write_header_if_needed()?;
|
||||
|
||||
if pushed_len == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
// let pushed_len = self.pushed_len();
|
||||
|
||||
let stored_len = self.stored_len();
|
||||
// if pushed_len == 0 {
|
||||
// return Ok(());
|
||||
// }
|
||||
|
||||
let mut file = file_opt.unwrap_or(self.open_file()?);
|
||||
// let stored_len = self.stored_len();
|
||||
|
||||
let mut pages_meta = (**self.pages_meta.load()).clone();
|
||||
// let mut file = file_opt.unwrap_or(self.open_file()?);
|
||||
|
||||
let mut starting_page_index = pages_meta.len();
|
||||
let mut values = vec![];
|
||||
let mut truncate_at = None;
|
||||
// let mut pages_meta = self.pages_meta.read();
|
||||
|
||||
if self.stored_len() % Self::PER_PAGE != 0 {
|
||||
if pages_meta.is_empty() {
|
||||
unreachable!()
|
||||
}
|
||||
// let mut starting_page_index = pages_meta.len();
|
||||
// let mut values = vec![];
|
||||
// let mut truncate_at = None;
|
||||
|
||||
let last_page_index = pages_meta.len() - 1;
|
||||
// if self.stored_len() % Self::PER_PAGE != 0 {
|
||||
// if pages_meta.is_empty() {
|
||||
// unreachable!()
|
||||
// }
|
||||
|
||||
let mmap = unsafe { Mmap::map(&file)? };
|
||||
// let last_page_index = pages_meta.len() - 1;
|
||||
|
||||
values = Self::decode_page_(stored_len, last_page_index, &mmap, &pages_meta)
|
||||
.inspect_err(|_| {
|
||||
dbg!(last_page_index, &pages_meta);
|
||||
})
|
||||
.unwrap();
|
||||
// let mmap = unsafe { Mmap::map(&file)? };
|
||||
|
||||
truncate_at.replace(pages_meta.pop().unwrap().start);
|
||||
starting_page_index = last_page_index;
|
||||
}
|
||||
// values = Self::decode_page_(stored_len, last_page_index, &mmap, &pages_meta)
|
||||
// .inspect_err(|_| {
|
||||
// dbg!(last_page_index, &pages_meta);
|
||||
// })
|
||||
// .unwrap();
|
||||
|
||||
let compressed = values
|
||||
.into_par_iter()
|
||||
.chain(mem::take(self.mut_pushed()).into_par_iter())
|
||||
.chunks(Self::PER_PAGE)
|
||||
.map(|chunk| (Self::compress_page(chunk.as_ref()), chunk.len()))
|
||||
.collect::<Vec<_>>();
|
||||
// truncate_at.replace(pages_meta.pop().unwrap().start);
|
||||
// starting_page_index = last_page_index;
|
||||
// }
|
||||
|
||||
compressed
|
||||
.iter()
|
||||
.enumerate()
|
||||
.for_each(|(i, (compressed_bytes, values_len))| {
|
||||
let page_index = starting_page_index + i;
|
||||
// let compressed = values
|
||||
// .into_par_iter()
|
||||
// .chain(mem::take(self.mut_pushed()).into_par_iter())
|
||||
// .chunks(Self::PER_PAGE)
|
||||
// .map(|chunk| (Self::compress_page(chunk.as_ref()), chunk.len()))
|
||||
// .collect::<Vec<_>>();
|
||||
|
||||
let start = if page_index != 0 {
|
||||
let prev = pages_meta.get(page_index - 1).unwrap();
|
||||
prev.start + prev.bytes_len as u64
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let offsetted_start = start + HEADER_OFFSET as u64;
|
||||
// compressed
|
||||
// .iter()
|
||||
// .enumerate()
|
||||
// .for_each(|(i, (compressed_bytes, values_len))| {
|
||||
// let page_index = starting_page_index + i;
|
||||
|
||||
let bytes_len = compressed_bytes.len() as u32;
|
||||
let values_len = *values_len as u32;
|
||||
// let start = if page_index != 0 {
|
||||
// let prev = pages_meta.get(page_index - 1).unwrap();
|
||||
// prev.start + prev.bytes_len as u64
|
||||
// } else {
|
||||
// 0
|
||||
// };
|
||||
// let offsetted_start = start + HEADER_OFFSET as u64;
|
||||
|
||||
let page = CompressedPageMetadata::new(offsetted_start, bytes_len, values_len);
|
||||
// let bytes_len = compressed_bytes.len() as u32;
|
||||
// let values_len = *values_len as u32;
|
||||
|
||||
pages_meta.push(page_index, page);
|
||||
});
|
||||
// let page = CompressedPageMetadata::new(offsetted_start, bytes_len, values_len);
|
||||
|
||||
let buf = compressed
|
||||
.into_iter()
|
||||
.flat_map(|(v, _)| v)
|
||||
.collect::<Vec<_>>();
|
||||
// pages_meta.push(page_index, page);
|
||||
// });
|
||||
|
||||
pages_meta.write()?;
|
||||
// let buf = compressed
|
||||
// .into_iter()
|
||||
// .flat_map(|(v, _)| v)
|
||||
// .collect::<Vec<_>>();
|
||||
|
||||
if let Some(truncate_at) = truncate_at {
|
||||
self.file_set_len(&mut file, truncate_at)?;
|
||||
}
|
||||
// pages_meta.write()?;
|
||||
|
||||
self.file_write_all(&mut file, &buf)?;
|
||||
// if let Some(truncate_at) = truncate_at {
|
||||
// self.file_set_len(&mut file, truncate_at)?;
|
||||
// }
|
||||
|
||||
self.pages_meta.store(Arc::new(pages_meta));
|
||||
// self.file_write_all(&mut file, &buf)?;
|
||||
|
||||
// self.pages_meta.store(Arc::new(pages_meta));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> Result<()> {
|
||||
let mut pages_meta = (**self.pages_meta.load()).clone();
|
||||
pages_meta.truncate(0);
|
||||
pages_meta.write()?;
|
||||
self.pages_meta.store(Arc::new(pages_meta));
|
||||
// let mut pages_meta = (**self.pages_meta.load()).clone();
|
||||
// pages_meta.truncate(0);
|
||||
// pages_meta.write()?;
|
||||
// self.pages_meta.store(Arc::new(pages_meta));
|
||||
self.reset_()
|
||||
}
|
||||
|
||||
@@ -341,15 +342,14 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut pages_meta = (**self.pages_meta.load()).clone();
|
||||
let mut pages_meta = self.pages_meta.write();
|
||||
|
||||
let page_index = Self::index_to_page_index(index);
|
||||
|
||||
let mut file = self.open_file()?;
|
||||
let reader = self.create_static_reader();
|
||||
let values = self.decode_page(page_index, &reader)?;
|
||||
drop(reader);
|
||||
|
||||
let mmap = unsafe { Mmap::map(&file)? };
|
||||
|
||||
let values = self.decode_page(page_index, &mmap)?;
|
||||
let mut buf = vec![];
|
||||
|
||||
let mut page = pages_meta.truncate(page_index).unwrap();
|
||||
@@ -371,9 +371,7 @@ where
|
||||
|
||||
pages_meta.write()?;
|
||||
|
||||
self.pages_meta.store(Arc::new(pages_meta));
|
||||
|
||||
self.file_truncate_and_write_all(&mut file, len, &buf)?;
|
||||
// self.file_truncate_and_write_all(&mut file, len, &buf)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -422,10 +420,9 @@ impl<I, T> Clone for CompressedVec<I, T> {
|
||||
#[derive(Debug)]
|
||||
pub struct CompressedVecIterator<'a, I, T> {
|
||||
vec: &'a CompressedVec<I, T>,
|
||||
mmap: Mmap,
|
||||
reader: Reader<'a>,
|
||||
decoded_page: Option<(usize, Vec<T>)>,
|
||||
// second_decoded_page?: Option<(usize, Vec<T>)>,
|
||||
pages_meta: Guard<Arc<CompressedPagesMetadata>>,
|
||||
pages_meta: RwLockReadGuard<'a, CompressedPagesMetadata>,
|
||||
stored_len: usize,
|
||||
index: usize,
|
||||
}
|
||||
@@ -468,7 +465,6 @@ where
|
||||
type Item = (I, Cow<'a, T>);
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let mmap = &self.mmap;
|
||||
let i = self.index;
|
||||
let stored_len = self.stored_len;
|
||||
|
||||
@@ -488,7 +484,7 @@ where
|
||||
let values = CompressedVec::<I, T>::decode_page_(
|
||||
stored_len,
|
||||
page_index,
|
||||
mmap,
|
||||
&self.reader,
|
||||
&self.pages_meta,
|
||||
)
|
||||
.unwrap();
|
||||
@@ -518,12 +514,12 @@ where
|
||||
type IntoIter = CompressedVecIterator<'a, I, T>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
let pages_meta = self.pages_meta.load();
|
||||
let pages_meta = self.pages_meta.read();
|
||||
let stored_len = self.stored_len();
|
||||
|
||||
CompressedVecIterator {
|
||||
vec: self,
|
||||
mmap: self.create_mmap().unwrap(),
|
||||
reader: self.create_static_reader(),
|
||||
decoded_page: None,
|
||||
pages_meta,
|
||||
index: 0,
|
||||
@@ -0,0 +1,21 @@
|
||||
use clap_derive::ValueEnum;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(
|
||||
Default, Debug, PartialEq, PartialOrd, Ord, Eq, Clone, Copy, Serialize, Deserialize, ValueEnum,
|
||||
)]
|
||||
pub enum Computation {
|
||||
Eager,
|
||||
#[default]
|
||||
Lazy,
|
||||
}
|
||||
|
||||
impl Computation {
|
||||
pub fn eager(&self) -> bool {
|
||||
*self == Self::Eager
|
||||
}
|
||||
|
||||
pub fn lazy(&self) -> bool {
|
||||
*self == Self::Lazy
|
||||
}
|
||||
}
|
||||
+14
-37
@@ -1,14 +1,11 @@
|
||||
use std::{borrow::Cow, fs, path::Path};
|
||||
|
||||
use brk_exit::Exit;
|
||||
use clap_derive::ValueEnum;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{borrow::Cow, sync::Arc};
|
||||
|
||||
use brk_core::{Result, StoredPhantom, Version};
|
||||
use brk_exit::Exit;
|
||||
|
||||
use crate::{
|
||||
AnyCollectableVec, AnyIterableVec, AnyVec, BaseVecIterator, BoxedAnyIterableVec,
|
||||
BoxedVecIterator, CollectableVec, Format, StoredIndex, StoredType,
|
||||
BoxedVecIterator, CollectableVec, File, Format, StoredIndex, StoredType,
|
||||
};
|
||||
|
||||
use super::{
|
||||
@@ -16,24 +13,9 @@ use super::{
|
||||
LazyVecFrom2, LazyVecFrom2Iterator, LazyVecFrom3, LazyVecFrom3Iterator, StoredVecIterator,
|
||||
};
|
||||
|
||||
#[derive(
|
||||
Default, Debug, PartialEq, PartialOrd, Ord, Eq, Clone, Copy, Serialize, Deserialize, ValueEnum,
|
||||
)]
|
||||
pub enum Computation {
|
||||
Eager,
|
||||
#[default]
|
||||
Lazy,
|
||||
}
|
||||
mod computation;
|
||||
|
||||
impl Computation {
|
||||
pub fn eager(&self) -> bool {
|
||||
*self == Self::Eager
|
||||
}
|
||||
|
||||
pub fn lazy(&self) -> bool {
|
||||
*self == Self::Lazy
|
||||
}
|
||||
}
|
||||
pub use computation::*;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum Dependencies<I, T, S1I, S1T, S2I, S2T, S3I, S3T>
|
||||
@@ -93,7 +75,7 @@ where
|
||||
{
|
||||
pub fn forced_import_or_init_from_1(
|
||||
computation: Computation,
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
version: Version,
|
||||
format: Format,
|
||||
@@ -102,11 +84,10 @@ where
|
||||
) -> Result<Self> {
|
||||
Ok(match computation {
|
||||
Computation::Eager => Self::Eager {
|
||||
vec: EagerVec::forced_import(path, name, version, format)?,
|
||||
vec: EagerVec::forced_import(file, name, version, format)?,
|
||||
deps: Dependencies::From1(source, compute),
|
||||
},
|
||||
Computation::Lazy => {
|
||||
let _ = fs::remove_dir_all(path.join(name).join(I::to_string()));
|
||||
Self::LazyFrom1(LazyVecFrom1::init(name, version, source, compute))
|
||||
}
|
||||
})
|
||||
@@ -115,7 +96,7 @@ where
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import_or_init_from_2(
|
||||
computation: Computation,
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
version: Version,
|
||||
format: Format,
|
||||
@@ -125,11 +106,10 @@ where
|
||||
) -> Result<Self> {
|
||||
Ok(match computation {
|
||||
Computation::Eager => Self::Eager {
|
||||
vec: EagerVec::forced_import(path, name, version, format)?,
|
||||
vec: EagerVec::forced_import(file, name, version, format)?,
|
||||
deps: Dependencies::From2((source1, source2), compute),
|
||||
},
|
||||
Computation::Lazy => {
|
||||
let _ = fs::remove_dir_all(path.join(name).join(I::to_string()));
|
||||
Self::LazyFrom2(LazyVecFrom2::init(name, version, source1, source2, compute))
|
||||
}
|
||||
})
|
||||
@@ -138,7 +118,7 @@ where
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn forced_import_or_init_from_3(
|
||||
computation: Computation,
|
||||
path: &Path,
|
||||
file: &Arc<File>,
|
||||
name: &str,
|
||||
version: Version,
|
||||
format: Format,
|
||||
@@ -149,15 +129,12 @@ where
|
||||
) -> Result<Self> {
|
||||
Ok(match computation {
|
||||
Computation::Eager => Self::Eager {
|
||||
vec: EagerVec::forced_import(path, name, version, format)?,
|
||||
vec: EagerVec::forced_import(file, name, version, format)?,
|
||||
deps: Dependencies::From3((source1, source2, source3), compute),
|
||||
},
|
||||
Computation::Lazy => {
|
||||
let _ = fs::remove_dir_all(path.join(name).join(I::to_string()));
|
||||
Self::LazyFrom3(LazyVecFrom3::init(
|
||||
name, version, source1, source2, source3, compute,
|
||||
))
|
||||
}
|
||||
Computation::Lazy => Self::LazyFrom3(LazyVecFrom3::init(
|
||||
name, version, source1, source2, source3, compute,
|
||||
)),
|
||||
})
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user