diff --git a/Cargo.lock b/Cargo.lock index f88b91dcd..d6d2c6612 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,9 +139,9 @@ checksum = "7992085ec035cfe96992dd31bfd495a2ebd31969bb95f624471cb6c0b349e571" [[package]] name = "arbitrary" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" dependencies = [ "derive_arbitrary", ] @@ -193,13 +193,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.88" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -684,11 +684,11 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19a9e21ea5789ad190ce5c572bdbba50e589f9dd01acd19600b86442b56f02d" dependencies = [ - "darling 0.21.1", + "darling 0.21.2", "proc-macro2", "quote", "serde_json", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1141,9 +1141,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.32" +version = "1.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" dependencies = [ "jobserver", "libc", @@ -1218,7 +1218,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1454,12 +1454,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b136475da5ef7b6ac596c0e956e37bad51b85b987ff3d5e230e964936736b2" +checksum = "08440b3dd222c3d0433e63e097463969485f112baff337dfdaca043a0d760570" dependencies = [ - "darling_core 0.21.1", - "darling_macro 0.21.1", + "darling_core 0.21.2", + "darling_macro 0.21.2", ] [[package]] @@ -1473,21 +1473,21 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "darling_core" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b44ad32f92b75fb438b04b68547e521a548be8acc339a6dacc4a7121488f53e6" +checksum = "d25b7912bc28a04ab1b7715a68ea03aaa15662b43a1a4b2c480531fd19f8bf7e" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1498,18 +1498,18 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "darling_macro" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b5be8a7a562d315a5b92a630c30cec6bcf663e6673f00fbb69cca66a6f521b9" +checksum = "ce154b9bea7fb0c8e8326e62d00354000c36e79770ff21b8c84e3aa267d9d531" dependencies = [ - "darling_core 0.21.1", + "darling_core 0.21.2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1538,13 +1538,13 @@ dependencies = [ [[package]] name = "derive_arbitrary" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1575,7 +1575,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "unicode-xid", ] @@ -1597,7 +1597,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1651,7 +1651,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1861,7 +1861,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -2338,7 +2338,7 @@ checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -2479,9 +2479,9 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "lsm-tree" -version = "2.10.2" +version = "2.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55b6d7475a8dd22e749186968daacf8e2a77932b061b1bd263157987bbfc0c6c" +checksum = "ab73c02eadb3dc12c0024e5b61d6284e6d59064e67e74fbad77856caa56f62c7" dependencies = [ "byteorder", "crossbeam-skiplist", @@ -2738,9 +2738,9 @@ dependencies = [ [[package]] name = "oxc-browserslist" -version = "2.0.15" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a174f4f965eb2aef7bbd356b0fc8f4e0f251d4c52f437c211ff3d3b9d3ec7fba" +checksum = "bd843fdc95833f2faf4251b701a812d488ef35958144af91b9ff540d9fe8ceaa" dependencies = [ "bincode", "flate2", @@ -2774,7 +2774,7 @@ checksum = "a6eabb57f935b454fbe0552ea0abaaf9eb0019b5fa05a7bbe7efd5bd8c765085" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -2817,7 +2817,7 @@ dependencies = [ "phf", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3323,7 +3323,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3352,7 +3352,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3451,7 +3451,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3576,9 +3576,9 @@ checksum = "efee4b7317469c6c6e7fdeee3d094313af846a97678d6ed971d83a852d730083" [[package]] name = "rayon" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" dependencies = [ "either", "rayon-core", @@ -3586,9 +3586,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -3620,7 +3620,7 @@ checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3799,7 +3799,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3863,7 +3863,7 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "seqdb" -version = "0.1.1" +version = "0.1.2" dependencies = [ "libc", "log", @@ -3900,7 +3900,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3911,7 +3911,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3987,7 +3987,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -4124,9 +4124,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.104" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", @@ -4147,7 +4147,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -4171,7 +4171,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -4242,7 +4242,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -4253,7 +4253,7 @@ checksum = "cc5b44b4ab9c2fdd0e0512e6bece8388e214c0749f5862b114cc5b7a25daf227" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -4332,7 +4332,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -4466,7 +4466,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -4556,7 +4556,7 @@ checksum = "e9d4ed7b4c18cc150a6a0a1e9ea1ecfa688791220781af6e119f9599a8502a0a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "termcolor", ] @@ -4688,7 +4688,7 @@ checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" [[package]] name = "vecdb" -version = "0.1.1" +version = "0.1.2" dependencies = [ "ctrlc", "log", @@ -4706,10 +4706,10 @@ dependencies = [ [[package]] name = "vecdb_derive" -version = "0.1.1" +version = "0.1.2" dependencies = [ "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -4786,7 +4786,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "wasm-bindgen-shared", ] @@ -4808,7 +4808,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4880,7 +4880,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -4891,7 +4891,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -5127,7 +5127,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "synstructure", ] @@ -5148,7 +5148,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -5168,7 +5168,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "synstructure", ] @@ -5202,7 +5202,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 398272080..dabe0883e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,14 +46,14 @@ jiff = "0.2.15" log = "0.4.27" minreq = { version = "2.14.0", features = ["https", "serde_json"] } parking_lot = "0.12.4" -rayon = "1.10.0" +rayon = "1.11.0" serde = "1.0.219" serde_bytes = "0.11.17" serde_derive = "1.0.219" serde_json = { version = "1.0.142", features = ["float_roundtrip"] } tokio = { version = "1.47.1", features = ["rt-multi-thread"] } vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]} -# vecdb = { version = "0.1.1", features = ["derive"]} +# vecdb = { version = "0.1.2", features = ["derive"]} zerocopy = "0.8.26" zerocopy-derive = "0.8.26" diff --git a/crates/brk_computer/src/blocks.rs b/crates/brk_computer/src/blocks.rs index 76126a72e..0f078ae47 100644 --- a/crates/brk_computer/src/blocks.rs +++ b/crates/brk_computer/src/blocks.rs @@ -6,7 +6,7 @@ use brk_structs::{ CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, Timestamp, Version, Weight, }; -use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, Format, PAGE_SIZE, VecIterator}; +use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, PAGE_SIZE, VecIterator}; use crate::grouped::Source; @@ -35,21 +35,15 @@ pub struct Vecs { } impl Vecs { - pub fn forced_import( - parent: &Path, - version: Version, - format: Format, - indexes: &indexes::Vecs, - ) -> Result { + pub fn forced_import(parent: &Path, version: Version, indexes: &indexes::Vecs) -> Result { let db = Database::open(&parent.join("blocks"))?; db.set_min_len(PAGE_SIZE * 1_000_000)?; Ok(Self { - height_to_interval: EagerVec::forced_import( + height_to_interval: EagerVec::forced_import_compressed( &db, "interval", version + VERSION + Version::ZERO, - format, )?, timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import( &db, @@ -94,11 +88,10 @@ impl Vecs { indexes, VecBuilderOptions::default().add_sum().add_cumulative(), )?, - height_to_vbytes: EagerVec::forced_import( + height_to_vbytes: EagerVec::forced_import_compressed( &db, "vbytes", version + VERSION + Version::ZERO, - format, )?, indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import( &db, @@ -108,17 +101,15 @@ impl Vecs { indexes, VecBuilderOptions::default().add_sum().add_cumulative(), )?, - difficultyepoch_to_timestamp: EagerVec::forced_import( + difficultyepoch_to_timestamp: EagerVec::forced_import_compressed( &db, "timestamp", version + VERSION + Version::ZERO, - format, )?, - halvingepoch_to_timestamp: EagerVec::forced_import( + halvingepoch_to_timestamp: EagerVec::forced_import_compressed( &db, "timestamp", version + VERSION + Version::ZERO, - format, )?, db, diff --git a/crates/brk_computer/src/grouped/builder_eager.rs b/crates/brk_computer/src/grouped/builder_eager.rs index b1de1dc85..0e483cdb3 100644 --- a/crates/brk_computer/src/grouped/builder_eager.rs +++ b/crates/brk_computer/src/grouped/builder_eager.rs @@ -36,6 +36,15 @@ where I: StoredIndex, T: ComputedType, { + pub fn forced_import_compressed( + db: &Database, + name: &str, + version: Version, + options: VecBuilderOptions, + ) -> Result { + Self::forced_import(db, name, version, Format::Compressed, options) + } + pub fn forced_import( db: &Database, name: &str, diff --git a/crates/brk_computer/src/grouped/from_dateindex.rs b/crates/brk_computer/src/grouped/from_dateindex.rs index ee85b88bb..562230ae2 100644 --- a/crates/brk_computer/src/grouped/from_dateindex.rs +++ b/crates/brk_computer/src/grouped/from_dateindex.rs @@ -4,9 +4,7 @@ use brk_indexer::Indexer; use brk_structs::{ DateIndex, DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex, }; -use vecdb::{ - AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit, Format, -}; +use vecdb::{AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit}; use crate::{Indexes, grouped::LazyVecBuilder, indexes}; @@ -42,17 +40,14 @@ where indexes: &indexes::Vecs, options: VecBuilderOptions, ) -> Result { - let format = Format::Compressed; - let dateindex = source.is_compute().then(|| { - EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format).unwrap() + EagerVec::forced_import_compressed(db, name, version + VERSION + Version::ZERO).unwrap() }); - let dateindex_extra = EagerVecBuilder::forced_import( + let dateindex_extra = EagerVecBuilder::forced_import_compressed( db, name, version + VERSION + Version::ZERO, - format, options.copy_self_extra(), )?; diff --git a/crates/brk_computer/src/grouped/from_height.rs b/crates/brk_computer/src/grouped/from_height.rs index 033e4e5b4..7b55656d2 100644 --- a/crates/brk_computer/src/grouped/from_height.rs +++ b/crates/brk_computer/src/grouped/from_height.rs @@ -5,9 +5,7 @@ use brk_structs::{ DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex, }; -use vecdb::{ - AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit, Format, -}; +use vecdb::{AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit}; use crate::{ Indexes, @@ -51,25 +49,21 @@ where indexes: &indexes::Vecs, options: VecBuilderOptions, ) -> Result { - let format = Format::Compressed; - let height = source.is_compute().then(|| { - EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format).unwrap() + EagerVec::forced_import_compressed(db, name, version + VERSION + Version::ZERO).unwrap() }); - let height_extra = EagerVecBuilder::forced_import( + let height_extra = EagerVecBuilder::forced_import_compressed( db, name, version + VERSION + Version::ZERO, - format, options.copy_self_extra(), )?; - let dateindex = EagerVecBuilder::forced_import( + let dateindex = EagerVecBuilder::forced_import_compressed( db, name, version + VERSION + Version::ZERO, - format, options, )?; @@ -128,11 +122,10 @@ where height, height_extra, dateindex, - difficultyepoch: EagerVecBuilder::forced_import( + difficultyepoch: EagerVecBuilder::forced_import_compressed( db, name, version + VERSION + Version::ZERO, - format, options, )?, }) diff --git a/crates/brk_computer/src/grouped/from_height_strict.rs b/crates/brk_computer/src/grouped/from_height_strict.rs index 49671d629..c3c156c14 100644 --- a/crates/brk_computer/src/grouped/from_height_strict.rs +++ b/crates/brk_computer/src/grouped/from_height_strict.rs @@ -2,7 +2,7 @@ use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{DifficultyEpoch, Height, Version}; -use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, Format}; +use vecdb::{AnyCollectableVec, Database, EagerVec, Exit}; use crate::{Indexes, indexes}; @@ -32,15 +32,13 @@ where version: Version, options: VecBuilderOptions, ) -> Result { - let format = Format::Compressed; + let height = + EagerVec::forced_import_compressed(db, name, version + VERSION + Version::ZERO)?; - let height = EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)?; - - let height_extra = EagerVecBuilder::forced_import( + let height_extra = EagerVecBuilder::forced_import_compressed( db, name, version + VERSION + Version::ZERO, - format, options.copy_self_extra(), )?; @@ -49,11 +47,10 @@ where Ok(Self { height, height_extra, - difficultyepoch: EagerVecBuilder::forced_import( + difficultyepoch: EagerVecBuilder::forced_import_compressed( db, name, version + VERSION + Version::ZERO, - format, options, )?, // halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?, diff --git a/crates/brk_computer/src/grouped/from_txindex.rs b/crates/brk_computer/src/grouped/from_txindex.rs index 835769efb..9e5c6d75d 100644 --- a/crates/brk_computer/src/grouped/from_txindex.rs +++ b/crates/brk_computer/src/grouped/from_txindex.rs @@ -6,7 +6,7 @@ use brk_structs::{ }; use vecdb::{ AnyCloneableIterableVec, AnyCollectableVec, AnyVec, CollectableVec, Database, EagerVec, Exit, - Format, GenericStoredVec, StoredIndex, VecIterator, + GenericStoredVec, StoredIndex, VecIterator, }; use crate::{ @@ -48,32 +48,29 @@ where name: &str, source: Source, version: Version, - format: Format, indexes: &indexes::Vecs, options: VecBuilderOptions, ) -> Result { let txindex = source.is_compute().then(|| { Box::new( - EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format) + EagerVec::forced_import_compressed(db, name, version + VERSION + Version::ZERO) .unwrap(), ) }); - let height = EagerVecBuilder::forced_import( + let height = EagerVecBuilder::forced_import_compressed( db, name, version + VERSION + Version::ZERO, - format, options, )?; let options = options.remove_percentiles(); - let dateindex = EagerVecBuilder::forced_import( + let dateindex = EagerVecBuilder::forced_import_compressed( db, name, version + VERSION + Version::ZERO, - format, options, )?; @@ -130,11 +127,10 @@ where txindex, height, dateindex, - difficultyepoch: EagerVecBuilder::forced_import( + difficultyepoch: EagerVecBuilder::forced_import_compressed( db, name, version + VERSION + Version::ZERO, - format, options, )?, // halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?, diff --git a/crates/brk_computer/src/grouped/value_from_txindex.rs b/crates/brk_computer/src/grouped/value_from_txindex.rs index 412a00eae..0a3ee5210 100644 --- a/crates/brk_computer/src/grouped/value_from_txindex.rs +++ b/crates/brk_computer/src/grouped/value_from_txindex.rs @@ -2,8 +2,8 @@ use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version}; use vecdb::{ - AnyCloneableIterableVec, AnyCollectableVec, CollectableVec, Database, Exit, Format, - LazyVecFrom1, LazyVecFrom3, StoredIndex, StoredVec, + AnyCloneableIterableVec, AnyCollectableVec, CollectableVec, Database, Exit, LazyVecFrom1, + LazyVecFrom3, StoredIndex, StoredVec, }; use crate::{Indexes, grouped::Source, indexes, price}; @@ -32,7 +32,6 @@ impl ComputedValueVecsFromTxindex { indexes: &indexes::Vecs, source: Source, version: Version, - format: Format, price: Option<&price::Vecs>, options: VecBuilderOptions, ) -> Result { @@ -46,7 +45,6 @@ impl ComputedValueVecsFromTxindex { name, source.clone(), version + VERSION, - format, indexes, options, )?; @@ -70,7 +68,6 @@ impl ComputedValueVecsFromTxindex { &name_in_btc, Source::None, version + VERSION, - format, indexes, options, )?; @@ -113,7 +110,6 @@ impl ComputedValueVecsFromTxindex { &name_in_usd, Source::None, version + VERSION, - format, indexes, options, ) diff --git a/crates/brk_computer/src/indexes.rs b/crates/brk_computer/src/indexes.rs index aa1672134..671821374 100644 --- a/crates/brk_computer/src/indexes.rs +++ b/crates/brk_computer/src/indexes.rs @@ -11,7 +11,7 @@ use brk_structs::{ Timestamp, TxIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex, }; use vecdb::{ - AnyCloneableIterableVec, AnyCollectableVec, Database, EagerVec, Exit, Format, LazyVecFrom1, + AnyCloneableIterableVec, AnyCollectableVec, Database, EagerVec, Exit, LazyVecFrom1, LazyVecFrom2, PAGE_SIZE, StoredIndex, VecIterator, }; @@ -255,263 +255,220 @@ impl Vecs { txindex_to_txindex, unknownoutputindex_to_unknownoutputindex, - dateindex_to_date: EagerVec::forced_import( + dateindex_to_date: EagerVec::forced_import_compressed( &db, "date", version + VERSION + Version::ZERO, - Format::Compressed, )?, - dateindex_to_dateindex: EagerVec::forced_import( + dateindex_to_dateindex: EagerVec::forced_import_compressed( &db, "dateindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - dateindex_to_first_height: EagerVec::forced_import( + dateindex_to_first_height: EagerVec::forced_import_compressed( &db, "first_height", version + VERSION + Version::ZERO, - Format::Compressed, )?, - dateindex_to_monthindex: EagerVec::forced_import( + dateindex_to_monthindex: EagerVec::forced_import_compressed( &db, "monthindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - dateindex_to_weekindex: EagerVec::forced_import( + dateindex_to_weekindex: EagerVec::forced_import_compressed( &db, "weekindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - decadeindex_to_decadeindex: EagerVec::forced_import( + decadeindex_to_decadeindex: EagerVec::forced_import_compressed( &db, "decadeindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - decadeindex_to_first_yearindex: EagerVec::forced_import( + decadeindex_to_first_yearindex: EagerVec::forced_import_compressed( &db, "first_yearindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - difficultyepoch_to_difficultyepoch: EagerVec::forced_import( + difficultyepoch_to_difficultyepoch: EagerVec::forced_import_compressed( &db, "difficultyepoch", version + VERSION + Version::ZERO, - Format::Compressed, )?, - difficultyepoch_to_first_height: EagerVec::forced_import( + difficultyepoch_to_first_height: EagerVec::forced_import_compressed( &db, "first_height", version + VERSION + Version::ZERO, - Format::Compressed, )?, - halvingepoch_to_first_height: EagerVec::forced_import( + halvingepoch_to_first_height: EagerVec::forced_import_compressed( &db, "first_height", version + VERSION + Version::ZERO, - Format::Compressed, )?, - halvingepoch_to_halvingepoch: EagerVec::forced_import( + halvingepoch_to_halvingepoch: EagerVec::forced_import_compressed( &db, "halvingepoch", version + VERSION + Version::ZERO, - Format::Compressed, )?, - height_to_date: EagerVec::forced_import( + height_to_date: EagerVec::forced_import_compressed( &db, "date", version + VERSION + Version::ZERO, - Format::Compressed, )?, - height_to_difficultyepoch: EagerVec::forced_import( + height_to_difficultyepoch: EagerVec::forced_import_compressed( &db, "difficultyepoch", version + VERSION + Version::ZERO, - Format::Compressed, )?, - height_to_halvingepoch: EagerVec::forced_import( + height_to_halvingepoch: EagerVec::forced_import_compressed( &db, "halvingepoch", version + VERSION + Version::ZERO, - Format::Compressed, )?, - height_to_height: EagerVec::forced_import( + height_to_height: EagerVec::forced_import_compressed( &db, "height", version + VERSION + Version::ZERO, - Format::Compressed, )?, - monthindex_to_first_dateindex: EagerVec::forced_import( + monthindex_to_first_dateindex: EagerVec::forced_import_compressed( &db, "first_dateindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - monthindex_to_monthindex: EagerVec::forced_import( + monthindex_to_monthindex: EagerVec::forced_import_compressed( &db, "monthindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - monthindex_to_quarterindex: EagerVec::forced_import( + monthindex_to_quarterindex: EagerVec::forced_import_compressed( &db, "quarterindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - monthindex_to_semesterindex: EagerVec::forced_import( + monthindex_to_semesterindex: EagerVec::forced_import_compressed( &db, "semesterindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - monthindex_to_yearindex: EagerVec::forced_import( + monthindex_to_yearindex: EagerVec::forced_import_compressed( &db, "yearindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - quarterindex_to_first_monthindex: EagerVec::forced_import( + quarterindex_to_first_monthindex: EagerVec::forced_import_compressed( &db, "first_monthindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - semesterindex_to_first_monthindex: EagerVec::forced_import( + semesterindex_to_first_monthindex: EagerVec::forced_import_compressed( &db, "first_monthindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - weekindex_to_first_dateindex: EagerVec::forced_import( + weekindex_to_first_dateindex: EagerVec::forced_import_compressed( &db, "first_dateindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - yearindex_to_first_monthindex: EagerVec::forced_import( + yearindex_to_first_monthindex: EagerVec::forced_import_compressed( &db, "first_monthindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - quarterindex_to_quarterindex: EagerVec::forced_import( + quarterindex_to_quarterindex: EagerVec::forced_import_compressed( &db, "quarterindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - semesterindex_to_semesterindex: EagerVec::forced_import( + semesterindex_to_semesterindex: EagerVec::forced_import_compressed( &db, "semesterindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - weekindex_to_weekindex: EagerVec::forced_import( + weekindex_to_weekindex: EagerVec::forced_import_compressed( &db, "weekindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - yearindex_to_decadeindex: EagerVec::forced_import( + yearindex_to_decadeindex: EagerVec::forced_import_compressed( &db, "decadeindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - yearindex_to_yearindex: EagerVec::forced_import( + yearindex_to_yearindex: EagerVec::forced_import_compressed( &db, "yearindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - height_to_date_fixed: EagerVec::forced_import( + height_to_date_fixed: EagerVec::forced_import_compressed( &db, "date_fixed", version + VERSION + Version::ZERO, - Format::Compressed, )?, - height_to_dateindex: EagerVec::forced_import( + height_to_dateindex: EagerVec::forced_import_compressed( &db, "dateindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, - txindex_to_height: EagerVec::forced_import( + txindex_to_height: EagerVec::forced_import_compressed( &db, "height", version + VERSION + Version::ZERO, - Format::Compressed, )?, - height_to_timestamp_fixed: EagerVec::forced_import( + height_to_timestamp_fixed: EagerVec::forced_import_compressed( &db, "timestamp_fixed", version + VERSION + Version::ZERO, - Format::Compressed, )?, - height_to_txindex_count: EagerVec::forced_import( + height_to_txindex_count: EagerVec::forced_import_compressed( &db, "txindex_count", version + VERSION + Version::ZERO, - Format::Compressed, )?, - dateindex_to_height_count: EagerVec::forced_import( + dateindex_to_height_count: EagerVec::forced_import_compressed( &db, "height_count", version + VERSION + Version::ZERO, - Format::Compressed, )?, - weekindex_to_dateindex_count: EagerVec::forced_import( + weekindex_to_dateindex_count: EagerVec::forced_import_compressed( &db, "dateindex_count", version + VERSION + Version::ZERO, - Format::Compressed, )?, - difficultyepoch_to_height_count: EagerVec::forced_import( + difficultyepoch_to_height_count: EagerVec::forced_import_compressed( &db, "height_count", version + VERSION + Version::ZERO, - Format::Compressed, )?, - monthindex_to_dateindex_count: EagerVec::forced_import( + monthindex_to_dateindex_count: EagerVec::forced_import_compressed( &db, "dateindex_count", version + VERSION + Version::ZERO, - Format::Compressed, )?, - quarterindex_to_monthindex_count: EagerVec::forced_import( + quarterindex_to_monthindex_count: EagerVec::forced_import_compressed( &db, "monthindex_count", version + VERSION + Version::ZERO, - Format::Compressed, )?, - semesterindex_to_monthindex_count: EagerVec::forced_import( + semesterindex_to_monthindex_count: EagerVec::forced_import_compressed( &db, "monthindex_count", version + VERSION + Version::ZERO, - Format::Compressed, )?, - yearindex_to_monthindex_count: EagerVec::forced_import( + yearindex_to_monthindex_count: EagerVec::forced_import_compressed( &db, "monthindex_count", version + VERSION + Version::ZERO, - Format::Compressed, )?, - decadeindex_to_yearindex_count: EagerVec::forced_import( + decadeindex_to_yearindex_count: EagerVec::forced_import_compressed( &db, "yearindex_count", version + VERSION + Version::ZERO, - Format::Compressed, )?, - outputindex_to_txindex: EagerVec::forced_import( + outputindex_to_txindex: EagerVec::forced_import_compressed( &db, "txindex", version + VERSION + Version::ZERO, - Format::Compressed, )?, db, diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 528be4c49..c609f9839 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -60,36 +60,23 @@ impl Computer { fetched::Vecs::forced_import(outputs_path, fetcher, VERSION + Version::ZERO).unwrap() }); - let format = Format::Compressed; - let price = fetched.is_some().then(|| { - price::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, format, &indexes) - .unwrap() + price::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, &indexes).unwrap() }); Ok(Self { - blocks: blocks::Vecs::forced_import( - &computed_path, - VERSION + Version::ZERO, - format, - &indexes, - )?, + blocks: blocks::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, &indexes)?, mining: mining::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, &indexes)?, constants: constants::Vecs::forced_import( &computed_path, VERSION + Version::ZERO, &indexes, )?, - market: market::Vecs::forced_import( - &computed_path, - VERSION + Version::ZERO, - format, - &indexes, - )?, + market: market::Vecs::forced_import(&computed_path, VERSION + Version::ZERO, &indexes)?, stateful: stateful::Vecs::forced_import( &computed_path, VERSION + Version::ZERO, - format, + Format::Compressed, &indexes, price.as_ref(), &computed_path.join("states"), @@ -99,7 +86,6 @@ impl Computer { VERSION + Version::ZERO, indexer, &indexes, - format, price.as_ref(), )?, cointime: cointime::Vecs::forced_import( diff --git a/crates/brk_computer/src/market.rs b/crates/brk_computer/src/market.rs index 91e8725a7..fde4af2b8 100644 --- a/crates/brk_computer/src/market.rs +++ b/crates/brk_computer/src/market.rs @@ -3,9 +3,7 @@ use std::{path::Path, thread}; use brk_error::Result; use brk_indexer::Indexer; use brk_structs::{Date, DateIndex, Dollars, Height, Sats, StoredF32, StoredU16, Version}; -use vecdb::{ - AnyCollectableVec, Database, EagerVec, Exit, Format, PAGE_SIZE, StoredIndex, VecIterator, -}; +use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, PAGE_SIZE, StoredIndex, VecIterator}; use crate::{ grouped::Source, @@ -170,33 +168,25 @@ pub struct Vecs { } impl Vecs { - pub fn forced_import( - parent: &Path, - version: Version, - format: Format, - indexes: &indexes::Vecs, - ) -> Result { + pub fn forced_import(parent: &Path, version: Version, indexes: &indexes::Vecs) -> Result { let db = Database::open(&parent.join("market"))?; db.set_min_len(PAGE_SIZE * 1_000_000)?; Ok(Self { - height_to_marketcap: EagerVec::forced_import( + height_to_marketcap: EagerVec::forced_import_compressed( &db, "marketcap", version + VERSION + Version::ZERO, - format, )?, - height_to_ath: EagerVec::forced_import( + height_to_ath: EagerVec::forced_import_compressed( &db, "ath", version + VERSION + Version::ZERO, - format, )?, - height_to_drawdown: EagerVec::forced_import( + height_to_drawdown: EagerVec::forced_import_compressed( &db, "drawdown", version + VERSION + Version::ZERO, - format, )?, indexes_to_marketcap: ComputedVecsFromDateIndex::forced_import( &db, diff --git a/crates/brk_computer/src/price.rs b/crates/brk_computer/src/price.rs index 40e065b9e..ce471c49a 100644 --- a/crates/brk_computer/src/price.rs +++ b/crates/brk_computer/src/price.rs @@ -7,7 +7,7 @@ use brk_structs::{ OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, SemesterIndex, Version, WeekIndex, YearIndex, }; use vecdb::{ - AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit, Format, + AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit, GenericStoredVec, PAGE_SIZE, RawVec, }; @@ -73,12 +73,7 @@ const VERSION: Version = Version::ZERO; const VERSION_IN_SATS: Version = Version::ZERO; impl Vecs { - pub fn forced_import( - parent: &Path, - version: Version, - format: Format, - indexes: &indexes::Vecs, - ) -> Result { + pub fn forced_import(parent: &Path, version: Version, indexes: &indexes::Vecs) -> Result { let db = Database::open(&parent.join("price"))?; db.set_min_len(PAGE_SIZE * 1_000_000)?; @@ -93,29 +88,25 @@ impl Vecs { "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, - dateindex_to_close_in_cents: EagerVec::forced_import( + dateindex_to_close_in_cents: EagerVec::forced_import_compressed( &db, "close_in_cents", version + VERSION + Version::ZERO, - format, )?, - dateindex_to_high_in_cents: EagerVec::forced_import( + dateindex_to_high_in_cents: EagerVec::forced_import_compressed( &db, "high_in_cents", version + VERSION + Version::ZERO, - format, )?, - dateindex_to_low_in_cents: EagerVec::forced_import( + dateindex_to_low_in_cents: EagerVec::forced_import_compressed( &db, "low_in_cents", version + VERSION + Version::ZERO, - format, )?, - dateindex_to_open_in_cents: EagerVec::forced_import( + dateindex_to_open_in_cents: EagerVec::forced_import_compressed( &db, "open_in_cents", version + VERSION + Version::ZERO, - format, )?, height_to_ohlc: RawVec::forced_import(&db, "ohlc", version + VERSION + Version::ZERO)?, height_to_ohlc_in_sats: RawVec::forced_import( @@ -123,29 +114,25 @@ impl Vecs { "ohlc_in_sats", version + VERSION + VERSION_IN_SATS + Version::ZERO, )?, - height_to_close_in_cents: EagerVec::forced_import( + height_to_close_in_cents: EagerVec::forced_import_compressed( &db, "close_in_cents", version + VERSION + Version::ZERO, - format, )?, - height_to_high_in_cents: EagerVec::forced_import( + height_to_high_in_cents: EagerVec::forced_import_compressed( &db, "high_in_cents", version + VERSION + Version::ZERO, - format, )?, - height_to_low_in_cents: EagerVec::forced_import( + height_to_low_in_cents: EagerVec::forced_import_compressed( &db, "low_in_cents", version + VERSION + Version::ZERO, - format, )?, - height_to_open_in_cents: EagerVec::forced_import( + height_to_open_in_cents: EagerVec::forced_import_compressed( &db, "open_in_cents", version + VERSION + Version::ZERO, - format, )?, timeindexes_to_open: ComputedVecsFromDateIndex::forced_import( &db, diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index c54ee0cc8..5db15f31c 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -105,11 +105,10 @@ impl Vecs { version + VERSION + Version::ZERO, )?, - height_to_unspendable_supply: EagerVec::forced_import( + height_to_unspendable_supply: EagerVec::forced_import_compressed( &db, "unspendable_supply", version + VERSION + Version::ZERO, - format, )?, indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import( &db, @@ -120,11 +119,10 @@ impl Vecs { compute_dollars, indexes, )?, - height_to_opreturn_supply: EagerVec::forced_import( + height_to_opreturn_supply: EagerVec::forced_import_compressed( &db, "opreturn_supply", version + VERSION + Version::ZERO, - format, )?, indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import( &db, @@ -153,105 +151,89 @@ impl Vecs { )?, addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount::from( ByAddressType { - p2pk65: EagerVec::forced_import( + p2pk65: EagerVec::forced_import_compressed( &db, "p2pk65_address_count", version + VERSION + Version::ZERO, - format, )?, - p2pk33: EagerVec::forced_import( + p2pk33: EagerVec::forced_import_compressed( &db, "p2pk33_address_count", version + VERSION + Version::ZERO, - format, )?, - p2pkh: EagerVec::forced_import( + p2pkh: EagerVec::forced_import_compressed( &db, "p2pkh_address_count", version + VERSION + Version::ZERO, - format, )?, - p2sh: EagerVec::forced_import( + p2sh: EagerVec::forced_import_compressed( &db, "p2sh_address_count", version + VERSION + Version::ZERO, - format, )?, - p2wpkh: EagerVec::forced_import( + p2wpkh: EagerVec::forced_import_compressed( &db, "p2wpkh_address_count", version + VERSION + Version::ZERO, - format, )?, - p2wsh: EagerVec::forced_import( + p2wsh: EagerVec::forced_import_compressed( &db, "p2wsh_address_count", version + VERSION + Version::ZERO, - format, )?, - p2tr: EagerVec::forced_import( + p2tr: EagerVec::forced_import_compressed( &db, "p2tr_address_count", version + VERSION + Version::ZERO, - format, )?, - p2a: EagerVec::forced_import( + p2a: EagerVec::forced_import_compressed( &db, "p2a_address_count", version + VERSION + Version::ZERO, - format, )?, }, ), addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount::from( ByAddressType { - p2pk65: EagerVec::forced_import( + p2pk65: EagerVec::forced_import_compressed( &db, "p2pk65_empty_address_count", version + VERSION + Version::ZERO, - format, )?, - p2pk33: EagerVec::forced_import( + p2pk33: EagerVec::forced_import_compressed( &db, "p2pk33_empty_address_count", version + VERSION + Version::ZERO, - format, )?, - p2pkh: EagerVec::forced_import( + p2pkh: EagerVec::forced_import_compressed( &db, "p2pkh_empty_address_count", version + VERSION + Version::ZERO, - format, )?, - p2sh: EagerVec::forced_import( + p2sh: EagerVec::forced_import_compressed( &db, "p2sh_empty_address_count", version + VERSION + Version::ZERO, - format, )?, - p2wpkh: EagerVec::forced_import( + p2wpkh: EagerVec::forced_import_compressed( &db, "p2wpkh_empty_address_count", version + VERSION + Version::ZERO, - format, )?, - p2wsh: EagerVec::forced_import( + p2wsh: EagerVec::forced_import_compressed( &db, "p2wsh_empty_address_count", version + VERSION + Version::ZERO, - format, )?, - p2tr: EagerVec::forced_import( + p2tr: EagerVec::forced_import_compressed( &db, "p2tr_empty_address_count", version + VERSION + Version::ZERO, - format, )?, - p2a: EagerVec::forced_import( + p2a: EagerVec::forced_import_compressed( &db, "p2a_empty_address_count", version + VERSION + Version::ZERO, - format, )?, }, ), diff --git a/crates/brk_computer/src/states/price_to_amount.rs b/crates/brk_computer/src/states/price_to_amount.rs index fae4c7f29..868c5d946 100644 --- a/crates/brk_computer/src/states/price_to_amount.rs +++ b/crates/brk_computer/src/states/price_to_amount.rs @@ -2,7 +2,6 @@ use std::{ collections::BTreeMap, fs, path::{Path, PathBuf}, - time::SystemTime, }; use brk_error::Result; @@ -10,6 +9,7 @@ use brk_structs::{Dollars, Height, Sats}; use derive_deref::{Deref, DerefMut}; use pco::standalone::{simple_decompress, simpler_compress}; use serde::{Deserialize, Serialize}; +use zerocopy::{FromBytes, IntoBytes}; use crate::states::SupplyState; @@ -78,23 +78,31 @@ impl PriceToAmount { } pub fn flush(&mut self, height: Height) -> Result<()> { - let mut files: Vec<(SystemTime, PathBuf)> = fs::read_dir(&self.pathbuf)? + let files: BTreeMap = fs::read_dir(&self.pathbuf)? .filter_map(|entry| { let path = entry.ok()?.path(); let name = path.file_name()?.to_str()?; - if name.starts_with(STATE_AT_) && name[STATE_AT_.len() + 1..].parse::().is_ok() - { - let modified = fs::metadata(&path).ok()?.modified().ok()?; - Some((modified, path)) + if let Some(height_str) = name.strip_prefix(STATE_AT_) { + if let Ok(h) = height_str.parse::().map(Height::from) { + if h < height { + Some((h, path)) + } else { + let _ = fs::remove_file(path); + None + } + } else { + None + } } else { None } }) .collect(); - files.sort_unstable_by_key(|(time, _)| *time); - - for (_, path) in files.iter().take(files.len().saturating_sub(STATE_TO_KEEP)) { + for (_, path) in files + .iter() + .take(files.len().saturating_sub(STATE_TO_KEEP - 1)) + { fs::remove_file(path)?; } @@ -128,8 +136,8 @@ impl State { let compressed_values = simpler_compress(&values, COMPRESSION_LEVEL)?; let mut buffer = Vec::new(); - buffer.extend(&(keys.len() as u64).to_ne_bytes()); - buffer.extend(&(compressed_keys.len() as u64).to_ne_bytes()); + buffer.extend(keys.len().as_bytes()); + buffer.extend(compressed_keys.len().as_bytes()); buffer.extend(compressed_keys); buffer.extend(compressed_values); @@ -137,8 +145,8 @@ impl State { } fn deserialize(data: &[u8]) -> vecdb::Result { - let entry_count = u64::from_ne_bytes(data[0..8].try_into().unwrap()) as usize; - let keys_len = u64::from_ne_bytes(data[8..16].try_into().unwrap()) as usize; + let entry_count = usize::read_from_bytes(&data[0..8])?; + let keys_len = usize::read_from_bytes(&data[8..16])?; let keys: Vec = simple_decompress(&data[16..16 + keys_len])?; let values: Vec = simple_decompress(&data[16 + keys_len..])?; diff --git a/crates/brk_computer/src/transactions.rs b/crates/brk_computer/src/transactions.rs index 53b035e07..9c5af35f9 100644 --- a/crates/brk_computer/src/transactions.rs +++ b/crates/brk_computer/src/transactions.rs @@ -7,7 +7,7 @@ use brk_structs::{ StoredU32, StoredU64, TxIndex, TxVersion, Version, Weight, }; use vecdb::{ - AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit, Format, + AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Database, EagerVec, Exit, LazyVecFrom1, LazyVecFrom2, LazyVecFrom3, PAGE_SIZE, StoredIndex, VecIterator, }; @@ -75,7 +75,6 @@ impl Vecs { version: Version, indexer: &Indexer, indexes: &indexes::Vecs, - format: Format, price: Option<&price::Vecs>, ) -> Result { let db = Database::open(&parent.join("transactions"))?; @@ -260,19 +259,11 @@ impl Vecs { // .add_cumulative(), // )?; - let txindex_to_fee = EagerVec::forced_import( - &db, - "fee", - version + VERSION + Version::ZERO, - Format::Compressed, - )?; + let txindex_to_fee = + EagerVec::forced_import_compressed(&db, "fee", version + VERSION + Version::ZERO)?; - let txindex_to_feerate = EagerVec::forced_import( - &db, - "feerate", - version + VERSION + Version::ZERO, - Format::Compressed, - )?; + let txindex_to_feerate = + EagerVec::forced_import_compressed(&db, "feerate", version + VERSION + Version::ZERO)?; Ok(Self { indexes_to_tx_count: ComputedVecsFromHeight::forced_import( @@ -293,7 +284,6 @@ impl Vecs { "input_count", Source::None, version + VERSION + Version::ZERO, - format, indexes, VecBuilderOptions::default() .add_average() @@ -307,7 +297,6 @@ impl Vecs { "output_count", Source::None, version + VERSION + Version::ZERO, - format, indexes, VecBuilderOptions::default() .add_average() @@ -346,7 +335,6 @@ impl Vecs { indexes, Source::Vec(txindex_to_fee.boxed_clone()), version + VERSION + Version::ZERO, - format, price, VecBuilderOptions::default() .add_sum() @@ -360,7 +348,6 @@ impl Vecs { "feerate", Source::None, version + VERSION + Version::ZERO, - format, indexes, VecBuilderOptions::default() .add_percentiles() @@ -372,7 +359,6 @@ impl Vecs { "tx_vsize", Source::None, version + VERSION + Version::ZERO, - format, indexes, VecBuilderOptions::default() .add_percentiles() @@ -384,7 +370,6 @@ impl Vecs { "tx_weight", Source::None, version + VERSION + Version::ZERO, - format, indexes, VecBuilderOptions::default() .add_percentiles() diff --git a/crates/brk_indexer/src/lib.rs b/crates/brk_indexer/src/lib.rs index 0f2d109a5..b82897e79 100644 --- a/crates/brk_indexer/src/lib.rs +++ b/crates/brk_indexer/src/lib.rs @@ -782,16 +782,17 @@ impl Indexer { }, )?; + txindex_to_first_outputindex_reader_opt.take(); + p2pk65addressindex_to_p2pk65bytes_reader_opt.take(); + p2pk33addressindex_to_p2pk33bytes_reader_opt.take(); + p2pkhaddressindex_to_p2pkhbytes_reader_opt.take(); + p2shaddressindex_to_p2shbytes_reader_opt.take(); + p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take(); + p2wshaddressindex_to_p2wshbytes_reader_opt.take(); + p2traddressindex_to_p2trbytes_reader_opt.take(); + p2aaddressindex_to_p2abytes_reader_opt.take(); + if should_export(idxs.height, true) { - txindex_to_first_outputindex_reader_opt.take(); - p2pk65addressindex_to_p2pk65bytes_reader_opt.take(); - p2pk33addressindex_to_p2pk33bytes_reader_opt.take(); - p2pkhaddressindex_to_p2pkhbytes_reader_opt.take(); - p2shaddressindex_to_p2shbytes_reader_opt.take(); - p2wpkhaddressindex_to_p2wpkhbytes_reader_opt.take(); - p2wshaddressindex_to_p2wshbytes_reader_opt.take(); - p2traddressindex_to_p2trbytes_reader_opt.take(); - p2aaddressindex_to_p2abytes_reader_opt.take(); export(stores, vecs, idxs.height, exit)?; } diff --git a/crates/brk_server/src/api/interface.rs b/crates/brk_server/src/api/interface.rs index 5482df1e3..a95571ab7 100644 --- a/crates/brk_server/src/api/interface.rs +++ b/crates/brk_server/src/api/interface.rs @@ -62,7 +62,7 @@ fn req_to_response_res( if weight > MAX_WEIGHT { return Err(Error::Str( - "Request is too heavy, max weight is {MAX_WEIGHT}", + "Request is too heavy, max weight is {MAX_WEIGHT} bytes", )); }