mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-10 15:03:32 -07:00
storable_vec: general update
This commit is contained in:
nym21
Generated
+446
-217
@@ -18,24 +18,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.11"
|
||||
name = "aead"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
|
||||
checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"getrandom",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
"crypto-common",
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "allocator-api2"
|
||||
version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.7.6"
|
||||
@@ -85,19 +76,29 @@ version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d"
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bindex"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bitcoin_hashes 0.16.0",
|
||||
"biter",
|
||||
"canopydb",
|
||||
"color-eyre",
|
||||
"ctrlc",
|
||||
"derive_deref",
|
||||
"heed3",
|
||||
"memmap2",
|
||||
"rayon",
|
||||
"snkrj",
|
||||
"storable_vec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -229,6 +230,9 @@ name = "bitflags"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
@@ -236,37 +240,6 @@ version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "canopydb"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/arthurprs/canopydb#daab91758a7905abc2a7d1a9c7e9f6c41227e72b"
|
||||
dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
"dashmap",
|
||||
"derive_more",
|
||||
"fail",
|
||||
"foldhash",
|
||||
"fslock",
|
||||
"hashbrown",
|
||||
"libc",
|
||||
"lock_api",
|
||||
"log",
|
||||
"lz4_flex",
|
||||
"memmap2",
|
||||
"nix",
|
||||
"parking_lot 0.12.3",
|
||||
"quick_cache",
|
||||
"rand",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"smallvec 2.0.0-alpha.9",
|
||||
"sptr",
|
||||
"tempfile",
|
||||
"triomphe",
|
||||
"xxhash-rust",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.4"
|
||||
@@ -371,6 +344,16 @@ version = "0.8.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ctrlc"
|
||||
version = "3.4.5"
|
||||
@@ -381,20 +364,6 @@ dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dashmap"
|
||||
version = "6.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"hashbrown",
|
||||
"lock_api",
|
||||
"once_cell",
|
||||
"parking_lot_core 0.9.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_deref"
|
||||
version = "1.1.1"
|
||||
@@ -406,27 +375,6 @@ dependencies = [
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05"
|
||||
dependencies = [
|
||||
"derive_more-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more-impl"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derived-deref"
|
||||
version = "2.1.0"
|
||||
@@ -438,28 +386,32 @@ dependencies = [
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "doxygen-rs"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9"
|
||||
dependencies = [
|
||||
"phf",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "eyre"
|
||||
version = "0.6.12"
|
||||
@@ -471,28 +423,14 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fail"
|
||||
version = "0.5.1"
|
||||
name = "form_urlencoded"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c"
|
||||
checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
|
||||
dependencies = [
|
||||
"log",
|
||||
"once_cell",
|
||||
"rand",
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
||||
|
||||
[[package]]
|
||||
name = "foldhash"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f"
|
||||
|
||||
[[package]]
|
||||
name = "fs2"
|
||||
version = "0.4.3"
|
||||
@@ -504,12 +442,14 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fslock"
|
||||
version = "0.2.1"
|
||||
source = "git+https://github.com/arthurprs/fslock.git?rev=7ec91154136a2b3d567b1f79e87cae5d3ca3d927#7ec91154136a2b3d567b1f79e87cae5d3ca3d927"
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
"serde",
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -530,13 +470,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
name = "heed-traits"
|
||||
version = "0.20.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff"
|
||||
|
||||
[[package]]
|
||||
name = "heed-types"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"allocator-api2",
|
||||
"bincode",
|
||||
"byteorder",
|
||||
"heed-traits",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heed3"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7abcf1276482447d5ea921dad8c72fa5b2cfcf4c5016a16491605240bf736205"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"bitflags 2.6.0",
|
||||
"byteorder",
|
||||
"generic-array",
|
||||
"heed-traits",
|
||||
"heed-types",
|
||||
"libc",
|
||||
"lmdb-master3-sys",
|
||||
"once_cell",
|
||||
"page_size",
|
||||
"serde",
|
||||
"synchronoise",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -563,6 +533,145 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd"
|
||||
|
||||
[[package]]
|
||||
name = "icu_collections"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locid"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"litemap",
|
||||
"tinystr",
|
||||
"writeable",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locid_transform"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locid",
|
||||
"icu_locid_transform_data",
|
||||
"icu_provider",
|
||||
"tinystr",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locid_transform_data"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_collections",
|
||||
"icu_normalizer_data",
|
||||
"icu_properties",
|
||||
"icu_provider",
|
||||
"smallvec",
|
||||
"utf16_iter",
|
||||
"utf8_iter",
|
||||
"write16",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer_data"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_collections",
|
||||
"icu_locid_transform",
|
||||
"icu_properties_data",
|
||||
"icu_provider",
|
||||
"tinystr",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties_data"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locid",
|
||||
"icu_provider_macros",
|
||||
"stable_deref_trait",
|
||||
"tinystr",
|
||||
"writeable",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider_macros"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
|
||||
dependencies = [
|
||||
"idna_adapter",
|
||||
"smallvec",
|
||||
"utf8_iter",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna_adapter"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"
|
||||
dependencies = [
|
||||
"icu_normalizer",
|
||||
"icu_properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indenter"
|
||||
version = "0.3.3"
|
||||
@@ -609,10 +718,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.4.15"
|
||||
name = "litemap"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
|
||||
checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104"
|
||||
|
||||
[[package]]
|
||||
name = "lmdb-master3-sys"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bea2558bc45a71673600dcd84070be37cdb7a87da2fc7255c6d377c498df00d"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"doxygen-rs",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
@@ -630,12 +750,6 @@ version = "0.4.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
|
||||
|
||||
[[package]]
|
||||
name = "lz4_flex"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.4"
|
||||
@@ -704,6 +818,16 @@ version = "3.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
|
||||
|
||||
[[package]]
|
||||
name = "page_size"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.2"
|
||||
@@ -712,17 +836,7 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
|
||||
dependencies = [
|
||||
"instant",
|
||||
"lock_api",
|
||||
"parking_lot_core 0.8.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core 0.9.10",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -734,22 +848,57 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall 0.2.16",
|
||||
"smallvec 1.13.2",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.10"
|
||||
name = "percent-encoding"
|
||||
version = "2.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
|
||||
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall 0.5.8",
|
||||
"smallvec 1.13.2",
|
||||
"windows-targets",
|
||||
"phf_macros",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
|
||||
dependencies = [
|
||||
"phf_shared",
|
||||
"rand",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -776,18 +925,6 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick_cache"
|
||||
version = "0.6.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d7c94f8935a9df96bb6380e8592c70edf497a643f94bd23b2f76b399385dbf4"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"equivalent",
|
||||
"hashbrown",
|
||||
"parking_lot 0.12.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.38"
|
||||
@@ -856,34 +993,12 @@ dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834"
|
||||
dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6"
|
||||
dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.18"
|
||||
@@ -899,7 +1014,7 @@ dependencies = [
|
||||
"fs2",
|
||||
"log",
|
||||
"memmap2",
|
||||
"parking_lot 0.11.2",
|
||||
"parking_lot",
|
||||
"sanakirja-core",
|
||||
"serde",
|
||||
"thiserror",
|
||||
@@ -985,18 +1100,18 @@ version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "2.0.0-alpha.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e22442b16a0c1bfae679ffed8ec5e160ae2aa8495cea891f0d2ff7b84fe5c4c"
|
||||
|
||||
[[package]]
|
||||
name = "snkrj"
|
||||
version = "0.1.1"
|
||||
@@ -1005,10 +1120,18 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sptr"
|
||||
version = "0.3.2"
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a"
|
||||
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
|
||||
|
||||
[[package]]
|
||||
name = "storable_vec"
|
||||
version = "0.1.2"
|
||||
dependencies = [
|
||||
"color-eyre",
|
||||
"memmap2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
@@ -1033,17 +1156,23 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.15.0"
|
||||
name = "synchronoise"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704"
|
||||
checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"fastrand",
|
||||
"getrandom",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys",
|
||||
"crossbeam-queue",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1076,6 +1205,16 @@ dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.41"
|
||||
@@ -1118,10 +1257,10 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "triomphe"
|
||||
version = "0.1.14"
|
||||
name = "typenum"
|
||||
version = "1.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef8f7726da4807b58ea5c96fdc122f80702030edc33b35aff9190a51148ccc85"
|
||||
checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
@@ -1130,10 +1269,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.6"
|
||||
name = "url"
|
||||
version = "2.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
||||
checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"idna",
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf16_iter"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
@@ -1249,10 +1405,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "xxhash-rust"
|
||||
version = "0.8.15"
|
||||
name = "write16"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
|
||||
checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
|
||||
|
||||
[[package]]
|
||||
name = "yoke"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"stable_deref_trait",
|
||||
"yoke-derive",
|
||||
"zerofrom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yoke-derive"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
@@ -1274,3 +1460,46 @@ dependencies = [
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e"
|
||||
dependencies = [
|
||||
"zerofrom-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom-derive"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
|
||||
dependencies = [
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec-derive"
|
||||
version = "0.10.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
@@ -9,7 +9,8 @@ biter = "0.2.2"
|
||||
color-eyre = "0.6.3"
|
||||
ctrlc = "3.4.5"
|
||||
derive_deref = "1.1.1"
|
||||
heed3 = "0.21.0"
|
||||
memmap2 = "0.9.5"
|
||||
rayon = "1.10.0"
|
||||
snkrj = { path = "../snkrj" }
|
||||
canopydb = { git = "https://github.com/arthurprs/canopydb" }
|
||||
storable_vec = { path = "../storable_vec" }
|
||||
|
||||
@@ -10,6 +10,7 @@ use biter::{
|
||||
bitcoin::{TxIn, TxOut, Txid},
|
||||
bitcoincore_rpc::{Auth, Client},
|
||||
};
|
||||
// use heed3::{Database, EnvOpenOptions};
|
||||
|
||||
mod structs;
|
||||
|
||||
@@ -17,7 +18,7 @@ use color_eyre::eyre::{eyre, ContextCompat};
|
||||
use rayon::prelude::*;
|
||||
use structs::{
|
||||
Addressbytes, AddressbytesPrefix, Addressindex, Addressindextxoutindex, Addresstype, Addresstypeindex, Amount,
|
||||
AnyVecdisk, BlockHashPrefix, Databases, Exit, Height, TxidPrefix, Txindex, Txindexvout, Txoutindex, Vecdisks,
|
||||
BlockHashPrefix, Databases, Exit, Height, StorableVecs, TxidPrefix, Txindex, Txindexvout, Txoutindex,
|
||||
};
|
||||
|
||||
// https://github.com/fjall-rs/fjall/discussions/72
|
||||
@@ -32,6 +33,8 @@ enum TxInOrAddressindextoutindex<'a> {
|
||||
const MONTHLY_BLOCK_TARGET: usize = 144 * 30;
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
let i = std::time::Instant::now();
|
||||
|
||||
let check_collisions = true;
|
||||
@@ -44,31 +47,47 @@ fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let path_database = Path::new("./database");
|
||||
|
||||
let mut vecdisks = Vecdisks::import(path_database)?;
|
||||
let mut vecs = StorableVecs::import(path_database)?;
|
||||
|
||||
// let env = unsafe { EnvOpenOptions::new().open(Path::new("./heed"))? };
|
||||
// let mut wtxn = env.write_txn()?;
|
||||
|
||||
// let addressbytes_prefix_to_addressindex: Database<AddressbytesPrefix, Addressindex> =
|
||||
// env.create_database(&mut wtxn, Some("addressbytes_prefix_to_addressindex"))?;
|
||||
// let addressindextxoutindex_in: Database<Addressindextxoutindex, ()> =
|
||||
// env.create_database(&mut wtxn, Some("addressindextxoutindex_in"))?;
|
||||
// let addressindextxoutindex_out: Database<Addressindextxoutindex, ()> =
|
||||
// env.create_database(&mut wtxn, Some("addressindextxoutindex_out"))?;
|
||||
// let blockhash_prefix_to_height: Database<BlockHashPrefix, Height> =
|
||||
// env.create_database(&mut wtxn, Some("blockhash_prefix_to_height"))?;
|
||||
// let txid_prefix_to_txindex: Database<TxidPrefix, Txindex> =
|
||||
// env.create_database(&mut wtxn, Some("txid_prefix_to_txindex"))?;
|
||||
// let txindexvout_to_txoutindex: Database<Txindexvout, Txoutindex> =
|
||||
// env.create_database(&mut wtxn, Some("txindexvout_to_txoutindex"))?;
|
||||
|
||||
let databases = Databases::open(path_database)?;
|
||||
|
||||
let mut height = Height::from(0_u32);
|
||||
|
||||
let mut txindex = vecdisks
|
||||
let mut txindex = vecs
|
||||
.height_to_first_txindex
|
||||
.get(height)?
|
||||
.cloned()
|
||||
.unwrap_or(Txindex::default());
|
||||
|
||||
let mut txoutindex = vecdisks
|
||||
let mut txoutindex = vecs
|
||||
.height_to_first_txoutindex
|
||||
.get(height)?
|
||||
.cloned()
|
||||
.unwrap_or(Txoutindex::default());
|
||||
|
||||
let mut addressindex = vecdisks
|
||||
let mut addressindex = vecs
|
||||
.height_to_first_addressindex
|
||||
.get(height)?
|
||||
.cloned()
|
||||
.unwrap_or(Addressindex::default());
|
||||
|
||||
let export = |databases: Databases, vecdisks: &mut Vecdisks, height: Height| -> color_eyre::Result<()> {
|
||||
let export = |databases: Databases, vecdisks: &mut StorableVecs, height: Height| -> color_eyre::Result<()> {
|
||||
exit.block();
|
||||
println!("Exporting...");
|
||||
databases.export();
|
||||
@@ -101,22 +120,23 @@ fn main() -> color_eyre::Result<()> {
|
||||
// if parts.blockhash_prefix_to_height.needs(height) {
|
||||
let blockhash_prefix = BlockHashPrefix::try_from(&blockhash)?;
|
||||
|
||||
if check_collisions {
|
||||
if let Some(prev_height) =
|
||||
databases.blockhash_prefix_to_height.get(&blockhash_prefix)
|
||||
{
|
||||
dbg!(blockhash, prev_height);
|
||||
return Err(eyre!("Collision, expect prefix to need be set yet"));
|
||||
}
|
||||
}
|
||||
|
||||
databases.blockhash_prefix_to_height.insert(blockhash_prefix,height);
|
||||
// if check_collisions {
|
||||
// if let Some(prev_height) =
|
||||
// databases.blockhash_prefix_to_height.get(&blockhash_prefix)
|
||||
// {
|
||||
// dbg!(blockhash, prev_height);
|
||||
// return Err(eyre!("Collision, expect prefix to need be set yet"));
|
||||
// }
|
||||
// }
|
||||
|
||||
vecdisks.height_to_blockhash.push_if_needed(height, blockhash)?;
|
||||
vecdisks.height_to_first_txindex.push_if_needed(height, txindex)?;
|
||||
vecdisks.height_to_first_txoutindex.push_if_needed(height, txoutindex)?;
|
||||
vecdisks.height_to_first_addressindex.push_if_needed(height, addressindex)?;
|
||||
databases.blockhash_prefix_to_height.insert(blockhash_prefix,height);
|
||||
// blockhash_prefix_to_height.put(&mut wtxn, &blockhash_prefix,&height);
|
||||
// }
|
||||
|
||||
vecs.height_to_blockhash.push_if_needed(height, blockhash)?;
|
||||
vecs.height_to_first_txindex.push_if_needed(height, txindex)?;
|
||||
vecs.height_to_first_txoutindex.push_if_needed(height, txoutindex)?;
|
||||
vecs.height_to_first_addressindex.push_if_needed(height, addressindex)?;
|
||||
|
||||
let outputs = block
|
||||
.txdata
|
||||
@@ -200,7 +220,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
dbg!(outpoint.txid, txindex_local, vout, txindexvout);
|
||||
})?;
|
||||
|
||||
let addressindex = *vecdisks.txoutindex_to_addressindex.get(txoutindex)?
|
||||
let addressindex = *vecs.txoutindex_to_addressindex.get(txoutindex)?
|
||||
.context("Expect addressindex to not be none")
|
||||
.inspect_err(|_| {
|
||||
// let height = vecdisks.txindex_to_height.get(txindex.into()).expect("txindex_to_height get not fail")
|
||||
@@ -321,7 +341,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
);
|
||||
// }
|
||||
|
||||
vecdisks.txoutindex_to_amount.push_if_needed(
|
||||
vecs.txoutindex_to_amount.push_if_needed(
|
||||
txoutindex,
|
||||
amount,
|
||||
)?;
|
||||
@@ -331,12 +351,12 @@ fn main() -> color_eyre::Result<()> {
|
||||
if let Some(addressindex) = addressindex_opt {
|
||||
addressindex_local = addressindex;
|
||||
} else {
|
||||
vecdisks.addressindex_to_addresstype.push_if_needed(addressindex_local, addresstype)?;
|
||||
vecs.addressindex_to_addresstype.push_if_needed(addressindex_local, addresstype)?;
|
||||
|
||||
// TODO: Create counter of other addresstypes instead
|
||||
let addresstypeindex = Addresstypeindex::from(vecdisks.addresstype_to_addressvecdisk(addresstype).map_or(0, |vecdisk| vecdisk.len()));
|
||||
let addresstypeindex = Addresstypeindex::from(vecs.addresstype_to_addressbytes(addresstype).map_or(0, |vecdisk| vecdisk.len()));
|
||||
|
||||
vecdisks.addressindex_to_addresstypeindex.push_if_needed(addressindex_local, addresstypeindex)?;
|
||||
vecs.addressindex_to_addresstypeindex.push_if_needed(addressindex_local, addresstypeindex)?;
|
||||
|
||||
if let Ok(addressbytes) = addressbytes_res {
|
||||
// if parts.addressbytes_prefix_to_addressindex.needs(height) {
|
||||
@@ -346,7 +366,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
);
|
||||
// }
|
||||
|
||||
vecdisks.push_addressbytes_if_needed(addresstypeindex, addressbytes)?;
|
||||
vecs.push_addressbytes_if_needed(addresstypeindex, addressbytes)?;
|
||||
}
|
||||
|
||||
addressindex.increment();
|
||||
@@ -354,7 +374,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
|
||||
new_txindexvout_to_addressindextxoutindex.insert(txindexvout, Addressindextxoutindex::from((addressindex_local, txoutindex)));
|
||||
|
||||
vecdisks.txoutindex_to_addressindex.push_if_needed(
|
||||
vecs.txoutindex_to_addressindex.push_if_needed(
|
||||
txoutindex,
|
||||
addressindex_local,
|
||||
)?;
|
||||
@@ -426,10 +446,10 @@ fn main() -> color_eyre::Result<()> {
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
let len = vecdisks.txindex_to_txid.len();
|
||||
let len = vecs.txindex_to_txid.len();
|
||||
// Ok if `get` is not par as should happen only twice
|
||||
let prev_txid =
|
||||
vecdisks.txindex_to_txid.get(prev_txindex)?
|
||||
vecs.txindex_to_txid.get(prev_txindex)?
|
||||
.context("To have txid for txindex").inspect_err(|_| {
|
||||
dbg!(txindex_local, txid, len);
|
||||
})?;
|
||||
@@ -444,7 +464,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
let is_dup = only_known_dup_txids.contains(prev_txid);
|
||||
|
||||
if !is_dup {
|
||||
let prev_height = vecdisks.txindex_to_height.get(prev_txindex)?.expect("To have height");
|
||||
let prev_height = vecs.txindex_to_height.get(prev_txindex)?.expect("To have height");
|
||||
dbg!(height, txid, txindex_local, prev_height, prev_txid, prev_txindex);
|
||||
return Err(eyre!("Expect none"));
|
||||
}
|
||||
@@ -456,18 +476,18 @@ fn main() -> color_eyre::Result<()> {
|
||||
)?;
|
||||
|
||||
txindex_to_txid.into_iter().try_for_each(|(txindex, txid)| -> color_eyre::Result<()> {
|
||||
vecdisks.txindex_to_txid.push_if_needed(txindex, txid)?;
|
||||
vecdisks.txindex_to_height.push_if_needed(txindex, height)?;
|
||||
vecs.txindex_to_txid.push_if_needed(txindex, txid)?;
|
||||
vecs.txindex_to_height.push_if_needed(txindex, height)?;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
vecdisks.height_to_last_txindex.push_if_needed(height, txindex.decremented())?;
|
||||
vecdisks.height_to_last_txoutindex.push_if_needed(height, txoutindex.decremented())?;
|
||||
vecdisks.height_to_last_addressindex.push_if_needed(height, addressindex.decremented())?;
|
||||
vecs.height_to_last_txindex.push_if_needed(height, txindex.decremented())?;
|
||||
vecs.height_to_last_txoutindex.push_if_needed(height, txoutindex.decremented())?;
|
||||
vecs.height_to_last_addressindex.push_if_needed(height, addressindex.decremented())?;
|
||||
|
||||
let should_snapshot = _height % MONTHLY_BLOCK_TARGET == 0 && !exit.active();
|
||||
if should_snapshot {
|
||||
export(databases, &mut vecdisks, height)?;
|
||||
export(databases, &mut vecs, height)?;
|
||||
databases_opt.replace(Databases::open(path_database)?);
|
||||
} else {
|
||||
databases_opt.replace(databases);
|
||||
@@ -484,9 +504,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
pause();
|
||||
|
||||
let databases = databases_opt.take().context("option should have wtx")?;
|
||||
export(databases, &mut vecdisks, height)?;
|
||||
|
||||
pause();
|
||||
export(databases, &mut vecs, height)?;
|
||||
|
||||
dbg!(i.elapsed());
|
||||
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use canopydb::{Database as CanopyDatabase, DbOptions};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use super::Environment;
|
||||
|
||||
#[derive(Debug, Deref, DerefMut)]
|
||||
pub struct Database(CanopyDatabase);
|
||||
|
||||
impl Database {
|
||||
pub fn new(environment: &Environment, name: &str) -> color_eyre::Result<Self> {
|
||||
let mut options = DbOptions::default();
|
||||
options.use_wal = false;
|
||||
options.checkpoint_interval = Duration::from_secs(u64::MAX);
|
||||
options.checkpoint_target_size = usize::MAX;
|
||||
options.throttle_memory_limit = usize::MAX;
|
||||
options.stall_memory_limit = usize::MAX;
|
||||
options.write_txn_memory_limit = usize::MAX;
|
||||
|
||||
Ok(Self(environment.get_or_create_database_with(name, options)?))
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
use std::path::Path;
|
||||
|
||||
use canopydb::{EnvOptions, Environment as CanopyEnvironment};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
|
||||
#[derive(Debug, Deref, DerefMut)]
|
||||
pub struct Environment(CanopyEnvironment);
|
||||
|
||||
impl Environment {
|
||||
pub fn new(path: &Path) -> color_eyre::Result<Self> {
|
||||
let mut options = EnvOptions::new(path);
|
||||
// options.use_mmap = true;
|
||||
options.disable_fsync = true;
|
||||
options.wal_new_file_on_checkpoint = false;
|
||||
options.wal_background_sync_interval = None;
|
||||
options.wal_write_batch_memory_limit = usize::MAX;
|
||||
|
||||
Ok(Self(CanopyEnvironment::with_options(options)?))
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
mod database;
|
||||
mod environment;
|
||||
// mod transaction;
|
||||
mod tree;
|
||||
|
||||
pub use database::*;
|
||||
pub use environment::*;
|
||||
// pub use transaction::*;
|
||||
pub use tree::*;
|
||||
@@ -1,19 +0,0 @@
|
||||
use canopydb::{Tree as CanopyTree, TreeOptions, WriteTransaction};
|
||||
|
||||
use super::{Database, Tree};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Transaction<'a, K, V> {
|
||||
tx: WriteTransaction,
|
||||
tree: Tree<'a, K, V>,
|
||||
}
|
||||
|
||||
impl<'a, K, V> Transaction<'a, K, V> {
|
||||
pub fn new(db: &Database) -> color_eyre::Result<Self> {
|
||||
let tx = db.begin_write()?;
|
||||
|
||||
let tree = Tree::new(&tx)?;
|
||||
|
||||
Ok(Self { tx, tree })
|
||||
}
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
marker::PhantomData,
|
||||
ops::{Deref, DerefMut},
|
||||
};
|
||||
|
||||
use canopydb::{Tree as CanopyTree, TreeOptions, WriteTransaction};
|
||||
use color_eyre::eyre::eyre;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Tree<'a, K, V> {
|
||||
tree: CanopyTree<'a>,
|
||||
k: PhantomData<K>,
|
||||
v: PhantomData<V>,
|
||||
}
|
||||
impl<'a, K, V> Deref for Tree<'a, K, V> {
|
||||
type Target = CanopyTree<'a>;
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.tree
|
||||
}
|
||||
}
|
||||
impl<'a, K, V> DerefMut for Tree<'a, K, V> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.tree
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, K, V> Tree<'a, K, V>
|
||||
where
|
||||
K: Debug + Sized,
|
||||
V: Debug + Sized + Clone + Copy,
|
||||
{
|
||||
const SIZE_OF_K: usize = size_of::<K>();
|
||||
const SIZE_OF_V: usize = size_of::<V>();
|
||||
|
||||
pub fn new(tx: &'a WriteTransaction) -> color_eyre::Result<Self> {
|
||||
let mut options = TreeOptions::new();
|
||||
options.compress_overflow_values = None;
|
||||
options.fixed_key_len = size_of::<K>() as i8;
|
||||
options.fixed_value_len = size_of::<V>() as i8;
|
||||
|
||||
Ok(Self {
|
||||
tree: tx.get_or_create_tree_with(b"tree", options)?,
|
||||
k: PhantomData,
|
||||
v: PhantomData,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get(&self, key: &K) -> color_eyre::Result<Option<V>> {
|
||||
let slice = self.tree.get(Self::key_as_slice(key))?;
|
||||
|
||||
if slice.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let slice = slice.unwrap();
|
||||
|
||||
let (prefix, shorts, suffix) = unsafe { slice.align_to::<V>() };
|
||||
|
||||
if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() {
|
||||
dbg!(&key, &prefix, &shorts, &suffix);
|
||||
return Err(eyre!("align_to issue"));
|
||||
}
|
||||
|
||||
Ok(Some(shorts[0]))
|
||||
}
|
||||
|
||||
pub fn insert(&mut self, key: &K, value: &V) -> Result<(), canopydb::Error> {
|
||||
self.tree
|
||||
.insert(Self::key_as_slice(key), Self::value_as_slice(value))
|
||||
}
|
||||
|
||||
fn key_as_slice(key: &K) -> &[u8] {
|
||||
let data: *const K = key;
|
||||
let data: *const u8 = data as *const u8;
|
||||
unsafe { std::slice::from_raw_parts(data, Self::SIZE_OF_K) }
|
||||
}
|
||||
|
||||
fn value_as_slice(value: &V) -> &[u8] {
|
||||
let data: *const V = value;
|
||||
let data: *const u8 = data as *const u8;
|
||||
unsafe { std::slice::from_raw_parts(data, Self::SIZE_OF_V) }
|
||||
}
|
||||
}
|
||||
@@ -4,17 +4,15 @@ mod addressindextxoutindex;
|
||||
mod addresstype;
|
||||
mod addresstypeindex;
|
||||
mod amount;
|
||||
mod canopy;
|
||||
mod databases;
|
||||
mod exit;
|
||||
mod height;
|
||||
mod prefix;
|
||||
mod slice;
|
||||
mod storable_vecs;
|
||||
mod txindex;
|
||||
mod txindexvout;
|
||||
mod txoutindex;
|
||||
mod vecdisk;
|
||||
mod vecdisks;
|
||||
|
||||
pub use addressbytes::*;
|
||||
pub use addressindex::*;
|
||||
@@ -22,14 +20,12 @@ pub use addressindextxoutindex::*;
|
||||
pub use addresstype::*;
|
||||
pub use addresstypeindex::*;
|
||||
pub use amount::*;
|
||||
pub use canopy::*;
|
||||
pub use databases::*;
|
||||
pub use exit::*;
|
||||
pub use height::*;
|
||||
pub use prefix::*;
|
||||
pub use slice::*;
|
||||
pub use storable_vecs::*;
|
||||
pub use txindex::*;
|
||||
pub use txindexvout::*;
|
||||
pub use txoutindex::*;
|
||||
pub use vecdisk::*;
|
||||
pub use vecdisks::*;
|
||||
|
||||
+50
-49
@@ -1,15 +1,16 @@
|
||||
use std::{fs, path::Path};
|
||||
use std::{fs, io, path::Path};
|
||||
|
||||
use biter::bitcoin::{BlockHash, Txid};
|
||||
use color_eyre::eyre::eyre;
|
||||
use storable_vec::{AnyStorableVec, StorableVec};
|
||||
|
||||
use super::{
|
||||
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, AnyVecdisk, Exit, Height, P2PK33AddressBytes,
|
||||
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, Exit, Height, P2PK33AddressBytes,
|
||||
P2PK65AddressBytes, P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes,
|
||||
Txindex, Txoutindex, Vecdisk,
|
||||
Txindex, Txoutindex,
|
||||
};
|
||||
|
||||
pub struct Vecdisks {
|
||||
pub struct StorableVecs {
|
||||
// TODO:
|
||||
//
|
||||
// Add
|
||||
@@ -22,59 +23,59 @@ pub struct Vecdisks {
|
||||
// date_to_fees
|
||||
// date_to_first_height
|
||||
// date_to_last_height
|
||||
pub addressindex_to_addresstype: Vecdisk<Addressindex, Addresstype>,
|
||||
pub addressindex_to_addresstypeindex: Vecdisk<Addressindex, Addresstypeindex>,
|
||||
pub height_to_blockhash: Vecdisk<Height, BlockHash>,
|
||||
pub height_to_first_addressindex: Vecdisk<Height, Addressindex>,
|
||||
pub height_to_first_txindex: Vecdisk<Height, Txindex>,
|
||||
pub height_to_first_txoutindex: Vecdisk<Height, Txoutindex>,
|
||||
pub height_to_last_addressindex: Vecdisk<Height, Addressindex>,
|
||||
pub height_to_last_txindex: Vecdisk<Height, Txindex>,
|
||||
pub height_to_last_txoutindex: Vecdisk<Height, Txoutindex>,
|
||||
pub p2pk65index_to_p2pk65addressbytes: Vecdisk<Addresstypeindex, P2PK65AddressBytes>,
|
||||
pub p2pk33index_to_p2pk33addressbytes: Vecdisk<Addresstypeindex, P2PK33AddressBytes>,
|
||||
pub p2pkhindex_to_p2pkhaddressbytes: Vecdisk<Addresstypeindex, P2PKHAddressBytes>,
|
||||
pub p2shindex_to_p2shaddressbytes: Vecdisk<Addresstypeindex, P2SHAddressBytes>,
|
||||
pub p2wpkhindex_to_p2wpkhaddressbytes: Vecdisk<Addresstypeindex, P2WPKHAddressBytes>,
|
||||
pub p2wshindex_to_p2wshaddressbytes: Vecdisk<Addresstypeindex, P2WSHAddressBytes>,
|
||||
pub p2trindex_to_p2traddressbytes: Vecdisk<Addresstypeindex, P2TRAddressBytes>,
|
||||
pub txindex_to_height: Vecdisk<Txindex, Height>,
|
||||
pub txindex_to_txid: Vecdisk<Txindex, Txid>,
|
||||
pub txoutindex_to_addressindex: Vecdisk<Txoutindex, Addressindex>,
|
||||
pub txoutindex_to_amount: Vecdisk<Txoutindex, Amount>,
|
||||
pub addressindex_to_addresstype: StorableVec<Addressindex, Addresstype>,
|
||||
pub addressindex_to_addresstypeindex: StorableVec<Addressindex, Addresstypeindex>,
|
||||
pub height_to_blockhash: StorableVec<Height, BlockHash>,
|
||||
pub height_to_first_addressindex: StorableVec<Height, Addressindex>,
|
||||
pub height_to_first_txindex: StorableVec<Height, Txindex>,
|
||||
pub height_to_first_txoutindex: StorableVec<Height, Txoutindex>,
|
||||
pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
|
||||
pub height_to_last_txindex: StorableVec<Height, Txindex>,
|
||||
pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
|
||||
pub p2pk65index_to_p2pk65addressbytes: StorableVec<Addresstypeindex, P2PK65AddressBytes>,
|
||||
pub p2pk33index_to_p2pk33addressbytes: StorableVec<Addresstypeindex, P2PK33AddressBytes>,
|
||||
pub p2pkhindex_to_p2pkhaddressbytes: StorableVec<Addresstypeindex, P2PKHAddressBytes>,
|
||||
pub p2shindex_to_p2shaddressbytes: StorableVec<Addresstypeindex, P2SHAddressBytes>,
|
||||
pub p2wpkhindex_to_p2wpkhaddressbytes: StorableVec<Addresstypeindex, P2WPKHAddressBytes>,
|
||||
pub p2wshindex_to_p2wshaddressbytes: StorableVec<Addresstypeindex, P2WSHAddressBytes>,
|
||||
pub p2trindex_to_p2traddressbytes: StorableVec<Addresstypeindex, P2TRAddressBytes>,
|
||||
pub txindex_to_height: StorableVec<Txindex, Height>,
|
||||
pub txindex_to_txid: StorableVec<Txindex, Txid>,
|
||||
pub txoutindex_to_addressindex: StorableVec<Txoutindex, Addressindex>,
|
||||
pub txoutindex_to_amount: StorableVec<Txoutindex, Amount>,
|
||||
}
|
||||
|
||||
// const UNSAFE_BLOCKS: usize = 100;
|
||||
|
||||
impl Vecdisks {
|
||||
impl StorableVecs {
|
||||
pub fn import(path: &Path) -> color_eyre::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
Ok(Self {
|
||||
addressindex_to_addresstype: Vecdisk::import(&path.join("addressindex_to_addresstype"))?,
|
||||
addressindex_to_addresstypeindex: Vecdisk::import(&path.join("addressindex_to_addresstypeindex"))?,
|
||||
height_to_blockhash: Vecdisk::import(&path.join("height_to_blockhash"))?,
|
||||
height_to_first_addressindex: Vecdisk::import(&path.join("height_to_first_addressindex"))?,
|
||||
height_to_first_txindex: Vecdisk::import(&path.join("height_to_first_txindex"))?,
|
||||
height_to_first_txoutindex: Vecdisk::import(&path.join("height_to_first_txoutindex"))?,
|
||||
height_to_last_addressindex: Vecdisk::import(&path.join("height_to_last_addressindex"))?,
|
||||
height_to_last_txindex: Vecdisk::import(&path.join("height_to_last_txindex"))?,
|
||||
height_to_last_txoutindex: Vecdisk::import(&path.join("height_to_last_txoutindex"))?,
|
||||
p2pk65index_to_p2pk65addressbytes: Vecdisk::import(&path.join("p2pk65index_to_p2pk65addressbytes"))?,
|
||||
p2pk33index_to_p2pk33addressbytes: Vecdisk::import(&path.join("p2pk33index_to_p2pk33addressbytes"))?,
|
||||
p2pkhindex_to_p2pkhaddressbytes: Vecdisk::import(&path.join("p2pkhindex_to_p2pkhaddressbytes"))?,
|
||||
p2shindex_to_p2shaddressbytes: Vecdisk::import(&path.join("p2shindex_to_p2shaddressbytes"))?,
|
||||
p2wpkhindex_to_p2wpkhaddressbytes: Vecdisk::import(&path.join("p2wpkhindex_to_p2wpkhaddressbytes"))?,
|
||||
p2wshindex_to_p2wshaddressbytes: Vecdisk::import(&path.join("p2wshindex_to_p2wshaddressbytes"))?,
|
||||
p2trindex_to_p2traddressbytes: Vecdisk::import(&path.join("p2trindex_to_p2traddressbytes"))?,
|
||||
txindex_to_height: Vecdisk::import(&path.join("txindex_to_height"))?,
|
||||
txindex_to_txid: Vecdisk::import(&path.join("txindex_to_txid"))?,
|
||||
txoutindex_to_addressindex: Vecdisk::import(&path.join("txoutindex_to_addressindex"))?,
|
||||
txoutindex_to_amount: Vecdisk::import(&path.join("txoutindex_to_amount"))?,
|
||||
addressindex_to_addresstype: StorableVec::import(&path.join("addressindex_to_addresstype"))?,
|
||||
addressindex_to_addresstypeindex: StorableVec::import(&path.join("addressindex_to_addresstypeindex"))?,
|
||||
height_to_blockhash: StorableVec::import(&path.join("height_to_blockhash"))?,
|
||||
height_to_first_addressindex: StorableVec::import(&path.join("height_to_first_addressindex"))?,
|
||||
height_to_first_txindex: StorableVec::import(&path.join("height_to_first_txindex"))?,
|
||||
height_to_first_txoutindex: StorableVec::import(&path.join("height_to_first_txoutindex"))?,
|
||||
height_to_last_addressindex: StorableVec::import(&path.join("height_to_last_addressindex"))?,
|
||||
height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"))?,
|
||||
height_to_last_txoutindex: StorableVec::import(&path.join("height_to_last_txoutindex"))?,
|
||||
p2pk65index_to_p2pk65addressbytes: StorableVec::import(&path.join("p2pk65index_to_p2pk65addressbytes"))?,
|
||||
p2pk33index_to_p2pk33addressbytes: StorableVec::import(&path.join("p2pk33index_to_p2pk33addressbytes"))?,
|
||||
p2pkhindex_to_p2pkhaddressbytes: StorableVec::import(&path.join("p2pkhindex_to_p2pkhaddressbytes"))?,
|
||||
p2shindex_to_p2shaddressbytes: StorableVec::import(&path.join("p2shindex_to_p2shaddressbytes"))?,
|
||||
p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::import(&path.join("p2wpkhindex_to_p2wpkhaddressbytes"))?,
|
||||
p2wshindex_to_p2wshaddressbytes: StorableVec::import(&path.join("p2wshindex_to_p2wshaddressbytes"))?,
|
||||
p2trindex_to_p2traddressbytes: StorableVec::import(&path.join("p2trindex_to_p2traddressbytes"))?,
|
||||
txindex_to_height: StorableVec::import(&path.join("txindex_to_height"))?,
|
||||
txindex_to_txid: StorableVec::import(&path.join("txindex_to_txid"))?,
|
||||
txoutindex_to_addressindex: StorableVec::import(&path.join("txoutindex_to_addressindex"))?,
|
||||
txoutindex_to_amount: StorableVec::import(&path.join("txoutindex_to_amount"))?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn addresstype_to_addressvecdisk(&self, addresstype: Addresstype) -> color_eyre::Result<&dyn AnyVecdisk> {
|
||||
pub fn addresstype_to_addressbytes(&self, addresstype: Addresstype) -> color_eyre::Result<&dyn AnyStorableVec> {
|
||||
match addresstype {
|
||||
Addresstype::P2PK65 => Ok(&self.p2pk65index_to_p2pk65addressbytes),
|
||||
Addresstype::P2PK33 => Ok(&self.p2pk33index_to_p2pk33addressbytes),
|
||||
@@ -217,11 +218,11 @@ impl Vecdisks {
|
||||
// Ok(())
|
||||
}
|
||||
|
||||
pub fn flush(&mut self) -> color_eyre::Result<()> {
|
||||
self.as_mut_vec().into_iter().try_for_each(AnyVecdisk::flush)
|
||||
pub fn flush(&mut self) -> io::Result<()> {
|
||||
self.as_mut_vec().into_iter().try_for_each(AnyStorableVec::flush)
|
||||
}
|
||||
|
||||
fn as_mut_vec(&mut self) -> Vec<&mut dyn AnyVecdisk> {
|
||||
fn as_mut_vec(&mut self) -> Vec<&mut dyn AnyStorableVec> {
|
||||
vec![
|
||||
&mut self.addressindex_to_addresstype,
|
||||
&mut self.addressindex_to_addresstypeindex,
|
||||
@@ -173,6 +173,11 @@ where
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect::<_>()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn len(&self) -> usize {
|
||||
self.iter_ram_then_disk().count()
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AnyDatabase {
|
||||
|
||||
Generated
+246
@@ -0,0 +1,246 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "addr2line"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
|
||||
dependencies = [
|
||||
"gimli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "adler"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.71"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d"
|
||||
dependencies = [
|
||||
"addr2line",
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"miniz_oxide",
|
||||
"object",
|
||||
"rustc-demangle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b"
|
||||
dependencies = [
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "color-eyre"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55146f5e46f237f7423d74111267d4597b59b0dad0ffaf7303bce9945d843ad5"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"color-spantrace",
|
||||
"eyre",
|
||||
"indenter",
|
||||
"once_cell",
|
||||
"owo-colors",
|
||||
"tracing-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "color-spantrace"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd6be1b2a7e382e2b98b43b2adcca6bb0e465af0bdd38123873ae61eb17a72c2"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"owo-colors",
|
||||
"tracing-core",
|
||||
"tracing-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "eyre"
|
||||
version = "0.6.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec"
|
||||
dependencies = [
|
||||
"indenter",
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gimli"
|
||||
version = "0.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
|
||||
|
||||
[[package]]
|
||||
name = "indenter"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.169"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
||||
|
||||
[[package]]
|
||||
name = "memmap2"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"
|
||||
dependencies = [
|
||||
"adler",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.32.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.20.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
|
||||
|
||||
[[package]]
|
||||
name = "owo-colors"
|
||||
version = "3.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
|
||||
|
||||
[[package]]
|
||||
name = "sharded-slab"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "storable_vec"
|
||||
version = "0.1.2"
|
||||
dependencies = [
|
||||
"color-eyre",
|
||||
"memmap2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
|
||||
dependencies = [
|
||||
"pin-project-lite",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-core"
|
||||
version = "0.1.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"valuable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-error"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db"
|
||||
dependencies = [
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-subscriber"
|
||||
version = "0.3.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
|
||||
dependencies = [
|
||||
"sharded-slab",
|
||||
"thread_local",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
|
||||
@@ -0,0 +1,12 @@
|
||||
[package]
|
||||
name = "storable_vec"
|
||||
description = "A very small, fast, efficient and simple storable Vec"
|
||||
version = "0.1.2"
|
||||
license = "MIT"
|
||||
keywords = ["vec", "disk", "data"]
|
||||
categories = ["database"]
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
color-eyre = "0.6.3"
|
||||
memmap2 = "0.9.5"
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 storable_vec
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,52 @@
|
||||
# storable_vec
|
||||
|
||||
A very small, fast, efficient and simple storable `vec` which uses `mmap2` for speed.
|
||||
|
||||
## Features
|
||||
|
||||
- [x] Get (Rayon compatible)
|
||||
- [x] Push
|
||||
- [ ] Update
|
||||
- [ ] Remove
|
||||
|
||||
## Example
|
||||
|
||||
```rust
|
||||
use std::path::Path;
|
||||
|
||||
use storable_vec::{AnyStorableVec, StorableVec};
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
{
|
||||
let mut vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
|
||||
vec.push(21);
|
||||
dbg!(vec.get(0)?); // 21
|
||||
vec.flush()?;
|
||||
}
|
||||
|
||||
{
|
||||
let vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
|
||||
dbg!(vec.get(0)?); // 21
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
## Disclaimer
|
||||
|
||||
Portability will depend on the type of values.
|
||||
|
||||
Non bytes/slices types (`u8`, `u16`, ...) will be read as slice in an unsafe manner (using `std::slice::from_raw_parts`) and thus have the endianness of the system. On the other hand, `&[u8]` should be inserted as is.
|
||||
|
||||
If portability is important to you, just create a wrapper struct which has custom `get`, `push`, ... methods and does something like:
|
||||
|
||||
```rust
|
||||
impl StorableVecU64 {
|
||||
pub fn push(&mut self, value: u64) {
|
||||
self.push(&value.to_be_bytes())
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -5,14 +5,13 @@ use std::{
|
||||
io::{self, Write},
|
||||
marker::PhantomData,
|
||||
mem,
|
||||
ops::Range,
|
||||
ops::{Deref, DerefMut, Range},
|
||||
path::Path,
|
||||
sync::OnceLock,
|
||||
};
|
||||
|
||||
use color_eyre::eyre::{eyre, ContextCompat};
|
||||
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use memmap2::{Mmap, MmapOptions};
|
||||
|
||||
///
|
||||
@@ -25,7 +24,7 @@ use memmap2::{Mmap, MmapOptions};
|
||||
/// The file isn't portable for speed reasons (TODO: but could be ?)
|
||||
///
|
||||
#[derive(Debug)]
|
||||
pub struct Vecdisk<I, T> {
|
||||
pub struct StorableVec<I, T> {
|
||||
file: File,
|
||||
mmaps: VecLazyMmap,
|
||||
disk_len: usize,
|
||||
@@ -36,7 +35,7 @@ pub struct Vecdisk<I, T> {
|
||||
/// In bytes
|
||||
const MAX_PAGE_SIZE: usize = 4096;
|
||||
|
||||
impl<I, T> Vecdisk<I, T>
|
||||
impl<I, T> StorableVec<I, T>
|
||||
where
|
||||
I: Into<usize>,
|
||||
T: Sized + Debug,
|
||||
@@ -103,10 +102,8 @@ where
|
||||
pub fn get(&self, index: I) -> color_eyre::Result<Option<&T>> {
|
||||
self._get(index.into())
|
||||
}
|
||||
fn _get(&self, index: usize) -> color_eyre::Result<Option<&T>> {
|
||||
if self.disk_len == 0 {
|
||||
Ok(None)
|
||||
} else if index > self.disk_len - 1 {
|
||||
pub fn _get(&self, index: usize) -> color_eyre::Result<Option<&T>> {
|
||||
if self.disk_len == 0 || index > self.disk_len - 1 {
|
||||
Ok(self.cache.get(index - self.disk_len))
|
||||
} else {
|
||||
let mmap_index = Self::index_to_mmap_index(index);
|
||||
@@ -115,7 +112,11 @@ where
|
||||
.mmaps
|
||||
.get(mmap_index)
|
||||
.context("Expect mmap to be open")?
|
||||
.get_or_load(MAX_PAGE_SIZE, (mmap_index * Self::PAGE_SIZE) as u64, &self.file);
|
||||
.get_or_load(
|
||||
MAX_PAGE_SIZE,
|
||||
(mmap_index * Self::PAGE_SIZE) as u64,
|
||||
&self.file,
|
||||
);
|
||||
|
||||
let range = Self::index_to_range(index);
|
||||
let src = &mmap[range];
|
||||
@@ -150,8 +151,10 @@ where
|
||||
}
|
||||
|
||||
pub fn push_if_needed(&mut self, index: I, value: T) -> color_eyre::Result<()> {
|
||||
self._push_if_needed(index.into(), value)
|
||||
}
|
||||
pub fn _push_if_needed(&mut self, index: usize, value: T) -> color_eyre::Result<()> {
|
||||
let len = self.len();
|
||||
let index = index.into();
|
||||
match len.cmp(&index) {
|
||||
Ordering::Greater => Ok(()),
|
||||
Ordering::Equal => {
|
||||
@@ -165,23 +168,16 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AnyVecdisk {
|
||||
fn len(&self) -> usize;
|
||||
fn flush(&mut self) -> color_eyre::Result<()>;
|
||||
}
|
||||
|
||||
impl<I, T> AnyVecdisk for Vecdisk<I, T>
|
||||
where
|
||||
I: Into<usize>,
|
||||
T: Sized + Debug,
|
||||
{
|
||||
fn len(&self) -> usize {
|
||||
pub fn len(&self) -> usize {
|
||||
self.disk_len + self.cache.len()
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> color_eyre::Result<()> {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
pub fn flush(&mut self) -> io::Result<()> {
|
||||
self.disk_len += self.cache.len();
|
||||
self.reset_mmaps();
|
||||
|
||||
@@ -194,26 +190,67 @@ where
|
||||
bytes.extend_from_slice(slice)
|
||||
});
|
||||
|
||||
self.file.write_all(&bytes)?;
|
||||
|
||||
Ok(())
|
||||
self.file.write_all(&bytes)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deref, DerefMut)]
|
||||
pub trait AnyStorableVec {
|
||||
fn len(&self) -> usize;
|
||||
fn is_empty(&self) -> bool;
|
||||
fn flush(&mut self) -> io::Result<()>;
|
||||
}
|
||||
|
||||
impl<I, T> AnyStorableVec for StorableVec<I, T>
|
||||
where
|
||||
I: Into<usize>,
|
||||
T: Sized + Debug,
|
||||
{
|
||||
fn len(&self) -> usize {
|
||||
self.len()
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
self.is_empty()
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.flush()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct VecLazyMmap(Vec<LazyMmap>);
|
||||
impl VecLazyMmap {
|
||||
pub fn reset(&mut self, len: usize) {
|
||||
self.clear();
|
||||
self.resize_with(len, Default::default);
|
||||
self.0.clear();
|
||||
self.0.resize_with(len, Default::default);
|
||||
}
|
||||
}
|
||||
impl Deref for VecLazyMmap {
|
||||
type Target = Vec<LazyMmap>;
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
impl DerefMut for VecLazyMmap {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Box to reduce the size, would be 24 instead
|
||||
#[derive(Debug, Default, Deref)]
|
||||
#[derive(Debug, Default)]
|
||||
struct LazyMmap(OnceLock<Box<Mmap>>);
|
||||
impl LazyMmap {
|
||||
pub fn get_or_load(&self, len: usize, offset: u64, file: &File) -> &Mmap {
|
||||
self.get_or_init(|| Box::new(unsafe { MmapOptions::new().len(len).offset(offset).map(file).unwrap() }))
|
||||
self.0.get_or_init(|| {
|
||||
Box::new(unsafe {
|
||||
MmapOptions::new()
|
||||
.len(len)
|
||||
.offset(offset)
|
||||
.map(file)
|
||||
.unwrap()
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
use std::path::Path;
|
||||
|
||||
use storable_vec::{AnyStorableVec, StorableVec};
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
|
||||
{
|
||||
let mut vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
|
||||
|
||||
vec.push(21);
|
||||
dbg!(vec.get(0)?); // 21
|
||||
|
||||
vec.flush()?;
|
||||
}
|
||||
|
||||
{
|
||||
let vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
|
||||
|
||||
dbg!(vec.get(0)?); // 21
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user