mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-06-10 06:53:33 -07:00
general: massive update
This commit is contained in:
nym21
+6
-1
@@ -22,10 +22,15 @@
|
||||
- Added API support for datasets by timestamp (by merging any dataset by height with the height to timestamp dataset and so it still uses heights as chunk ids) [#ca00f3f](https://github.com/kibo-money/kibo/commit/ca00f3f71526f0c5c16021024fec7e5c6e47221c)
|
||||
- `/api/realized-price?kind=t`
|
||||
- `/api/realized-price?kind=timestamp&chunk=860000`
|
||||
- Created separate crate for indexing called `bindex`
|
||||
- Created a crate a storage engine specialized in storing datasets that have indexes as keys and thus can be represented by an array/vec called `storable-vec`
|
||||
- Removed the need for the `-txindex=1` parameter when starting your Bitcoin Core node as kibō has its own indexes now
|
||||
- Tried different storage engines such as `fjall`, `canopydb` and `heed`, the first ended up being 3 times slower than `sanakirja` and the rest wouldn't play nice with `rayon` which is a dealbreaker
|
||||
- `snkrj` added a robust auto defragmentation to improve disk usage without the need for user's intervention
|
||||
|
||||
## Git
|
||||
|
||||
Added git tags for each version tough Markdown won't display formatted on Github so left the default text
|
||||
Added git tags for each version though Markdown won't display formatted on Github so left the default text
|
||||
|
||||
# [v0.5.0](https://github.com/kibo-money/kibo/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
|
||||
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# Guidelines
|
||||
|
||||
## Parser
|
||||
|
||||
- Avoid floats as much as possible
|
||||
- Use structs like `Amount` and `Price` for calculations
|
||||
- **Only** use `Amount.to_btc()` when inserting or computing inside a dataset. It is **very** expensive.
|
||||
- No `Arc`, `Rc`, `Mutex` even from third party libraries, they're slower
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 kibō
|
||||
Copyright (c) 2024 kibō.money
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
Generated
+45
-485
@@ -17,16 +17,6 @@ version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
|
||||
|
||||
[[package]]
|
||||
name = "aead"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
|
||||
dependencies = [
|
||||
"crypto-common",
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.7.6"
|
||||
@@ -76,15 +66,6 @@ version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d"
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bindex"
|
||||
version = "0.1.0"
|
||||
@@ -94,8 +75,7 @@ dependencies = [
|
||||
"color-eyre",
|
||||
"ctrlc",
|
||||
"derive_deref",
|
||||
"heed3",
|
||||
"memmap2",
|
||||
"jiff",
|
||||
"rayon",
|
||||
"snkrj",
|
||||
"storable_vec",
|
||||
@@ -230,9 +210,6 @@ name = "bitflags"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
@@ -344,16 +321,6 @@ version = "0.8.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ctrlc"
|
||||
version = "3.4.5"
|
||||
@@ -386,26 +353,6 @@ dependencies = [
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "doxygen-rs"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9"
|
||||
dependencies = [
|
||||
"phf",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.13.0"
|
||||
@@ -422,15 +369,6 @@ dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs2"
|
||||
version = "0.4.3"
|
||||
@@ -441,17 +379,6 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.15"
|
||||
@@ -469,46 +396,6 @@ version = "0.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
|
||||
|
||||
[[package]]
|
||||
name = "heed-traits"
|
||||
version = "0.20.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff"
|
||||
|
||||
[[package]]
|
||||
name = "heed-types"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"byteorder",
|
||||
"heed-traits",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heed3"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7abcf1276482447d5ea921dad8c72fa5b2cfcf4c5016a16491605240bf736205"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"bitflags 2.6.0",
|
||||
"byteorder",
|
||||
"generic-array",
|
||||
"heed-traits",
|
||||
"heed-types",
|
||||
"libc",
|
||||
"lmdb-master3-sys",
|
||||
"once_cell",
|
||||
"page_size",
|
||||
"serde",
|
||||
"synchronoise",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hex-conservative"
|
||||
version = "0.2.1"
|
||||
@@ -533,145 +420,6 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd"
|
||||
|
||||
[[package]]
|
||||
name = "icu_collections"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locid"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"litemap",
|
||||
"tinystr",
|
||||
"writeable",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locid_transform"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locid",
|
||||
"icu_locid_transform_data",
|
||||
"icu_provider",
|
||||
"tinystr",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locid_transform_data"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_collections",
|
||||
"icu_normalizer_data",
|
||||
"icu_properties",
|
||||
"icu_provider",
|
||||
"smallvec",
|
||||
"utf16_iter",
|
||||
"utf8_iter",
|
||||
"write16",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer_data"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_collections",
|
||||
"icu_locid_transform",
|
||||
"icu_properties_data",
|
||||
"icu_provider",
|
||||
"tinystr",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties_data"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locid",
|
||||
"icu_provider_macros",
|
||||
"stable_deref_trait",
|
||||
"tinystr",
|
||||
"writeable",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider_macros"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
|
||||
dependencies = [
|
||||
"idna_adapter",
|
||||
"smallvec",
|
||||
"utf8_iter",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna_adapter"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"
|
||||
dependencies = [
|
||||
"icu_normalizer",
|
||||
"icu_properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indenter"
|
||||
version = "0.3.3"
|
||||
@@ -693,6 +441,35 @@ version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
|
||||
|
||||
[[package]]
|
||||
name = "jiff"
|
||||
version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2bb0c2e28117985a4d90e3bc70092bc8f226f434c7ec7e23dd9ff99c5c5721a"
|
||||
dependencies = [
|
||||
"jiff-tzdb-platform",
|
||||
"log",
|
||||
"portable-atomic",
|
||||
"portable-atomic-util",
|
||||
"serde",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jiff-tzdb"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf2cec2f5d266af45a071ece48b1fb89f3b00b2421ac3a5fe10285a6caaa60d3"
|
||||
|
||||
[[package]]
|
||||
name = "jiff-tzdb-platform"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a63c62e404e7b92979d2792352d885a7f8f83fd1d0d31eea582d77b2ceca697e"
|
||||
dependencies = [
|
||||
"jiff-tzdb",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonrpc"
|
||||
version = "0.18.0"
|
||||
@@ -717,23 +494,6 @@ version = "0.2.169"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
|
||||
|
||||
[[package]]
|
||||
name = "litemap"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104"
|
||||
|
||||
[[package]]
|
||||
name = "lmdb-master3-sys"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bea2558bc45a71673600dcd84070be37cdb7a87da2fc7255c6d377c498df00d"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"doxygen-rs",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.12"
|
||||
@@ -818,16 +578,6 @@ version = "3.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
|
||||
|
||||
[[package]]
|
||||
name = "page_size"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.2"
|
||||
@@ -853,60 +603,27 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
|
||||
dependencies = [
|
||||
"phf_shared",
|
||||
"rand",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic-util"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
|
||||
dependencies = [
|
||||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.20"
|
||||
@@ -1100,12 +817,6 @@ version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.13.2"
|
||||
@@ -1119,17 +830,10 @@ dependencies = [
|
||||
"sanakirja",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
|
||||
|
||||
[[package]]
|
||||
name = "storable_vec"
|
||||
version = "0.1.2"
|
||||
dependencies = [
|
||||
"color-eyre",
|
||||
"memmap2",
|
||||
]
|
||||
|
||||
@@ -1155,26 +859,6 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synchronoise"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2"
|
||||
dependencies = [
|
||||
"crossbeam-queue",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.69"
|
||||
@@ -1205,16 +889,6 @@ dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.41"
|
||||
@@ -1256,53 +930,18 @@ dependencies = [
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"idna",
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf16_iter"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
@@ -1404,42 +1043,6 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "write16"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
|
||||
|
||||
[[package]]
|
||||
name = "yoke"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"stable_deref_trait",
|
||||
"yoke-derive",
|
||||
"zerofrom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yoke-derive"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.35"
|
||||
@@ -1460,46 +1063,3 @@ dependencies = [
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e"
|
||||
dependencies = [
|
||||
"zerofrom-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom-derive"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
|
||||
dependencies = [
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec-derive"
|
||||
version = "0.10.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.95",
|
||||
]
|
||||
|
||||
@@ -9,8 +9,7 @@ biter = "0.2.2"
|
||||
color-eyre = "0.6.3"
|
||||
ctrlc = "3.4.5"
|
||||
derive_deref = "1.1.1"
|
||||
heed3 = "0.21.0"
|
||||
memmap2 = "0.9.5"
|
||||
jiff = "0.1.24"
|
||||
rayon = "1.10.0"
|
||||
snkrj = { path = "../snkrj" }
|
||||
storable_vec = { path = "../storable_vec" }
|
||||
|
||||
+168
-126
@@ -7,10 +7,9 @@ use std::{
|
||||
};
|
||||
|
||||
use biter::{
|
||||
bitcoin::{TxIn, TxOut, Txid},
|
||||
bitcoin::{Transaction, TxIn, TxOut, Txid},
|
||||
bitcoincore_rpc::{Auth, Client},
|
||||
};
|
||||
// use heed3::{Database, EnvOpenOptions};
|
||||
|
||||
mod structs;
|
||||
|
||||
@@ -18,10 +17,9 @@ use color_eyre::eyre::{eyre, ContextCompat};
|
||||
use rayon::prelude::*;
|
||||
use structs::{
|
||||
Addressbytes, AddressbytesPrefix, Addressindex, Addressindextxoutindex, Addresstype, Addresstypeindex, Amount,
|
||||
BlockHashPrefix, Databases, Exit, Height, StorableVecs, TxidPrefix, Txindex, Txindexvout, Txoutindex,
|
||||
BlockHashPrefix, Date, Exit, Height, Stores, Timestamp, TxidPrefix, Txindex, Txindexvout, Txoutindex, Vecs,
|
||||
};
|
||||
|
||||
// https://github.com/fjall-rs/fjall/discussions/72
|
||||
// https://github.com/romanz/electrs/blob/master/doc/schema.md
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -30,7 +28,8 @@ enum TxInOrAddressindextoutindex<'a> {
|
||||
Addressindextoutindex(Addressindextxoutindex),
|
||||
}
|
||||
|
||||
const MONTHLY_BLOCK_TARGET: usize = 144 * 30;
|
||||
const UNSAFE_BLOCKS: u32 = 100;
|
||||
const SNAPSHOT_BLOCK_RANGE: usize = 4_200; // MUST 210_000 % THIS == 0
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
@@ -46,28 +45,19 @@ fn main() -> color_eyre::Result<()> {
|
||||
let exit = Exit::new();
|
||||
|
||||
let path_database = Path::new("./database");
|
||||
let path_stores = path_database.join("stores");
|
||||
|
||||
let mut vecs = StorableVecs::import(path_database)?;
|
||||
let stores = Stores::open(&path_stores)?;
|
||||
|
||||
// let env = unsafe { EnvOpenOptions::new().open(Path::new("./heed"))? };
|
||||
// let mut wtxn = env.write_txn()?;
|
||||
let mut vecs = Vecs::import(&path_database.join("vecs"))?;
|
||||
|
||||
// let addressbytes_prefix_to_addressindex: Database<AddressbytesPrefix, Addressindex> =
|
||||
// env.create_database(&mut wtxn, Some("addressbytes_prefix_to_addressindex"))?;
|
||||
// let addressindextxoutindex_in: Database<Addressindextxoutindex, ()> =
|
||||
// env.create_database(&mut wtxn, Some("addressindextxoutindex_in"))?;
|
||||
// let addressindextxoutindex_out: Database<Addressindextxoutindex, ()> =
|
||||
// env.create_database(&mut wtxn, Some("addressindextxoutindex_out"))?;
|
||||
// let blockhash_prefix_to_height: Database<BlockHashPrefix, Height> =
|
||||
// env.create_database(&mut wtxn, Some("blockhash_prefix_to_height"))?;
|
||||
// let txid_prefix_to_txindex: Database<TxidPrefix, Txindex> =
|
||||
// env.create_database(&mut wtxn, Some("txid_prefix_to_txindex"))?;
|
||||
// let txindexvout_to_txoutindex: Database<Txindexvout, Txoutindex> =
|
||||
// env.create_database(&mut wtxn, Some("txindexvout_to_txoutindex"))?;
|
||||
|
||||
let databases = Databases::open(path_database)?;
|
||||
|
||||
let mut height = Height::from(0_u32);
|
||||
let mut height = vecs
|
||||
.min_height()
|
||||
.unwrap_or_default()
|
||||
.min(stores.min_height())
|
||||
.and_then(|h| h.checked_sub(UNSAFE_BLOCKS))
|
||||
.map(Height::from)
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut txindex = vecs
|
||||
.height_to_first_txindex
|
||||
@@ -87,56 +77,82 @@ fn main() -> color_eyre::Result<()> {
|
||||
.cloned()
|
||||
.unwrap_or(Addressindex::default());
|
||||
|
||||
let export = |databases: Databases, vecdisks: &mut StorableVecs, height: Height| -> color_eyre::Result<()> {
|
||||
let export = |stores: Stores, vecs: &mut Vecs, height: Height| -> color_eyre::Result<()> {
|
||||
exit.block();
|
||||
println!("Exporting...");
|
||||
databases.export();
|
||||
vecdisks.flush()?;
|
||||
// Memory: 2.87
|
||||
// Real Memory: 16.23
|
||||
// Private Memory: 10.8
|
||||
if height > Height::from(400_000_u32) {
|
||||
pause();
|
||||
}
|
||||
vecs.reset_cache();
|
||||
println!("Resetted cache");
|
||||
// Memory: 2.87
|
||||
// Real Memory: 13.24
|
||||
// Private Memory: 10.8
|
||||
if height > Height::from(400_000_u32) {
|
||||
pause();
|
||||
}
|
||||
vecs.flush(height)?;
|
||||
println!("Vecs flushed");
|
||||
// Memory: 3.36
|
||||
// Real Memory: 13.55
|
||||
// Private Memory: 10.66
|
||||
// Gone up wtf
|
||||
if height > Height::from(400_000_u32) {
|
||||
pause();
|
||||
}
|
||||
stores.export(height);
|
||||
println!("Export done");
|
||||
if height > Height::from(400_000_u32) {
|
||||
pause();
|
||||
}
|
||||
exit.unblock();
|
||||
Ok(())
|
||||
};
|
||||
|
||||
let mut databases_opt = Some(databases);
|
||||
let mut stores_opt = Some(stores);
|
||||
|
||||
biter::new(data_dir, Some(height.into()), Some(400_000), rpc)
|
||||
biter::new(data_dir, Some(height.into()), None, rpc)
|
||||
.iter()
|
||||
.try_for_each(|(_height, block, blockhash)| -> color_eyre::Result<()> {
|
||||
println!("Processing block {_height}...");
|
||||
|
||||
height = Height::from(_height);
|
||||
let timestamp = Timestamp::try_from(block.header.time)?;
|
||||
let date = Date::from(×tamp);
|
||||
|
||||
let mut databases = databases_opt.take().context("option should have wtx")?;
|
||||
let mut stores = stores_opt.take().context("option should have wtx")?;
|
||||
|
||||
// if let Some(saved_blockhash) = vecdisks.height_to_blockhash.get(height)? {
|
||||
// if &blockhash != saved_blockhash {
|
||||
// parts.rollback_from(&mut wtx, height, &exit)?;
|
||||
// } else {
|
||||
// wtx_opt.replace(wtx);
|
||||
// return Ok(());
|
||||
// }
|
||||
// }
|
||||
if let Some(saved_blockhash) = vecs.height_to_blockhash.get(height)? {
|
||||
if &blockhash != saved_blockhash {
|
||||
todo!("Rollback not implemented");
|
||||
// parts.rollback_from(&mut wtx, height, &exit)?;
|
||||
}
|
||||
}
|
||||
|
||||
// if parts.blockhash_prefix_to_height.needs(height) {
|
||||
let blockhash_prefix = BlockHashPrefix::try_from(&blockhash)?;
|
||||
if stores.blockhash_prefix_to_height.needs(height) {
|
||||
let blockhash_prefix = BlockHashPrefix::try_from(&blockhash)?;
|
||||
|
||||
// if check_collisions {
|
||||
// if let Some(prev_height) =
|
||||
// databases.blockhash_prefix_to_height.get(&blockhash_prefix)
|
||||
// {
|
||||
// dbg!(blockhash, prev_height);
|
||||
// return Err(eyre!("Collision, expect prefix to need be set yet"));
|
||||
// }
|
||||
// }
|
||||
if check_collisions {
|
||||
if let Some(prev_height) =
|
||||
stores.blockhash_prefix_to_height.get(&blockhash_prefix)
|
||||
{
|
||||
dbg!(blockhash, prev_height);
|
||||
return Err(eyre!("Collision, expect prefix to need be set yet"));
|
||||
}
|
||||
}
|
||||
|
||||
databases.blockhash_prefix_to_height.insert(blockhash_prefix,height);
|
||||
// blockhash_prefix_to_height.put(&mut wtxn, &blockhash_prefix,&height);
|
||||
// }
|
||||
stores.blockhash_prefix_to_height.insert(blockhash_prefix,height);
|
||||
}
|
||||
|
||||
vecs.height_to_blockhash.push_if_needed(height, blockhash)?;
|
||||
vecs.height_to_first_txindex.push_if_needed(height, txindex)?;
|
||||
vecs.height_to_first_txoutindex.push_if_needed(height, txoutindex)?;
|
||||
vecs.height_to_first_addressindex.push_if_needed(height, addressindex)?;
|
||||
vecs.height_to_timestamp.push_if_needed(height, timestamp)?;
|
||||
vecs.height_to_date.push_if_needed(height, date)?;
|
||||
|
||||
let outputs = block
|
||||
.txdata
|
||||
@@ -163,14 +179,14 @@ fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let txid_prefix = TxidPrefix::try_from(&txid)?;
|
||||
|
||||
let prev_txindex_slice_opt = if check_collisions {
|
||||
let prev_txindex_slice_opt = if check_collisions && stores.txid_prefix_to_txindex.needs(height) {
|
||||
// Should only find collisions for two txids (duplicates), see below
|
||||
databases.txid_prefix_to_txindex.get(&txid_prefix).cloned()
|
||||
stores.txid_prefix_to_txindex.get(&txid_prefix).cloned()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok((txid_prefix, (txid, Txindex::from(index), prev_txindex_slice_opt)))
|
||||
Ok((txid_prefix, (tx, txid, Txindex::from(index), prev_txindex_slice_opt)))
|
||||
})
|
||||
.try_fold(
|
||||
BTreeMap::new,
|
||||
@@ -201,8 +217,11 @@ fn main() -> color_eyre::Result<()> {
|
||||
let txid = outpoint.txid;
|
||||
let vout = outpoint.vout;
|
||||
|
||||
let txindex_local = if let Some(txindex_local) = databases.txid_prefix_to_txindex
|
||||
.get(&TxidPrefix::try_from(&txid)?)
|
||||
let txindex_local = if let Some(txindex_local) = stores.txid_prefix_to_txindex
|
||||
.get(&TxidPrefix::try_from(&txid)?).and_then(|txindex_local| {
|
||||
// Checking if not finding txindex from the future
|
||||
(txindex_local < &txindex).then_some(txindex_local)
|
||||
})
|
||||
{
|
||||
*txindex_local
|
||||
} else {
|
||||
@@ -212,11 +231,9 @@ fn main() -> color_eyre::Result<()> {
|
||||
let txindexvout = Txindexvout::from((txindex_local, vout));
|
||||
|
||||
let txoutindex =
|
||||
*databases.txindexvout_to_txoutindex.get(&txindexvout)
|
||||
*stores.txindexvout_to_txoutindex.get(&txindexvout)
|
||||
.context("Expect txoutindex to not be none")
|
||||
.inspect_err(|_| {
|
||||
// let height = vecdisks.txindex_to_height.get(txindex.into()).expect("txindex_to_height get not fail")
|
||||
// .expect("Expect height for txindex");
|
||||
dbg!(outpoint.txid, txindex_local, vout, txindexvout);
|
||||
})?;
|
||||
|
||||
@@ -236,13 +253,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
.try_fold(
|
||||
Vec::new,
|
||||
|mut vec, addressindextxoutindex| {
|
||||
// There is no need to check for bad_tx as there are only 2 instances known
|
||||
// Which you can find below and which are coinbase tx and thus which are already filtered
|
||||
|
||||
// if parts.addressindextxoutindex_out.needs(height) {
|
||||
vec.push(addressindextxoutindex?);
|
||||
// }
|
||||
|
||||
vec.push(addressindextxoutindex?);
|
||||
Ok(vec)
|
||||
},
|
||||
)
|
||||
@@ -257,11 +268,12 @@ fn main() -> color_eyre::Result<()> {
|
||||
})
|
||||
});
|
||||
|
||||
let txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle = scope.spawn(|| -> color_eyre::Result<BTreeMap<Txoutindex,
|
||||
(&TxOut, Txindexvout, Addresstype, color_eyre::Result<Addressbytes>, Option<Addressindex>)>> {
|
||||
let txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt_handle = scope.spawn(|| {
|
||||
outputs.into_par_iter().enumerate()
|
||||
.map(
|
||||
|(block_txoutindex, (block_txindex, vout, txout))| {
|
||||
#[allow(clippy::type_complexity)]
|
||||
|(block_txoutindex, (block_txindex, vout, txout))| -> color_eyre::Result<(Txoutindex,
|
||||
(&TxOut, Txindexvout, Addresstype, color_eyre::Result<Addressbytes>, Option<Addressindex>))> {
|
||||
let txindex_local = txindex + block_txindex;
|
||||
let txindexvout = Txindexvout::from((txindex_local, vout));
|
||||
let txoutindex_local = txoutindex + Txoutindex::from(block_txoutindex);
|
||||
@@ -274,28 +286,40 @@ fn main() -> color_eyre::Result<()> {
|
||||
// dbg!(&txout, height, txi, &tx.compute_txid());
|
||||
});
|
||||
|
||||
let addressindex_slice_opt = addressbytes_res.as_ref().ok().and_then(|addressbytes| {
|
||||
databases.addressbytes_prefix_to_addressindex.get(
|
||||
&AddressbytesPrefix::try_from(addressbytes).unwrap(),
|
||||
).cloned()
|
||||
let addressindex_opt = addressbytes_res.as_ref().ok().and_then(|addressbytes| {
|
||||
stores.addressbytes_prefix_to_addressindex.get(
|
||||
&AddressbytesPrefix::from((addressbytes, addresstype)),
|
||||
)
|
||||
.cloned()
|
||||
// Checking if not in the future
|
||||
.and_then(|addressindex_local| (addressindex_local < addressindex)
|
||||
.then_some(addressindex_local))
|
||||
});
|
||||
|
||||
let is_new_address = addressindex_slice_opt.is_none();
|
||||
if let Some(Some(addressindex)) = check_collisions.then_some(addressindex_opt) {
|
||||
let addressbytes = addressbytes_res.as_ref().unwrap();
|
||||
|
||||
if check_collisions && is_new_address {
|
||||
if let Ok(addressbytes) = &addressbytes_res {
|
||||
if let Some(prev) = databases.addressbytes_prefix_to_addressindex.get(
|
||||
&AddressbytesPrefix::try_from(addressbytes)?,
|
||||
) {
|
||||
dbg!(prev);
|
||||
return Err(eyre!("addressbytes_prefix_to_addressindex collision, expect none"));
|
||||
}
|
||||
let prev_addresstype = *vecs.addressindex_to_addresstype.get(
|
||||
addressindex,
|
||||
)?.context("Expect to have address type")?;
|
||||
|
||||
let addresstypeindex = *vecs.addressindex_to_addresstypeindex.get(
|
||||
addressindex,
|
||||
)?.context("Expect to have address type index")?;
|
||||
|
||||
let prev_addressbytes_opt= vecs.get_addressbytes(prev_addresstype, addresstypeindex)?;
|
||||
|
||||
let prev_addressbytes = prev_addressbytes_opt.as_ref().context("Expect to have addressbytes")?;
|
||||
|
||||
if (vecs.addressindex_to_addresstype.hasnt(addressindex) && addresstype != prev_addresstype) || (stores.addressbytes_prefix_to_addressindex.needs(height) && prev_addressbytes != addressbytes) {
|
||||
dbg!(addresstype, prev_addresstype, prev_addressbytes, addressbytes, addressindex, addresstypeindex, txout, AddressbytesPrefix::from((addressbytes, addresstype)), AddressbytesPrefix::from((prev_addressbytes, prev_addresstype)));
|
||||
panic!()
|
||||
}
|
||||
}
|
||||
|
||||
Ok((
|
||||
txoutindex_local,
|
||||
(txout, txindexvout, addresstype, addressbytes_res, addressindex_slice_opt),
|
||||
(txout, txindexvout, addresstype, addressbytes_res, addressindex_opt),
|
||||
))
|
||||
},
|
||||
)
|
||||
@@ -329,17 +353,18 @@ fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let mut new_txindexvout_to_addressindextxoutindex: BTreeMap<Txindexvout, Addressindextxoutindex> = BTreeMap::new();
|
||||
|
||||
let mut already_added_addressbytes_prefix: BTreeMap<AddressbytesPrefix, Addressindex> = BTreeMap::new();
|
||||
|
||||
txoutindex_to_txout_addresstype_addressbytes_res_addressindex_opt
|
||||
.into_iter()
|
||||
.try_for_each(|(txoutindex, (txout, txindexvout, addresstype, addressbytes_res, addressindex_opt))| -> color_eyre::Result<()> {
|
||||
let amount = Amount::from(txout.value);
|
||||
|
||||
// if parts.txindexvout_to_txoutindex.needs(height) {
|
||||
databases.txindexvout_to_txoutindex.insert(
|
||||
txindexvout,
|
||||
txoutindex,
|
||||
);
|
||||
// }
|
||||
stores.txindexvout_to_txoutindex.insert_if_needed(
|
||||
txindexvout,
|
||||
txoutindex,
|
||||
height,
|
||||
);
|
||||
|
||||
vecs.txoutindex_to_amount.push_if_needed(
|
||||
txoutindex,
|
||||
@@ -348,49 +373,63 @@ fn main() -> color_eyre::Result<()> {
|
||||
|
||||
let mut addressindex_local = addressindex;
|
||||
|
||||
if let Some(addressindex) = addressindex_opt {
|
||||
let mut addressbytes_prefix= None;
|
||||
|
||||
if let Some(addressindex) = addressindex_opt.or_else(|| addressbytes_res.as_ref().ok().and_then(|addressbytes| {
|
||||
// Check if address was first seen before in this iterator
|
||||
// Example: https://mempool.space/address/046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0c
|
||||
addressbytes_prefix.replace(AddressbytesPrefix::from((addressbytes, addresstype)));
|
||||
already_added_addressbytes_prefix.get(
|
||||
addressbytes_prefix.as_ref().unwrap(),
|
||||
).cloned()
|
||||
})) {
|
||||
addressindex_local = addressindex;
|
||||
} else {
|
||||
vecs.addressindex_to_addresstype.push_if_needed(addressindex_local, addresstype)?;
|
||||
addressindex.increment();
|
||||
|
||||
// TODO: Create counter of other addresstypes instead
|
||||
let addresstypeindex = Addresstypeindex::from(vecs.addresstype_to_addressbytes(addresstype).map_or(0, |vecdisk| vecdisk.len()));
|
||||
let addresstypeindex = Addresstypeindex::from(vecs.addresstype_to_addressbytes(addresstype).map_or(0, |vec| vec.len()));
|
||||
|
||||
vecs.addressindex_to_addresstype.push_if_needed(addressindex_local, addresstype)?;
|
||||
|
||||
vecs.addressindex_to_addresstypeindex.push_if_needed(addressindex_local, addresstypeindex)?;
|
||||
|
||||
if let Ok(addressbytes) = addressbytes_res {
|
||||
// if parts.addressbytes_prefix_to_addressindex.needs(height) {
|
||||
databases.addressbytes_prefix_to_addressindex.insert(
|
||||
AddressbytesPrefix::try_from(&addressbytes)?,
|
||||
addressindex_local,
|
||||
);
|
||||
// }
|
||||
let addressbytes_prefix = addressbytes_prefix.unwrap();
|
||||
|
||||
already_added_addressbytes_prefix.insert(addressbytes_prefix.clone(), addressindex_local);
|
||||
|
||||
stores.addressbytes_prefix_to_addressindex.insert_if_needed(
|
||||
addressbytes_prefix,
|
||||
addressindex_local,
|
||||
height
|
||||
);
|
||||
|
||||
vecs.push_addressbytes_if_needed(addresstypeindex, addressbytes)?;
|
||||
}
|
||||
|
||||
addressindex.increment();
|
||||
}
|
||||
|
||||
new_txindexvout_to_addressindextxoutindex.insert(txindexvout, Addressindextxoutindex::from((addressindex_local, txoutindex)));
|
||||
let addressindextxoutindex = Addressindextxoutindex::from((addressindex_local, txoutindex));
|
||||
|
||||
new_txindexvout_to_addressindextxoutindex.insert(txindexvout, addressindextxoutindex);
|
||||
|
||||
vecs.txoutindex_to_addressindex.push_if_needed(
|
||||
txoutindex,
|
||||
addressindex_local,
|
||||
)?;
|
||||
|
||||
// if parts.addressindextxoutindex_in.needs(height) {
|
||||
let addressindextxoutindex = Addressindextxoutindex::from((addressindex_local, txoutindex));
|
||||
databases.addressindextxoutindex_in.insert(
|
||||
addressindextxoutindex,
|
||||
(),
|
||||
);
|
||||
// }
|
||||
stores.addressindextxoutindex_in.insert_if_needed(
|
||||
addressindextxoutindex,
|
||||
(),
|
||||
height,
|
||||
);
|
||||
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
// if parts.addressindextxoutindex_out.needs(height) {
|
||||
drop(already_added_addressbytes_prefix);
|
||||
|
||||
if stores.addressindextxoutindex_out.needs(height) {
|
||||
txin_or_addressindextxoutindex_vec
|
||||
.into_iter()
|
||||
.map(|txin_or_addressindextxoutindex| -> color_eyre::Result<Addressindextxoutindex> {
|
||||
@@ -402,7 +441,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
let vout = outpoint.vout;
|
||||
let index = txid_prefix_to_txid_and_block_txindex_and_prev_txindex
|
||||
.get(&TxidPrefix::try_from(&txid)?)
|
||||
.context("txid should be in same block")?.1;
|
||||
.context("txid should be in same block")?.2;
|
||||
let txindex_local = txindex + index;
|
||||
|
||||
let txindexvout = Txindexvout::from((txindex_local, vout));
|
||||
@@ -416,30 +455,28 @@ fn main() -> color_eyre::Result<()> {
|
||||
}
|
||||
})
|
||||
.try_for_each(|addressindextxoutindex| -> color_eyre::Result<()> {
|
||||
databases.addressindextxoutindex_out.insert(
|
||||
stores.addressindextxoutindex_out.insert(
|
||||
addressindextxoutindex?,
|
||||
(),
|
||||
);
|
||||
Ok(())
|
||||
})?;
|
||||
// }
|
||||
}
|
||||
|
||||
drop(new_txindexvout_to_addressindextxoutindex);
|
||||
|
||||
let mut txindex_to_txid: BTreeMap<Txindex, Txid> = BTreeMap::default();
|
||||
let mut txindex_to_tx_and_txid: BTreeMap<Txindex, (&Transaction, Txid)> = BTreeMap::default();
|
||||
|
||||
txid_prefix_to_txid_and_block_txindex_and_prev_txindex.into_iter().try_for_each(
|
||||
|(txid_prefix, (txid, index, prev_txindex_opt))| -> color_eyre::Result<()> {
|
||||
|(txid_prefix, (tx, txid, index, prev_txindex_opt))| -> color_eyre::Result<()> {
|
||||
let txindex_local = txindex + index;
|
||||
|
||||
txindex_to_txid.insert(txindex_local, txid);
|
||||
txindex_to_tx_and_txid.insert(txindex_local, (tx, txid));
|
||||
|
||||
match prev_txindex_opt {
|
||||
None => {
|
||||
// if parts.txid_prefix_to_txindex.needs(height) {
|
||||
databases.txid_prefix_to_txindex.insert(txid_prefix, txindex_local);
|
||||
// }
|
||||
}
|
||||
stores.txid_prefix_to_txindex.insert_if_needed(txid_prefix, txindex_local, height);
|
||||
},
|
||||
Some(prev_txindex) => {
|
||||
// In case if we start at an already parsed height
|
||||
if txindex_local == prev_txindex {
|
||||
@@ -475,9 +512,12 @@ fn main() -> color_eyre::Result<()> {
|
||||
},
|
||||
)?;
|
||||
|
||||
txindex_to_txid.into_iter().try_for_each(|(txindex, txid)| -> color_eyre::Result<()> {
|
||||
txindex_to_tx_and_txid.into_iter().try_for_each(|(txindex, (tx, txid))| -> color_eyre::Result<()> {
|
||||
vecs.txindex_to_txversion.push_if_needed(txindex, tx.version)?;
|
||||
vecs.txindex_to_txid.push_if_needed(txindex, txid)?;
|
||||
vecs.txindex_to_height.push_if_needed(txindex, height)?;
|
||||
vecs.txindex_to_inputcount.push_if_needed(txindex, tx.input.len() as u32)?;
|
||||
vecs.txindex_to_outputcount.push_if_needed(txindex, tx.output.len() as u32)?;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
@@ -485,12 +525,12 @@ fn main() -> color_eyre::Result<()> {
|
||||
vecs.height_to_last_txoutindex.push_if_needed(height, txoutindex.decremented())?;
|
||||
vecs.height_to_last_addressindex.push_if_needed(height, addressindex.decremented())?;
|
||||
|
||||
let should_snapshot = _height % MONTHLY_BLOCK_TARGET == 0 && !exit.active();
|
||||
let should_snapshot = _height % SNAPSHOT_BLOCK_RANGE == 0 && !exit.active();
|
||||
if should_snapshot {
|
||||
export(databases, &mut vecs, height)?;
|
||||
databases_opt.replace(Databases::open(path_database)?);
|
||||
export(stores, &mut vecs, height)?;
|
||||
stores_opt.replace(Stores::open(&path_stores)?);
|
||||
} else {
|
||||
databases_opt.replace(databases);
|
||||
stores_opt.replace(stores);
|
||||
}
|
||||
|
||||
txindex += Txindex::from(tx_len);
|
||||
@@ -503,8 +543,10 @@ fn main() -> color_eyre::Result<()> {
|
||||
|
||||
pause();
|
||||
|
||||
let databases = databases_opt.take().context("option should have wtx")?;
|
||||
export(databases, &mut vecs, height)?;
|
||||
let stores = stores_opt.take().context("option should have wtx")?;
|
||||
export(stores, &mut vecs, height)?;
|
||||
|
||||
pause();
|
||||
|
||||
dbg!(i.elapsed());
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ use derive_deref::{Deref, DerefMut};
|
||||
|
||||
use super::Addresstype;
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Addressbytes {
|
||||
P2PK65(P2PK65AddressBytes),
|
||||
P2PK33(P2PK33AddressBytes),
|
||||
@@ -15,6 +15,20 @@ pub enum Addressbytes {
|
||||
P2TR(P2TRAddressBytes),
|
||||
}
|
||||
|
||||
impl Addressbytes {
|
||||
pub fn as_slice(&self) -> &[u8] {
|
||||
match self {
|
||||
Addressbytes::P2PK65(bytes) => &bytes[..],
|
||||
Addressbytes::P2PK33(bytes) => &bytes[..],
|
||||
Addressbytes::P2PKH(bytes) => &bytes[..],
|
||||
Addressbytes::P2SH(bytes) => &bytes[..],
|
||||
Addressbytes::P2WPKH(bytes) => &bytes[..],
|
||||
Addressbytes::P2WSH(bytes) => &bytes[..],
|
||||
Addressbytes::P2TR(bytes) => &bytes[..],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from(tuple: (&ScriptBuf, Addresstype)) -> Result<Self, Self::Error> {
|
||||
@@ -72,28 +86,64 @@ impl TryFrom<(&ScriptBuf, Addresstype)> for Addressbytes {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deref)]
|
||||
impl From<P2PK65AddressBytes> for Addressbytes {
|
||||
fn from(value: P2PK65AddressBytes) -> Self {
|
||||
Self::P2PK65(value)
|
||||
}
|
||||
}
|
||||
impl From<P2PK33AddressBytes> for Addressbytes {
|
||||
fn from(value: P2PK33AddressBytes) -> Self {
|
||||
Self::P2PK33(value)
|
||||
}
|
||||
}
|
||||
impl From<P2PKHAddressBytes> for Addressbytes {
|
||||
fn from(value: P2PKHAddressBytes) -> Self {
|
||||
Self::P2PKH(value)
|
||||
}
|
||||
}
|
||||
impl From<P2SHAddressBytes> for Addressbytes {
|
||||
fn from(value: P2SHAddressBytes) -> Self {
|
||||
Self::P2SH(value)
|
||||
}
|
||||
}
|
||||
impl From<P2WPKHAddressBytes> for Addressbytes {
|
||||
fn from(value: P2WPKHAddressBytes) -> Self {
|
||||
Self::P2WPKH(value)
|
||||
}
|
||||
}
|
||||
impl From<P2WSHAddressBytes> for Addressbytes {
|
||||
fn from(value: P2WSHAddressBytes) -> Self {
|
||||
Self::P2WSH(value)
|
||||
}
|
||||
}
|
||||
impl From<P2TRAddressBytes> for Addressbytes {
|
||||
fn from(value: P2TRAddressBytes) -> Self {
|
||||
Self::P2TR(value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
|
||||
pub struct P2PK65AddressBytes(U8x65);
|
||||
|
||||
#[derive(Debug, Clone, Deref)]
|
||||
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
|
||||
pub struct P2PK33AddressBytes(U8x33);
|
||||
|
||||
#[derive(Debug, Clone, Deref)]
|
||||
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
|
||||
pub struct P2PKHAddressBytes(U8x20);
|
||||
|
||||
#[derive(Debug, Clone, Deref)]
|
||||
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
|
||||
pub struct P2SHAddressBytes(U8x20);
|
||||
|
||||
#[derive(Debug, Clone, Deref)]
|
||||
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
|
||||
pub struct P2WPKHAddressBytes(U8x20);
|
||||
|
||||
#[derive(Debug, Clone, Deref)]
|
||||
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
|
||||
pub struct P2WSHAddressBytes(U8x32);
|
||||
|
||||
#[derive(Debug, Clone, Deref)]
|
||||
#[derive(Debug, Clone, Deref, PartialEq, Eq)]
|
||||
pub struct P2TRAddressBytes(U8x32);
|
||||
|
||||
#[derive(Debug, Clone, Deref, DerefMut)]
|
||||
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
|
||||
pub struct U8x20([u8; 20]);
|
||||
impl From<&[u8]> for U8x20 {
|
||||
fn from(slice: &[u8]) -> Self {
|
||||
@@ -103,7 +153,7 @@ impl From<&[u8]> for U8x20 {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deref, DerefMut)]
|
||||
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
|
||||
pub struct U8x32([u8; 32]);
|
||||
impl From<&[u8]> for U8x32 {
|
||||
fn from(slice: &[u8]) -> Self {
|
||||
@@ -113,7 +163,7 @@ impl From<&[u8]> for U8x32 {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deref, DerefMut)]
|
||||
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
|
||||
pub struct U8x33([u8; 33]);
|
||||
impl From<&[u8]> for U8x33 {
|
||||
fn from(slice: &[u8]) -> Self {
|
||||
@@ -123,7 +173,7 @@ impl From<&[u8]> for U8x33 {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deref, DerefMut)]
|
||||
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
|
||||
pub struct U8x64([u8; 64]);
|
||||
impl From<&[u8]> for U8x64 {
|
||||
fn from(slice: &[u8]) -> Self {
|
||||
@@ -133,7 +183,7 @@ impl From<&[u8]> for U8x64 {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deref, DerefMut)]
|
||||
#[derive(Debug, Clone, Deref, DerefMut, PartialEq, Eq)]
|
||||
pub struct U8x65([u8; 65]);
|
||||
impl From<&[u8]> for U8x65 {
|
||||
fn from(slice: &[u8]) -> Self {
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
use jiff::tz::TimeZone;
|
||||
|
||||
use super::Timestamp;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Date(jiff::civil::Date);
|
||||
|
||||
impl From<&Timestamp> for Date {
|
||||
fn from(value: &Timestamp) -> Self {
|
||||
Self(jiff::civil::Date::from(value.to_zoned(TimeZone::UTC)))
|
||||
}
|
||||
}
|
||||
@@ -1,16 +1,24 @@
|
||||
use std::{
|
||||
fmt,
|
||||
ops::{Add, AddAssign, Sub},
|
||||
fmt, fs, io,
|
||||
ops::{Add, AddAssign, Rem, Sub},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
use biter::bitcoincore_rpc::{self, RpcApi};
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use snkrj::{direct_repr, Storable, UnsizedStorable};
|
||||
use storable_vec::UnsafeSizedSerDe;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord, Default)]
|
||||
pub struct Height(u32);
|
||||
direct_repr!(Height);
|
||||
|
||||
impl Height {
|
||||
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
|
||||
fs::write(path, self.unsafe_as_slice())
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq<u64> for Height {
|
||||
fn eq(&self, other: &u64) -> bool {
|
||||
**self == *other as u32
|
||||
@@ -68,6 +76,13 @@ impl AddAssign<usize> for Height {
|
||||
}
|
||||
}
|
||||
|
||||
impl Rem<usize> for Height {
|
||||
type Output = Height;
|
||||
fn rem(self, rhs: usize) -> Self::Output {
|
||||
Self(self.abs_diff(Height::from(rhs).0))
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Height {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", **self)
|
||||
@@ -91,6 +106,13 @@ impl From<Height> for usize {
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<&Path> for Height {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from(value: &Path) -> Result<Self, Self::Error> {
|
||||
Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<&bitcoincore_rpc::Client> for Height {
|
||||
type Error = bitcoincore_rpc::Error;
|
||||
fn try_from(value: &bitcoincore_rpc::Client) -> Result<Self, Self::Error> {
|
||||
|
||||
@@ -4,15 +4,20 @@ mod addressindextxoutindex;
|
||||
mod addresstype;
|
||||
mod addresstypeindex;
|
||||
mod amount;
|
||||
mod databases;
|
||||
mod date;
|
||||
mod exit;
|
||||
mod height;
|
||||
mod prefix;
|
||||
mod slice;
|
||||
mod storable_vecs;
|
||||
mod store;
|
||||
mod stores;
|
||||
mod timestamp;
|
||||
mod txindex;
|
||||
mod txindexvout;
|
||||
mod txoutindex;
|
||||
mod vec;
|
||||
mod vecs;
|
||||
mod version;
|
||||
|
||||
pub use addressbytes::*;
|
||||
pub use addressindex::*;
|
||||
@@ -20,12 +25,17 @@ pub use addressindextxoutindex::*;
|
||||
pub use addresstype::*;
|
||||
pub use addresstypeindex::*;
|
||||
pub use amount::*;
|
||||
pub use databases::*;
|
||||
pub use date::*;
|
||||
pub use exit::*;
|
||||
pub use height::*;
|
||||
pub use prefix::*;
|
||||
pub use slice::*;
|
||||
pub use storable_vecs::*;
|
||||
pub use store::*;
|
||||
pub use stores::*;
|
||||
pub use timestamp::*;
|
||||
pub use txindex::*;
|
||||
pub use txindexvout::*;
|
||||
pub use txoutindex::*;
|
||||
pub use vec::*;
|
||||
pub use vecs::*;
|
||||
pub use version::*;
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
pub use fjall::{PartitionCreateOptions, PersistMode, Result, TransactionalKeyspace, TransactionalPartitionHandle};
|
||||
|
||||
use crate::structs::{Height, Version};
|
||||
|
||||
use super::Exit;
|
||||
|
||||
pub struct Partition {
|
||||
version: Version,
|
||||
data: TransactionalPartitionHandle,
|
||||
meta: TransactionalPartitionHandle,
|
||||
height: Option<Height>,
|
||||
}
|
||||
|
||||
impl Partition {
|
||||
pub const VERSION: &str = "version";
|
||||
pub const HEIGHT: &str = "height";
|
||||
|
||||
pub fn import(
|
||||
keyspace: &TransactionalKeyspace,
|
||||
name: &str,
|
||||
version: Version,
|
||||
exit: &Exit,
|
||||
) -> color_eyre::Result<Self> {
|
||||
let data = Self::open_data(keyspace, name)?;
|
||||
let meta = Self::open_meta(keyspace, name)?;
|
||||
|
||||
let mut height = None;
|
||||
if let Some(height_res) = meta.get(Self::HEIGHT)?.map(Height::try_from) {
|
||||
height = Some(height_res?);
|
||||
}
|
||||
|
||||
let mut this = Self {
|
||||
version,
|
||||
height,
|
||||
data,
|
||||
meta,
|
||||
};
|
||||
|
||||
let mut different_version = false;
|
||||
if let Some(slice) = this.meta.get(Self::VERSION)? {
|
||||
different_version = Version::try_from(slice).map_or(true, |version2| version != version2);
|
||||
}
|
||||
|
||||
if different_version {
|
||||
this = this.reset(keyspace, name, exit)?;
|
||||
}
|
||||
|
||||
Ok(this)
|
||||
}
|
||||
|
||||
fn open_data(keyspace: &TransactionalKeyspace, name: &str) -> Result<TransactionalPartitionHandle> {
|
||||
keyspace.open_partition(&format!("{name}_data"), Self::create_options())
|
||||
}
|
||||
|
||||
fn open_meta(keyspace: &TransactionalKeyspace, name: &str) -> Result<TransactionalPartitionHandle> {
|
||||
keyspace.open_partition(&format!("{name}_meta"), Self::create_options())
|
||||
}
|
||||
|
||||
fn create_options() -> PartitionCreateOptions {
|
||||
PartitionCreateOptions::default().manual_journal_persist(true)
|
||||
}
|
||||
|
||||
// TODO: Still needed ?
|
||||
pub fn is_safe(&self, height: Height) -> bool {
|
||||
self.height.is_some_and(|self_height| self_height >= height)
|
||||
}
|
||||
|
||||
pub fn needs(&self, height: Height) -> bool {
|
||||
!self.is_safe(height)
|
||||
}
|
||||
|
||||
pub fn version(&self) -> Version {
|
||||
self.version
|
||||
}
|
||||
|
||||
pub fn data(&self) -> &TransactionalPartitionHandle {
|
||||
&self.data
|
||||
}
|
||||
|
||||
pub fn meta(&self) -> &TransactionalPartitionHandle {
|
||||
&self.meta
|
||||
}
|
||||
|
||||
fn reset(mut self, keyspace: &TransactionalKeyspace, name: &str, exit: &Exit) -> Result<Self> {
|
||||
exit.block();
|
||||
|
||||
keyspace.delete_partition(self.data)?;
|
||||
keyspace.delete_partition(self.meta)?;
|
||||
|
||||
keyspace.persist(PersistMode::SyncAll)?;
|
||||
|
||||
self.data = Self::open_data(keyspace, name)?;
|
||||
self.meta = Self::open_meta(keyspace, name)?;
|
||||
self.height = None;
|
||||
|
||||
exit.unblock();
|
||||
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
pub fn height(&self) -> &Option<Height> {
|
||||
&self.height
|
||||
}
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
use std::ops::Sub;
|
||||
|
||||
use fjall::{Slice, TransactionalKeyspace, WriteTransaction};
|
||||
|
||||
use crate::structs::{Exit, Height, Partition, Version};
|
||||
|
||||
pub struct Partitions {
|
||||
pub addressbytes_prefix_to_addressindex: Partition,
|
||||
pub addressindextxoutindex_in: Partition,
|
||||
pub addressindextxoutindex_out: Partition,
|
||||
pub blockhash_prefix_to_height: Partition,
|
||||
pub txid_prefix_to_txindex: Partition,
|
||||
pub txindexvout_to_txoutindex: Partition,
|
||||
}
|
||||
|
||||
const UNSAFE_BLOCKS: usize = 100;
|
||||
|
||||
impl Partitions {
|
||||
pub fn import(keyspace: &TransactionalKeyspace, exit: &Exit) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
addressbytes_prefix_to_addressindex: Partition::import(
|
||||
keyspace,
|
||||
"addressbytes_prefix_to_addressindex",
|
||||
Version::from(1),
|
||||
exit,
|
||||
)?,
|
||||
addressindextxoutindex_in: Partition::import(keyspace, "addresstxoutindexes_in", Version::from(1), exit)?,
|
||||
addressindextxoutindex_out: Partition::import(keyspace, "addresstxoutindexes_out", Version::from(1), exit)?,
|
||||
blockhash_prefix_to_height: Partition::import(
|
||||
keyspace,
|
||||
"blockhash_prefix_to_height",
|
||||
Version::from(1),
|
||||
exit,
|
||||
)?,
|
||||
txid_prefix_to_txindex: Partition::import(keyspace, "txid_prefix_to_txindex", Version::from(1), exit)?,
|
||||
txindexvout_to_txoutindex: Partition::import(
|
||||
keyspace,
|
||||
"txindexvout_to_txoutindex",
|
||||
Version::from(1),
|
||||
exit,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn udpate_meta(&self, wtx: &mut WriteTransaction, height: Height) {
|
||||
self.to_vec().into_iter().for_each(|part| {
|
||||
let meta = part.meta();
|
||||
wtx.insert(meta, Partition::VERSION, Slice::from(part.version()));
|
||||
wtx.insert(meta, Partition::HEIGHT, height.to_be_bytes());
|
||||
});
|
||||
}
|
||||
|
||||
pub fn start_height(&self) -> Height {
|
||||
self.min_height()
|
||||
.map(|height| height.sub(UNSAFE_BLOCKS))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn min_height(&self) -> Option<Height> {
|
||||
self.to_vec()
|
||||
.into_iter()
|
||||
.map(|part| part.height())
|
||||
.map(ToOwned::to_owned)
|
||||
.min()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
pub fn rollback_from(
|
||||
&mut self,
|
||||
_wtx: &mut WriteTransaction,
|
||||
_height: Height,
|
||||
_exit: &Exit,
|
||||
) -> color_eyre::Result<()> {
|
||||
panic!();
|
||||
// let mut txindex = None;
|
||||
|
||||
// wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
|
||||
// .try_for_each(|slice| -> color_eyre::Result<()> {
|
||||
// let (height_slice, slice_blockhash) = slice?;
|
||||
// let blockhash = BlockHash::from_slice(&slice_blockhash)?;
|
||||
|
||||
// wtx.remove(self.height_to_blockhash.data(), height_slice);
|
||||
|
||||
// wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
|
||||
|
||||
// if txindex.is_none() {
|
||||
// txindex.replace(
|
||||
// wtx.get(self.height_to_first_txindex.data(), height_slice)?
|
||||
// .context("for height to have first txindex")?,
|
||||
// );
|
||||
// }
|
||||
// wtx.remove(self.height_to_first_txindex.data(), height_slice);
|
||||
// wtx.remove(self.height_to_last_txindex.data(), height_slice);
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
|
||||
// let txindex = txindex.context("txindex to not be none by now")?;
|
||||
|
||||
// wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
|
||||
// .try_for_each(|slice| -> color_eyre::Result<()> {
|
||||
// let (slice_txindex, slice_txid) = slice?;
|
||||
// let txindex = Txindex::from(slice_txindex);
|
||||
// let txid = Txid::from_slice(&slice_txid)?;
|
||||
|
||||
// wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
|
||||
// wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
|
||||
// wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
|
||||
// let txoutindex = Txoutindex::from(txindex);
|
||||
|
||||
// let mut addressindexes = BTreeSet::new();
|
||||
|
||||
// wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
|
||||
// .try_for_each(|slice| -> color_eyre::Result<()> {
|
||||
// let (txoutindex_slice, _) = slice?;
|
||||
|
||||
// wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
|
||||
|
||||
// if let Some(addressindex_slice) =
|
||||
// wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
|
||||
// {
|
||||
// wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
|
||||
|
||||
// let addressindex = Addressindex::from(addressindex_slice);
|
||||
// addressindexes.insert(addressindex);
|
||||
|
||||
// let txoutindex = Txoutindex::from(txoutindex_slice);
|
||||
// let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
|
||||
|
||||
// wtx.remove(
|
||||
// self.addressindex_to_txoutindexes.data(),
|
||||
// Slice::from(addresstxoutindex),
|
||||
// );
|
||||
// }
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
|
||||
// addressindexes
|
||||
// .into_iter()
|
||||
// .filter(|addressindex| {
|
||||
// let is_empty = wtx
|
||||
// .prefix(
|
||||
// self.addressindex_to_txoutindexes.data(),
|
||||
// Slice::from(*addressindex),
|
||||
// )
|
||||
// .next()
|
||||
// .is_none();
|
||||
// is_empty
|
||||
// })
|
||||
// .try_for_each(|addressindex| -> color_eyre::Result<()> {
|
||||
// let addressindex_slice = Slice::from(addressindex);
|
||||
|
||||
// let addressbytes = Addressbytes::from(
|
||||
// wtx.get(
|
||||
// self.addressindex_to_addressbytes.data(),
|
||||
// &addressindex_slice,
|
||||
// )?
|
||||
// .context("addressindex_to_address to have value")?,
|
||||
// );
|
||||
// wtx.remove(
|
||||
// self.addressbytes_prefix_to_addressindex.data(),
|
||||
// addressbytes.prefix(),
|
||||
// );
|
||||
// wtx.remove(
|
||||
// self.addressindex_to_addressbytes.data(),
|
||||
// &addressindex_slice,
|
||||
// );
|
||||
// wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
//
|
||||
|
||||
// todo!("clear addresstxoutindexes_out")
|
||||
// todo!("clear addresstxoutindexes_in")
|
||||
// todo!("clear zero_txoutindexes")
|
||||
// todo!("clear txindexvout_to_txoutindex")
|
||||
|
||||
// Ok(())
|
||||
}
|
||||
|
||||
fn to_vec(&self) -> Vec<&Partition> {
|
||||
vec![
|
||||
&self.addressbytes_prefix_to_addressindex,
|
||||
&self.addressindextxoutindex_in,
|
||||
&self.addressindextxoutindex_out,
|
||||
&self.blockhash_prefix_to_height,
|
||||
&self.txid_prefix_to_txindex,
|
||||
&self.txindexvout_to_txoutindex,
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -2,52 +2,49 @@ use biter::bitcoin::{BlockHash, Txid};
|
||||
use derive_deref::Deref;
|
||||
use snkrj::{direct_repr, Storable, UnsizedStorable};
|
||||
|
||||
use super::{Addressbytes, SliceExtended};
|
||||
use super::{Addressbytes, Addresstype, SliceExtended};
|
||||
|
||||
#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Prefix([u8; 8]);
|
||||
direct_repr!(Prefix);
|
||||
impl TryFrom<&[u8]> for Prefix {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
|
||||
Ok(Self(value.read_8xU8()?))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct AddressbytesPrefix(Prefix);
|
||||
pub struct AddressbytesPrefix([u8; 8]);
|
||||
direct_repr!(AddressbytesPrefix);
|
||||
impl TryFrom<&Addressbytes> for AddressbytesPrefix {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from(value: &Addressbytes) -> Result<Self, Self::Error> {
|
||||
Ok(Self(Prefix::try_from(match value {
|
||||
Addressbytes::P2PK65(bytes) => &bytes[..],
|
||||
Addressbytes::P2PK33(bytes) => &bytes[..],
|
||||
Addressbytes::P2PKH(bytes) => &bytes[..],
|
||||
Addressbytes::P2SH(bytes) => &bytes[..],
|
||||
Addressbytes::P2WPKH(bytes) => &bytes[..],
|
||||
Addressbytes::P2WSH(bytes) => &bytes[..],
|
||||
Addressbytes::P2TR(bytes) => &bytes[..],
|
||||
})?))
|
||||
impl From<(&Addressbytes, Addresstype)> for AddressbytesPrefix {
|
||||
fn from((addressbytes, addresstype): (&Addressbytes, Addresstype)) -> Self {
|
||||
let shorten = |slice: &[u8]| {
|
||||
let len = slice.len();
|
||||
let mut buf: [u8; 8] = [0; 8];
|
||||
// Using both ends for collision reasons despite rehashing the addresses
|
||||
(0..4_usize).for_each(|i| {
|
||||
buf[i] = slice[i];
|
||||
buf[4 + i] = slice[len - 4 + i];
|
||||
});
|
||||
buf[4] = addresstype as u8;
|
||||
// Put in the middle and not at the start because either the first or the last byte can be used to split and if the type is used it wouldn't have the 0..256 range
|
||||
// End result:
|
||||
// [ i=0, i=1, i=2, i=3, type as u8, i=len-3, i=len-2, i=len-1 ]
|
||||
buf
|
||||
};
|
||||
Self(shorten(
|
||||
bitcoin_hashes::hash160::Hash::hash(addressbytes.as_slice()).as_byte_array(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct BlockHashPrefix(Prefix);
|
||||
pub struct BlockHashPrefix([u8; 8]);
|
||||
direct_repr!(BlockHashPrefix);
|
||||
impl TryFrom<&BlockHash> for BlockHashPrefix {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from(value: &BlockHash) -> Result<Self, Self::Error> {
|
||||
Ok(Self(Prefix::try_from(&value[..])?))
|
||||
Ok(Self((&value[..]).read_8x_u8()?))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deref, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct TxidPrefix(Prefix);
|
||||
pub struct TxidPrefix([u8; 8]);
|
||||
direct_repr!(TxidPrefix);
|
||||
impl TryFrom<&Txid> for TxidPrefix {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from(value: &Txid) -> Result<Self, Self::Error> {
|
||||
Ok(Self(Prefix::try_from(&value[..])?))
|
||||
Ok(Self((&value[..]).read_8x_u8()?))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ use color_eyre::eyre::eyre;
|
||||
|
||||
#[allow(unused)]
|
||||
pub trait SliceExtended {
|
||||
fn read_8xU8(&self) -> color_eyre::Result<[u8; 8]>;
|
||||
fn read_8x_u8(&self) -> color_eyre::Result<[u8; 8]>;
|
||||
fn read_be_u8(&self) -> color_eyre::Result<u8>;
|
||||
fn read_be_u16(&self) -> color_eyre::Result<u16>;
|
||||
fn read_be_u32(&self) -> color_eyre::Result<u32>;
|
||||
@@ -11,7 +11,7 @@ pub trait SliceExtended {
|
||||
}
|
||||
|
||||
impl SliceExtended for &[u8] {
|
||||
fn read_8xU8(&self) -> color_eyre::Result<[u8; 8]> {
|
||||
fn read_8x_u8(&self) -> color_eyre::Result<[u8; 8]> {
|
||||
let mut buf: [u8; 8] = [0; 8];
|
||||
(&self[..8]).read_exact(&mut buf)?;
|
||||
Ok(buf)
|
||||
|
||||
@@ -1,249 +0,0 @@
|
||||
use std::{fs, io, path::Path};
|
||||
|
||||
use biter::bitcoin::{BlockHash, Txid};
|
||||
use color_eyre::eyre::eyre;
|
||||
use storable_vec::{AnyStorableVec, StorableVec};
|
||||
|
||||
use super::{
|
||||
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, Exit, Height, P2PK33AddressBytes,
|
||||
P2PK65AddressBytes, P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes, P2WSHAddressBytes,
|
||||
Txindex, Txoutindex,
|
||||
};
|
||||
|
||||
pub struct StorableVecs {
|
||||
// TODO:
|
||||
//
|
||||
// Add
|
||||
// txindex_to_fees
|
||||
// height_to_fees
|
||||
// height_to_utc_date
|
||||
// height_to_timestamp
|
||||
//
|
||||
// NOT the following as because of reorg it's subjective
|
||||
// date_to_fees
|
||||
// date_to_first_height
|
||||
// date_to_last_height
|
||||
pub addressindex_to_addresstype: StorableVec<Addressindex, Addresstype>,
|
||||
pub addressindex_to_addresstypeindex: StorableVec<Addressindex, Addresstypeindex>,
|
||||
pub height_to_blockhash: StorableVec<Height, BlockHash>,
|
||||
pub height_to_first_addressindex: StorableVec<Height, Addressindex>,
|
||||
pub height_to_first_txindex: StorableVec<Height, Txindex>,
|
||||
pub height_to_first_txoutindex: StorableVec<Height, Txoutindex>,
|
||||
pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
|
||||
pub height_to_last_txindex: StorableVec<Height, Txindex>,
|
||||
pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
|
||||
pub p2pk65index_to_p2pk65addressbytes: StorableVec<Addresstypeindex, P2PK65AddressBytes>,
|
||||
pub p2pk33index_to_p2pk33addressbytes: StorableVec<Addresstypeindex, P2PK33AddressBytes>,
|
||||
pub p2pkhindex_to_p2pkhaddressbytes: StorableVec<Addresstypeindex, P2PKHAddressBytes>,
|
||||
pub p2shindex_to_p2shaddressbytes: StorableVec<Addresstypeindex, P2SHAddressBytes>,
|
||||
pub p2wpkhindex_to_p2wpkhaddressbytes: StorableVec<Addresstypeindex, P2WPKHAddressBytes>,
|
||||
pub p2wshindex_to_p2wshaddressbytes: StorableVec<Addresstypeindex, P2WSHAddressBytes>,
|
||||
pub p2trindex_to_p2traddressbytes: StorableVec<Addresstypeindex, P2TRAddressBytes>,
|
||||
pub txindex_to_height: StorableVec<Txindex, Height>,
|
||||
pub txindex_to_txid: StorableVec<Txindex, Txid>,
|
||||
pub txoutindex_to_addressindex: StorableVec<Txoutindex, Addressindex>,
|
||||
pub txoutindex_to_amount: StorableVec<Txoutindex, Amount>,
|
||||
}
|
||||
|
||||
// const UNSAFE_BLOCKS: usize = 100;
|
||||
|
||||
impl StorableVecs {
|
||||
pub fn import(path: &Path) -> color_eyre::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
Ok(Self {
|
||||
addressindex_to_addresstype: StorableVec::import(&path.join("addressindex_to_addresstype"))?,
|
||||
addressindex_to_addresstypeindex: StorableVec::import(&path.join("addressindex_to_addresstypeindex"))?,
|
||||
height_to_blockhash: StorableVec::import(&path.join("height_to_blockhash"))?,
|
||||
height_to_first_addressindex: StorableVec::import(&path.join("height_to_first_addressindex"))?,
|
||||
height_to_first_txindex: StorableVec::import(&path.join("height_to_first_txindex"))?,
|
||||
height_to_first_txoutindex: StorableVec::import(&path.join("height_to_first_txoutindex"))?,
|
||||
height_to_last_addressindex: StorableVec::import(&path.join("height_to_last_addressindex"))?,
|
||||
height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"))?,
|
||||
height_to_last_txoutindex: StorableVec::import(&path.join("height_to_last_txoutindex"))?,
|
||||
p2pk65index_to_p2pk65addressbytes: StorableVec::import(&path.join("p2pk65index_to_p2pk65addressbytes"))?,
|
||||
p2pk33index_to_p2pk33addressbytes: StorableVec::import(&path.join("p2pk33index_to_p2pk33addressbytes"))?,
|
||||
p2pkhindex_to_p2pkhaddressbytes: StorableVec::import(&path.join("p2pkhindex_to_p2pkhaddressbytes"))?,
|
||||
p2shindex_to_p2shaddressbytes: StorableVec::import(&path.join("p2shindex_to_p2shaddressbytes"))?,
|
||||
p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::import(&path.join("p2wpkhindex_to_p2wpkhaddressbytes"))?,
|
||||
p2wshindex_to_p2wshaddressbytes: StorableVec::import(&path.join("p2wshindex_to_p2wshaddressbytes"))?,
|
||||
p2trindex_to_p2traddressbytes: StorableVec::import(&path.join("p2trindex_to_p2traddressbytes"))?,
|
||||
txindex_to_height: StorableVec::import(&path.join("txindex_to_height"))?,
|
||||
txindex_to_txid: StorableVec::import(&path.join("txindex_to_txid"))?,
|
||||
txoutindex_to_addressindex: StorableVec::import(&path.join("txoutindex_to_addressindex"))?,
|
||||
txoutindex_to_amount: StorableVec::import(&path.join("txoutindex_to_amount"))?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn addresstype_to_addressbytes(&self, addresstype: Addresstype) -> color_eyre::Result<&dyn AnyStorableVec> {
|
||||
match addresstype {
|
||||
Addresstype::P2PK65 => Ok(&self.p2pk65index_to_p2pk65addressbytes),
|
||||
Addresstype::P2PK33 => Ok(&self.p2pk33index_to_p2pk33addressbytes),
|
||||
Addresstype::P2PKH => Ok(&self.p2pkhindex_to_p2pkhaddressbytes),
|
||||
Addresstype::P2SH => Ok(&self.p2shindex_to_p2shaddressbytes),
|
||||
Addresstype::P2WPKH => Ok(&self.p2wpkhindex_to_p2wpkhaddressbytes),
|
||||
Addresstype::P2WSH => Ok(&self.p2wshindex_to_p2wshaddressbytes),
|
||||
Addresstype::P2TR => Ok(&self.p2trindex_to_p2traddressbytes),
|
||||
_ => Err(eyre!("wrong address type")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn push_addressbytes_if_needed(
|
||||
&mut self,
|
||||
index: Addresstypeindex,
|
||||
addressbytes: Addressbytes,
|
||||
) -> color_eyre::Result<()> {
|
||||
match addressbytes {
|
||||
Addressbytes::P2PK65(bytes) => self.p2pk65index_to_p2pk65addressbytes.push_if_needed(index, bytes),
|
||||
Addressbytes::P2PK33(bytes) => self.p2pk33index_to_p2pk33addressbytes.push_if_needed(index, bytes),
|
||||
Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index, bytes),
|
||||
Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index, bytes),
|
||||
Addressbytes::P2WPKH(bytes) => self.p2wpkhindex_to_p2wpkhaddressbytes.push_if_needed(index, bytes),
|
||||
Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index, bytes),
|
||||
Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index, bytes),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rollback_from(&mut self, _height: Height, _exit: &Exit) -> color_eyre::Result<()> {
|
||||
panic!();
|
||||
// let mut txindex = None;
|
||||
|
||||
// wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
|
||||
// .try_for_each(|slice| -> color_eyre::Result<()> {
|
||||
// let (height_slice, slice_blockhash) = slice?;
|
||||
// let blockhash = BlockHash::from_slice(&slice_blockhash)?;
|
||||
|
||||
// wtx.remove(self.height_to_blockhash.data(), height_slice);
|
||||
|
||||
// wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
|
||||
|
||||
// if txindex.is_none() {
|
||||
// txindex.replace(
|
||||
// wtx.get(self.height_to_first_txindex.data(), height_slice)?
|
||||
// .context("for height to have first txindex")?,
|
||||
// );
|
||||
// }
|
||||
// wtx.remove(self.height_to_first_txindex.data(), height_slice);
|
||||
// wtx.remove(self.height_to_last_txindex.data(), height_slice);
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
|
||||
// let txindex = txindex.context("txindex to not be none by now")?;
|
||||
|
||||
// wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
|
||||
// .try_for_each(|slice| -> color_eyre::Result<()> {
|
||||
// let (slice_txindex, slice_txid) = slice?;
|
||||
// let txindex = Txindex::from(slice_txindex);
|
||||
// let txid = Txid::from_slice(&slice_txid)?;
|
||||
|
||||
// wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
|
||||
// wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
|
||||
// wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
|
||||
// let txoutindex = Txoutindex::from(txindex);
|
||||
|
||||
// let mut addressindexes = BTreeSet::new();
|
||||
|
||||
// wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
|
||||
// .try_for_each(|slice| -> color_eyre::Result<()> {
|
||||
// let (txoutindex_slice, _) = slice?;
|
||||
|
||||
// wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
|
||||
|
||||
// if let Some(addressindex_slice) =
|
||||
// wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
|
||||
// {
|
||||
// wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
|
||||
|
||||
// let addressindex = Addressindex::from(addressindex_slice);
|
||||
// addressindexes.insert(addressindex);
|
||||
|
||||
// let txoutindex = Txoutindex::from(txoutindex_slice);
|
||||
// let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
|
||||
|
||||
// wtx.remove(
|
||||
// self.addressindex_to_txoutindexes.data(),
|
||||
// Slice::from(addresstxoutindex),
|
||||
// );
|
||||
// }
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
|
||||
// addressindexes
|
||||
// .into_iter()
|
||||
// .filter(|addressindex| {
|
||||
// let is_empty = wtx
|
||||
// .prefix(
|
||||
// self.addressindex_to_txoutindexes.data(),
|
||||
// Slice::from(*addressindex),
|
||||
// )
|
||||
// .next()
|
||||
// .is_none();
|
||||
// is_empty
|
||||
// })
|
||||
// .try_for_each(|addressindex| -> color_eyre::Result<()> {
|
||||
// let addressindex_slice = Slice::from(addressindex);
|
||||
|
||||
// let addressbytes = Addressbytes::from(
|
||||
// wtx.get(
|
||||
// self.addressindex_to_addressbytes.data(),
|
||||
// &addressindex_slice,
|
||||
// )?
|
||||
// .context("addressindex_to_address to have value")?,
|
||||
// );
|
||||
// wtx.remove(
|
||||
// self.addressbytes_prefix_to_addressindex.data(),
|
||||
// addressbytes.prefix(),
|
||||
// );
|
||||
// wtx.remove(
|
||||
// self.addressindex_to_addressbytes.data(),
|
||||
// &addressindex_slice,
|
||||
// );
|
||||
// wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
//
|
||||
|
||||
// todo!("clear addresstxoutindexes_out")
|
||||
// todo!("clear addresstxoutindexes_in")
|
||||
// todo!("clear zero_txoutindexes")
|
||||
|
||||
// Ok(())
|
||||
}
|
||||
|
||||
pub fn flush(&mut self) -> io::Result<()> {
|
||||
self.as_mut_vec().into_iter().try_for_each(AnyStorableVec::flush)
|
||||
}
|
||||
|
||||
fn as_mut_vec(&mut self) -> Vec<&mut dyn AnyStorableVec> {
|
||||
vec![
|
||||
&mut self.addressindex_to_addresstype,
|
||||
&mut self.addressindex_to_addresstypeindex,
|
||||
&mut self.height_to_blockhash,
|
||||
&mut self.height_to_first_addressindex,
|
||||
&mut self.height_to_first_txindex,
|
||||
&mut self.height_to_first_txoutindex,
|
||||
&mut self.height_to_last_addressindex,
|
||||
&mut self.height_to_last_txindex,
|
||||
&mut self.height_to_last_txoutindex,
|
||||
&mut self.p2pk65index_to_p2pk65addressbytes,
|
||||
&mut self.p2pk33index_to_p2pk33addressbytes,
|
||||
&mut self.p2pkhindex_to_p2pkhaddressbytes,
|
||||
&mut self.p2shindex_to_p2shaddressbytes,
|
||||
&mut self.p2wpkhindex_to_p2wpkhaddressbytes,
|
||||
&mut self.p2wshindex_to_p2wshaddressbytes,
|
||||
&mut self.p2trindex_to_p2traddressbytes,
|
||||
&mut self.txindex_to_height,
|
||||
&mut self.txindex_to_txid,
|
||||
&mut self.txoutindex_to_addressindex,
|
||||
&mut self.txoutindex_to_amount,
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
use std::{
|
||||
array, fs,
|
||||
path::{Path, PathBuf},
|
||||
sync::OnceLock,
|
||||
};
|
||||
|
||||
use rayon::prelude::*;
|
||||
use snkrj::{Database, DatabaseKey, DatabaseValue, UnitDatabase};
|
||||
use storable_vec::UnsafeSizedSerDe;
|
||||
|
||||
use super::{Height, Version};
|
||||
|
||||
pub struct Store<K, V>
|
||||
where
|
||||
K: DatabaseKey,
|
||||
V: DatabaseValue,
|
||||
{
|
||||
pathbuf: PathBuf,
|
||||
version: Version,
|
||||
height: Option<Height>,
|
||||
len: usize,
|
||||
pub parts: [OnceLock<Box<Database<K, V>>>; 256],
|
||||
}
|
||||
|
||||
impl<K, V> Store<K, V>
|
||||
where
|
||||
K: DatabaseKey,
|
||||
V: DatabaseValue,
|
||||
{
|
||||
pub fn open(path: &Path, version: Version) -> Result<Self, snkrj::Error> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
let is_same_version =
|
||||
Version::try_from(Self::path_version_(path).as_path()).is_ok_and(|prev_version| version == prev_version);
|
||||
|
||||
if !is_same_version {
|
||||
fs::remove_dir(path)?;
|
||||
fs::create_dir_all(path)?;
|
||||
}
|
||||
|
||||
let height = Height::try_from(Self::path_height_(path).as_path()).ok();
|
||||
|
||||
Ok(Self {
|
||||
pathbuf: path.to_owned(),
|
||||
version,
|
||||
height,
|
||||
len: UnitDatabase::read_length_(path),
|
||||
parts: array::from_fn(|_| OnceLock::new()),
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn len(&self) -> usize {
|
||||
self.len
|
||||
}
|
||||
|
||||
fn key_to_byte(key: &K) -> u8 {
|
||||
let slice = key.unsafe_as_slice();
|
||||
|
||||
*(if cfg!(target_endian = "big") {
|
||||
slice.last()
|
||||
} else {
|
||||
slice.first()
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn get_or_init_store(&self, key: &K) -> &Database<K, V> {
|
||||
self.get_or_init_store_(Self::key_to_byte(key) as usize)
|
||||
}
|
||||
|
||||
fn get_or_init_store_(&self, storeindex: usize) -> &Database<K, V> {
|
||||
self.parts[storeindex]
|
||||
.get_or_init(|| Box::new(Database::open(self.path_parts().join(storeindex.to_string())).unwrap()))
|
||||
}
|
||||
|
||||
fn get_or_init_mut_store(&mut self, key: &K) -> &mut Database<K, V> {
|
||||
self.get_or_init_store(key);
|
||||
|
||||
self.parts
|
||||
.get_mut(Self::key_to_byte(key) as usize)
|
||||
.unwrap()
|
||||
.get_mut()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn open_all(&self) {
|
||||
(0..=(u8::MAX) as usize).for_each(|storeindex| {
|
||||
self.get_or_init_store_(storeindex);
|
||||
});
|
||||
}
|
||||
|
||||
pub fn get(&self, key: &K) -> Option<&V> {
|
||||
self.get_or_init_store(key).get(key)
|
||||
}
|
||||
|
||||
pub fn insert(&mut self, key: K, value: V) -> Option<V> {
|
||||
self.len += 1;
|
||||
self.get_or_init_mut_store(&key).insert(key, value)
|
||||
}
|
||||
|
||||
pub fn insert_if_needed(&mut self, key: K, value: V, height: Height) {
|
||||
if self.needs(height) {
|
||||
self.insert(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn export(mut self, height: Height) -> Result<(), snkrj::Error> {
|
||||
if self.height.is_some_and(|self_height| self_height >= height) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.height = Some(height);
|
||||
self.version.write(&self.path_version())?;
|
||||
height.write(&self.path_height())?;
|
||||
UnitDatabase::write_length_(&self.pathbuf, self.len)?;
|
||||
self.parts.into_par_iter().try_for_each(|s| {
|
||||
if let Some(db) = s.into_inner() {
|
||||
db.export()
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn path_parts(&self) -> PathBuf {
|
||||
Self::path_parts_(&self.pathbuf)
|
||||
}
|
||||
fn path_parts_(path: &Path) -> PathBuf {
|
||||
path.join("parts")
|
||||
}
|
||||
|
||||
fn path_version(&self) -> PathBuf {
|
||||
Self::path_version_(&self.pathbuf)
|
||||
}
|
||||
fn path_version_(path: &Path) -> PathBuf {
|
||||
path.join("version")
|
||||
}
|
||||
|
||||
pub fn height(&self) -> Option<&Height> {
|
||||
self.height.as_ref()
|
||||
}
|
||||
pub fn needs(&self, height: Height) -> bool {
|
||||
self.height.is_none_or(|self_height| height > self_height)
|
||||
}
|
||||
fn path_height(&self) -> PathBuf {
|
||||
Self::path_height_(&self.pathbuf)
|
||||
}
|
||||
fn path_height_(path: &Path) -> PathBuf {
|
||||
path.join("height")
|
||||
}
|
||||
}
|
||||
+38
-46
@@ -1,34 +1,33 @@
|
||||
use std::{path::Path, thread};
|
||||
|
||||
use snkrj::{AnyDatabase, Database};
|
||||
|
||||
use crate::structs::Height;
|
||||
|
||||
use super::{
|
||||
AddressbytesPrefix, Addressindex, Addressindextxoutindex, BlockHashPrefix, TxidPrefix, Txindex, Txindexvout,
|
||||
Txoutindex,
|
||||
AddressbytesPrefix, Addressindex, Addressindextxoutindex, BlockHashPrefix, Store, TxidPrefix, Txindex, Txindexvout,
|
||||
Txoutindex, Version,
|
||||
};
|
||||
|
||||
pub struct Databases {
|
||||
pub addressbytes_prefix_to_addressindex: Database<AddressbytesPrefix, Addressindex>,
|
||||
pub addressindextxoutindex_in: Database<Addressindextxoutindex, ()>,
|
||||
pub addressindextxoutindex_out: Database<Addressindextxoutindex, ()>,
|
||||
pub blockhash_prefix_to_height: Database<BlockHashPrefix, Height>,
|
||||
pub txid_prefix_to_txindex: Database<TxidPrefix, Txindex>,
|
||||
pub txindexvout_to_txoutindex: Database<Txindexvout, Txoutindex>,
|
||||
pub struct Stores {
|
||||
pub addressbytes_prefix_to_addressindex: Store<AddressbytesPrefix, Addressindex>,
|
||||
pub addressindextxoutindex_in: Store<Addressindextxoutindex, ()>,
|
||||
pub addressindextxoutindex_out: Store<Addressindextxoutindex, ()>,
|
||||
pub blockhash_prefix_to_height: Store<BlockHashPrefix, Height>,
|
||||
pub txid_prefix_to_txindex: Store<TxidPrefix, Txindex>,
|
||||
pub txindexvout_to_txoutindex: Store<Txindexvout, Txoutindex>,
|
||||
}
|
||||
|
||||
const UNSAFE_BLOCKS: usize = 100;
|
||||
|
||||
impl Databases {
|
||||
impl Stores {
|
||||
pub fn open(path: &Path) -> color_eyre::Result<Self> {
|
||||
Ok(Self {
|
||||
addressbytes_prefix_to_addressindex: Database::open(path.join("addressbytes_prefix_to_addressindex"))?,
|
||||
addressindextxoutindex_in: Database::open(path.join("addresstxoutindexes_in"))?,
|
||||
addressindextxoutindex_out: Database::open(path.join("addresstxoutindexes_out"))?,
|
||||
blockhash_prefix_to_height: Database::open(path.join("blockhash_prefix_to_height"))?,
|
||||
txid_prefix_to_txindex: Database::open(path.join("txid_prefix_to_txindex"))?,
|
||||
txindexvout_to_txoutindex: Database::open(path.join("txindexvout_to_txoutindex"))?,
|
||||
addressbytes_prefix_to_addressindex: Store::open(
|
||||
&path.join("addressbytes_prefix_to_addressindex"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
addressindextxoutindex_in: Store::open(&path.join("addresstxoutindexes_in"), Version::from(1))?,
|
||||
addressindextxoutindex_out: Store::open(&path.join("addresstxoutindexes_out"), Version::from(1))?,
|
||||
blockhash_prefix_to_height: Store::open(&path.join("blockhash_prefix_to_height"), Version::from(1))?,
|
||||
txid_prefix_to_txindex: Store::open(&path.join("txid_prefix_to_txindex"), Version::from(1))?,
|
||||
txindexvout_to_txoutindex: Store::open(&path.join("txindexvout_to_txoutindex"), Version::from(1))?,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -151,36 +150,29 @@ impl Databases {
|
||||
// Ok(())
|
||||
// }
|
||||
|
||||
fn to_ref_vec(&self) -> Vec<&dyn AnyDatabase> {
|
||||
vec![
|
||||
&self.addressbytes_prefix_to_addressindex as &dyn AnyDatabase,
|
||||
&self.addressindextxoutindex_in,
|
||||
&self.addressindextxoutindex_out,
|
||||
&self.blockhash_prefix_to_height,
|
||||
&self.txid_prefix_to_txindex,
|
||||
&self.txindexvout_to_txoutindex,
|
||||
pub fn min_height(&self) -> Option<Height> {
|
||||
[
|
||||
self.addressbytes_prefix_to_addressindex.height(),
|
||||
self.addressindextxoutindex_in.height(),
|
||||
self.addressindextxoutindex_out.height(),
|
||||
self.blockhash_prefix_to_height.height(),
|
||||
self.txid_prefix_to_txindex.height(),
|
||||
self.txindexvout_to_txoutindex.height(),
|
||||
]
|
||||
.into_iter()
|
||||
.min()
|
||||
.flatten()
|
||||
.cloned()
|
||||
}
|
||||
|
||||
fn to_ref_mut_vec(&mut self) -> Vec<&mut dyn AnyDatabase> {
|
||||
vec![
|
||||
&mut self.addressbytes_prefix_to_addressindex as &mut dyn AnyDatabase,
|
||||
&mut self.addressindextxoutindex_in,
|
||||
&mut self.addressindextxoutindex_out,
|
||||
&mut self.blockhash_prefix_to_height,
|
||||
&mut self.txid_prefix_to_txindex,
|
||||
&mut self.txindexvout_to_txoutindex,
|
||||
]
|
||||
}
|
||||
|
||||
pub fn export(self) {
|
||||
pub fn export(self, height: Height) {
|
||||
thread::scope(|scope| {
|
||||
scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(false));
|
||||
scope.spawn(|| self.addressindextxoutindex_in.export(false));
|
||||
scope.spawn(|| self.addressindextxoutindex_out.export(false));
|
||||
scope.spawn(|| self.blockhash_prefix_to_height.export(false));
|
||||
scope.spawn(|| self.txid_prefix_to_txindex.export(false));
|
||||
scope.spawn(|| self.txindexvout_to_txoutindex.export(false));
|
||||
scope.spawn(|| self.addressbytes_prefix_to_addressindex.export(height));
|
||||
scope.spawn(|| self.addressindextxoutindex_in.export(height));
|
||||
scope.spawn(|| self.addressindextxoutindex_out.export(height));
|
||||
scope.spawn(|| self.blockhash_prefix_to_height.export(height));
|
||||
scope.spawn(|| self.txid_prefix_to_txindex.export(height));
|
||||
scope.spawn(|| self.txindexvout_to_txoutindex.export(height));
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
use derive_deref::Deref;
|
||||
|
||||
#[derive(Debug, Deref, Clone)]
|
||||
pub struct Timestamp(jiff::Timestamp);
|
||||
|
||||
impl TryFrom<u32> for Timestamp {
|
||||
type Error = jiff::Error;
|
||||
fn try_from(value: u32) -> Result<Self, Self::Error> {
|
||||
Ok(Self(jiff::Timestamp::from_second(value as i64)?))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
fs, io,
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use super::{Height, Version};
|
||||
|
||||
pub struct StorableVec<I, T> {
|
||||
pathbuf: PathBuf,
|
||||
version: Version,
|
||||
vec: storable_vec::StorableVec<I, T>,
|
||||
}
|
||||
|
||||
impl<I, T> StorableVec<I, T>
|
||||
where
|
||||
I: Into<usize>,
|
||||
T: Sized + Debug,
|
||||
{
|
||||
pub fn import(path: &Path, version: Version) -> io::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
let pathbuf = path.to_owned();
|
||||
let path_vec = Self::_path_vec(path);
|
||||
let path_version = Self::_path_version(path);
|
||||
|
||||
let is_same_version =
|
||||
Version::try_from(path_version.as_path()).is_ok_and(|prev_version| version == prev_version);
|
||||
if !is_same_version {
|
||||
let _ = fs::remove_file(&path_vec);
|
||||
let _ = fs::remove_file(&path_version);
|
||||
let _ = fs::remove_file(Self::_path_height(path));
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
pathbuf,
|
||||
version,
|
||||
vec: storable_vec::StorableVec::import(&path_vec)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn flush(&mut self, height: Height) -> io::Result<()> {
|
||||
height.write(&self.path_height())?;
|
||||
self.version.write(&self.path_version())?;
|
||||
self.vec.flush()
|
||||
}
|
||||
|
||||
// fn path_vec(&self) -> PathBuf {
|
||||
// Self::_path_vec(&self.path)
|
||||
// }
|
||||
fn _path_vec(path: &Path) -> PathBuf {
|
||||
path.join("vec")
|
||||
}
|
||||
|
||||
fn path_version(&self) -> PathBuf {
|
||||
Self::_path_version(&self.pathbuf)
|
||||
}
|
||||
fn _path_version(path: &Path) -> PathBuf {
|
||||
path.join("version")
|
||||
}
|
||||
|
||||
pub fn height(&self) -> color_eyre::Result<Height> {
|
||||
Height::try_from(self.path_height().as_path())
|
||||
}
|
||||
fn path_height(&self) -> PathBuf {
|
||||
Self::_path_height(&self.pathbuf)
|
||||
}
|
||||
fn _path_height(path: &Path) -> PathBuf {
|
||||
path.join("height")
|
||||
}
|
||||
|
||||
fn reset_cache(&mut self) {
|
||||
self.vec.reset_cache();
|
||||
}
|
||||
|
||||
// pub fn needs(&self, height: Height) -> bool {
|
||||
// self.height() // store height in struct
|
||||
// }
|
||||
}
|
||||
|
||||
impl<I, T> Deref for StorableVec<I, T> {
|
||||
type Target = storable_vec::StorableVec<I, T>;
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.vec
|
||||
}
|
||||
}
|
||||
impl<I, T> DerefMut for StorableVec<I, T> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.vec
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AnyBindexVec {
|
||||
fn height(&self) -> color_eyre::Result<Height>;
|
||||
fn reset_cache(&mut self);
|
||||
fn flush(&mut self, height: Height) -> io::Result<()>;
|
||||
}
|
||||
|
||||
impl<I, T> AnyBindexVec for StorableVec<I, T>
|
||||
where
|
||||
I: Into<usize>,
|
||||
T: Sized + Debug,
|
||||
{
|
||||
fn height(&self) -> color_eyre::Result<Height> {
|
||||
self.height()
|
||||
}
|
||||
|
||||
fn reset_cache(&mut self) {
|
||||
self.reset_cache();
|
||||
}
|
||||
|
||||
fn flush(&mut self, height: Height) -> io::Result<()> {
|
||||
self.flush(height)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
use std::{fs, io, path::Path};
|
||||
|
||||
use biter::bitcoin::{transaction, BlockHash, Txid};
|
||||
use color_eyre::eyre::eyre;
|
||||
use rayon::prelude::*;
|
||||
use storable_vec::AnyStorableVec;
|
||||
|
||||
use super::{
|
||||
Addressbytes, Addressindex, Addresstype, Addresstypeindex, Amount, AnyBindexVec, Date, Exit, Height,
|
||||
P2PK33AddressBytes, P2PK65AddressBytes, P2PKHAddressBytes, P2SHAddressBytes, P2TRAddressBytes, P2WPKHAddressBytes,
|
||||
P2WSHAddressBytes, StorableVec, Timestamp, Txindex, Txoutindex, Version,
|
||||
};
|
||||
|
||||
pub struct Vecs {
|
||||
pub addressindex_to_addresstype: StorableVec<Addressindex, Addresstype>,
|
||||
pub addressindex_to_addresstypeindex: StorableVec<Addressindex, Addresstypeindex>,
|
||||
pub height_to_blockhash: StorableVec<Height, BlockHash>,
|
||||
pub height_to_date: StorableVec<Height, Date>,
|
||||
pub height_to_totalfees: StorableVec<Height, Amount>,
|
||||
pub height_to_first_addressindex: StorableVec<Height, Addressindex>,
|
||||
pub height_to_first_txindex: StorableVec<Height, Txindex>,
|
||||
pub height_to_first_txoutindex: StorableVec<Height, Txoutindex>,
|
||||
pub height_to_inputcount: StorableVec<Txindex, u32>,
|
||||
pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
|
||||
pub height_to_last_txindex: StorableVec<Height, Txindex>,
|
||||
pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
|
||||
pub height_to_outputcount: StorableVec<Txindex, u32>,
|
||||
pub height_to_timestamp: StorableVec<Height, Timestamp>,
|
||||
pub height_to_txcount: StorableVec<Txindex, u32>,
|
||||
// pub height_to_size: StorableVec<Txindex, u32>,
|
||||
// pub height_to_weight: StorableVec<Txindex, u32>,
|
||||
// pub height_to_subsidy: StorableVec<Txindex, u32>,
|
||||
// pub height_to_minfeerate: StorableVec<Txindex, u32>,
|
||||
// pub height_to_maxfeerate: StorableVec<Txindex, u32>,
|
||||
// pub height_to_medianfeerate: StorableVec<Txindex, u32>,
|
||||
pub p2pk33index_to_p2pk33addressbytes: StorableVec<Addresstypeindex, P2PK33AddressBytes>,
|
||||
pub p2pk65index_to_p2pk65addressbytes: StorableVec<Addresstypeindex, P2PK65AddressBytes>,
|
||||
pub p2pkhindex_to_p2pkhaddressbytes: StorableVec<Addresstypeindex, P2PKHAddressBytes>,
|
||||
pub p2shindex_to_p2shaddressbytes: StorableVec<Addresstypeindex, P2SHAddressBytes>,
|
||||
pub p2trindex_to_p2traddressbytes: StorableVec<Addresstypeindex, P2TRAddressBytes>,
|
||||
pub p2wpkhindex_to_p2wpkhaddressbytes: StorableVec<Addresstypeindex, P2WPKHAddressBytes>,
|
||||
pub p2wshindex_to_p2wshaddressbytes: StorableVec<Addresstypeindex, P2WSHAddressBytes>,
|
||||
pub txindex_to_fee: StorableVec<Txindex, Amount>,
|
||||
// pub txindex_to_feerate: StorableVec<Txindex, Feerate>,
|
||||
pub txindex_to_height: StorableVec<Txindex, Height>,
|
||||
pub txindex_to_inputcount: StorableVec<Txindex, u32>,
|
||||
pub txindex_to_outputcount: StorableVec<Txindex, u32>,
|
||||
pub txindex_to_txid: StorableVec<Txindex, Txid>,
|
||||
pub txindex_to_txversion: StorableVec<Txindex, transaction::Version>,
|
||||
pub txoutindex_to_addressindex: StorableVec<Txoutindex, Addressindex>,
|
||||
pub txoutindex_to_amount: StorableVec<Txoutindex, Amount>,
|
||||
}
|
||||
|
||||
// const UNSAFE_BLOCKS: usize = 100;
|
||||
|
||||
impl Vecs {
|
||||
pub fn import(path: &Path) -> color_eyre::Result<Self> {
|
||||
fs::create_dir_all(path)?;
|
||||
|
||||
Ok(Self {
|
||||
addressindex_to_addresstype: StorableVec::import(
|
||||
&path.join("addressindex_to_addresstype"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
addressindex_to_addresstypeindex: StorableVec::import(
|
||||
&path.join("addressindex_to_addresstypeindex"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
height_to_blockhash: StorableVec::import(&path.join("height_to_blockhash"), Version::from(1))?,
|
||||
height_to_date: StorableVec::import(&path.join("height_to_date"), Version::from(1))?,
|
||||
height_to_first_addressindex: StorableVec::import(
|
||||
&path.join("height_to_first_addressindex"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
height_to_first_txindex: StorableVec::import(&path.join("height_to_first_txindex"), Version::from(1))?,
|
||||
height_to_first_txoutindex: StorableVec::import(
|
||||
&path.join("height_to_first_txoutindex"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
height_to_inputcount: StorableVec::import(&path.join("height_to_inputcount"), Version::from(1))?,
|
||||
height_to_last_addressindex: StorableVec::import(
|
||||
&path.join("height_to_last_addressindex"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
height_to_last_txindex: StorableVec::import(&path.join("height_to_last_txindex"), Version::from(1))?,
|
||||
height_to_last_txoutindex: StorableVec::import(&path.join("height_to_last_txoutindex"), Version::from(1))?,
|
||||
height_to_outputcount: StorableVec::import(&path.join("height_to_outputcount"), Version::from(1))?,
|
||||
height_to_timestamp: StorableVec::import(&path.join("height_to_timestamp"), Version::from(1))?,
|
||||
height_to_totalfees: StorableVec::import(&path.join("height_to_totalfees"), Version::from(1))?,
|
||||
height_to_txcount: StorableVec::import(&path.join("height_to_txcount"), Version::from(1))?,
|
||||
p2pk33index_to_p2pk33addressbytes: StorableVec::import(
|
||||
&path.join("p2pk33index_to_p2pk33addressbytes"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
p2pk65index_to_p2pk65addressbytes: StorableVec::import(
|
||||
&path.join("p2pk65index_to_p2pk65addressbytes"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
p2pkhindex_to_p2pkhaddressbytes: StorableVec::import(
|
||||
&path.join("p2pkhindex_to_p2pkhaddressbytes"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
p2shindex_to_p2shaddressbytes: StorableVec::import(
|
||||
&path.join("p2shindex_to_p2shaddressbytes"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
p2trindex_to_p2traddressbytes: StorableVec::import(
|
||||
&path.join("p2trindex_to_p2traddressbytes"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::import(
|
||||
&path.join("p2wpkhindex_to_p2wpkhaddressbytes"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
p2wshindex_to_p2wshaddressbytes: StorableVec::import(
|
||||
&path.join("p2wshindex_to_p2wshaddressbytes"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
txindex_to_fee: StorableVec::import(&path.join("txindex_to_fee"), Version::from(1))?,
|
||||
txindex_to_height: StorableVec::import(&path.join("txindex_to_height"), Version::from(1))?,
|
||||
txindex_to_inputcount: StorableVec::import(&path.join("txindex_to_inputcount"), Version::from(1))?,
|
||||
txindex_to_outputcount: StorableVec::import(&path.join("txindex_to_outputcount"), Version::from(1))?,
|
||||
txindex_to_txid: StorableVec::import(&path.join("txindex_to_txid"), Version::from(1))?,
|
||||
txindex_to_txversion: StorableVec::import(&path.join("txindex_to_txversion"), Version::from(1))?,
|
||||
txoutindex_to_addressindex: StorableVec::import(
|
||||
&path.join("txoutindex_to_addressindex"),
|
||||
Version::from(1),
|
||||
)?,
|
||||
txoutindex_to_amount: StorableVec::import(&path.join("txoutindex_to_amount"), Version::from(1))?,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn addresstype_to_addressbytes(&self, addresstype: Addresstype) -> color_eyre::Result<&dyn AnyStorableVec> {
|
||||
match addresstype {
|
||||
Addresstype::P2PK65 => Ok(&*self.p2pk65index_to_p2pk65addressbytes),
|
||||
Addresstype::P2PK33 => Ok(&*self.p2pk33index_to_p2pk33addressbytes),
|
||||
Addresstype::P2PKH => Ok(&*self.p2pkhindex_to_p2pkhaddressbytes),
|
||||
Addresstype::P2SH => Ok(&*self.p2shindex_to_p2shaddressbytes),
|
||||
Addresstype::P2WPKH => Ok(&*self.p2wpkhindex_to_p2wpkhaddressbytes),
|
||||
Addresstype::P2WSH => Ok(&*self.p2wshindex_to_p2wshaddressbytes),
|
||||
Addresstype::P2TR => Ok(&*self.p2trindex_to_p2traddressbytes),
|
||||
_ => Err(eyre!("wrong address type")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn push_addressbytes_if_needed(
|
||||
&mut self,
|
||||
index: Addresstypeindex,
|
||||
addressbytes: Addressbytes,
|
||||
) -> storable_vec::Result<()> {
|
||||
match addressbytes {
|
||||
Addressbytes::P2PK65(bytes) => self.p2pk65index_to_p2pk65addressbytes.push_if_needed(index, bytes),
|
||||
Addressbytes::P2PK33(bytes) => self.p2pk33index_to_p2pk33addressbytes.push_if_needed(index, bytes),
|
||||
Addressbytes::P2PKH(bytes) => self.p2pkhindex_to_p2pkhaddressbytes.push_if_needed(index, bytes),
|
||||
Addressbytes::P2SH(bytes) => self.p2shindex_to_p2shaddressbytes.push_if_needed(index, bytes),
|
||||
Addressbytes::P2WPKH(bytes) => self.p2wpkhindex_to_p2wpkhaddressbytes.push_if_needed(index, bytes),
|
||||
Addressbytes::P2WSH(bytes) => self.p2wshindex_to_p2wshaddressbytes.push_if_needed(index, bytes),
|
||||
Addressbytes::P2TR(bytes) => self.p2trindex_to_p2traddressbytes.push_if_needed(index, bytes),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_addressbytes(
|
||||
&self,
|
||||
addresstype: Addresstype,
|
||||
addresstypeindex: Addresstypeindex,
|
||||
) -> storable_vec::Result<Option<Addressbytes>> {
|
||||
Ok(match addresstype {
|
||||
Addresstype::P2PK65 => self
|
||||
.p2pk65index_to_p2pk65addressbytes
|
||||
.get(addresstypeindex)?
|
||||
.cloned()
|
||||
.map(Addressbytes::from),
|
||||
Addresstype::P2PK33 => self
|
||||
.p2pk33index_to_p2pk33addressbytes
|
||||
.get(addresstypeindex)?
|
||||
.cloned()
|
||||
.map(Addressbytes::from),
|
||||
Addresstype::P2PKH => self
|
||||
.p2pkhindex_to_p2pkhaddressbytes
|
||||
.get(addresstypeindex)?
|
||||
.cloned()
|
||||
.map(Addressbytes::from),
|
||||
Addresstype::P2SH => self
|
||||
.p2shindex_to_p2shaddressbytes
|
||||
.get(addresstypeindex)?
|
||||
.cloned()
|
||||
.map(Addressbytes::from),
|
||||
Addresstype::P2WPKH => self
|
||||
.p2wpkhindex_to_p2wpkhaddressbytes
|
||||
.get(addresstypeindex)?
|
||||
.cloned()
|
||||
.map(Addressbytes::from),
|
||||
Addresstype::P2WSH => self
|
||||
.p2wshindex_to_p2wshaddressbytes
|
||||
.get(addresstypeindex)?
|
||||
.cloned()
|
||||
.map(Addressbytes::from),
|
||||
Addresstype::P2TR => self
|
||||
.p2trindex_to_p2traddressbytes
|
||||
.get(addresstypeindex)?
|
||||
.cloned()
|
||||
.map(Addressbytes::from),
|
||||
_ => unreachable!(),
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn rollback_from(&mut self, _height: Height, _exit: &Exit) -> color_eyre::Result<()> {
|
||||
panic!();
|
||||
// let mut txindex = None;
|
||||
|
||||
// wtx.range(self.height_to_blockhash.data(), Slice::from(height)..)
|
||||
// .try_for_each(|slice| -> color_eyre::Result<()> {
|
||||
// let (height_slice, slice_blockhash) = slice?;
|
||||
// let blockhash = BlockHash::from_slice(&slice_blockhash)?;
|
||||
|
||||
// wtx.remove(self.height_to_blockhash.data(), height_slice);
|
||||
|
||||
// wtx.remove(self.blockhash_prefix_to_height.data(), blockhash.prefix());
|
||||
|
||||
// if txindex.is_none() {
|
||||
// txindex.replace(
|
||||
// wtx.get(self.height_to_first_txindex.data(), height_slice)?
|
||||
// .context("for height to have first txindex")?,
|
||||
// );
|
||||
// }
|
||||
// wtx.remove(self.height_to_first_txindex.data(), height_slice);
|
||||
// wtx.remove(self.height_to_last_txindex.data(), height_slice);
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
|
||||
// let txindex = txindex.context("txindex to not be none by now")?;
|
||||
|
||||
// wtx.range(self.txindex_to_txid.data(), Slice::from(txindex)..)
|
||||
// .try_for_each(|slice| -> color_eyre::Result<()> {
|
||||
// let (slice_txindex, slice_txid) = slice?;
|
||||
// let txindex = Txindex::from(slice_txindex);
|
||||
// let txid = Txid::from_slice(&slice_txid)?;
|
||||
|
||||
// wtx.remove(self.txindex_to_txid.data(), Slice::from(txindex));
|
||||
// wtx.remove(self.txindex_to_height.data(), Slice::from(txindex));
|
||||
// wtx.remove(self.txid_prefix_to_txindex.data(), txid.prefix());
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
|
||||
// let txoutindex = Txoutindex::from(txindex);
|
||||
|
||||
// let mut addressindexes = BTreeSet::new();
|
||||
|
||||
// wtx.range(self.txoutindex_to_amount.data(), Slice::from(txoutindex)..)
|
||||
// .try_for_each(|slice| -> color_eyre::Result<()> {
|
||||
// let (txoutindex_slice, _) = slice?;
|
||||
|
||||
// wtx.remove(self.txoutindex_to_amount.data(), txoutindex_slice);
|
||||
|
||||
// if let Some(addressindex_slice) =
|
||||
// wtx.get(self.txoutindex_to_addressindex.data(), txoutindex_slice)?
|
||||
// {
|
||||
// wtx.remove(self.txoutindex_to_addressindex.data(), txoutindex_slice);
|
||||
|
||||
// let addressindex = Addressindex::from(addressindex_slice);
|
||||
// addressindexes.insert(addressindex);
|
||||
|
||||
// let txoutindex = Txoutindex::from(txoutindex_slice);
|
||||
// let addresstxoutindex = Addresstxoutindex::from((addressindex, txoutindex));
|
||||
|
||||
// wtx.remove(
|
||||
// self.addressindex_to_txoutindexes.data(),
|
||||
// Slice::from(addresstxoutindex),
|
||||
// );
|
||||
// }
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
|
||||
// addressindexes
|
||||
// .into_iter()
|
||||
// .filter(|addressindex| {
|
||||
// let is_empty = wtx
|
||||
// .prefix(
|
||||
// self.addressindex_to_txoutindexes.data(),
|
||||
// Slice::from(*addressindex),
|
||||
// )
|
||||
// .next()
|
||||
// .is_none();
|
||||
// is_empty
|
||||
// })
|
||||
// .try_for_each(|addressindex| -> color_eyre::Result<()> {
|
||||
// let addressindex_slice = Slice::from(addressindex);
|
||||
|
||||
// let addressbytes = Addressbytes::from(
|
||||
// wtx.get(
|
||||
// self.addressindex_to_addressbytes.data(),
|
||||
// &addressindex_slice,
|
||||
// )?
|
||||
// .context("addressindex_to_address to have value")?,
|
||||
// );
|
||||
// wtx.remove(
|
||||
// self.addressbytes_prefix_to_addressindex.data(),
|
||||
// addressbytes.prefix(),
|
||||
// );
|
||||
// wtx.remove(
|
||||
// self.addressindex_to_addressbytes.data(),
|
||||
// &addressindex_slice,
|
||||
// );
|
||||
// wtx.remove(self.addressindex_to_addresstype.data(), &addressindex_slice);
|
||||
|
||||
// Ok(())
|
||||
// })?;
|
||||
//
|
||||
|
||||
// todo!("clear addresstxoutindexes_out")
|
||||
// todo!("clear addresstxoutindexes_in")
|
||||
// todo!("clear zero_txoutindexes")
|
||||
|
||||
// Ok(())
|
||||
}
|
||||
|
||||
pub fn flush(&mut self, height: Height) -> io::Result<()> {
|
||||
self.as_mut_slice()
|
||||
.into_par_iter()
|
||||
.try_for_each(|vec| vec.flush(height))
|
||||
}
|
||||
|
||||
pub fn reset_cache(&mut self) {
|
||||
self.as_mut_slice().par_iter_mut().for_each(|vec| {
|
||||
vec.reset_cache();
|
||||
})
|
||||
}
|
||||
|
||||
pub fn min_height(&self) -> color_eyre::Result<Option<Height>> {
|
||||
Ok(self
|
||||
.as_slice()
|
||||
.into_iter()
|
||||
.map(|vec| vec.height().unwrap_or_default())
|
||||
.min())
|
||||
}
|
||||
|
||||
pub fn as_slice(&self) -> [&dyn AnyBindexVec; 30] {
|
||||
[
|
||||
&self.addressindex_to_addresstype as &dyn AnyBindexVec,
|
||||
&self.addressindex_to_addresstypeindex,
|
||||
&self.height_to_blockhash,
|
||||
&self.height_to_date,
|
||||
&self.height_to_totalfees,
|
||||
&self.height_to_first_addressindex,
|
||||
&self.height_to_first_txindex,
|
||||
&self.height_to_first_txoutindex,
|
||||
&self.height_to_inputcount,
|
||||
&self.height_to_last_addressindex,
|
||||
&self.height_to_last_txindex,
|
||||
&self.height_to_last_txoutindex,
|
||||
&self.height_to_outputcount,
|
||||
&self.height_to_timestamp,
|
||||
&self.height_to_txcount,
|
||||
&self.p2pk33index_to_p2pk33addressbytes,
|
||||
&self.p2pk65index_to_p2pk65addressbytes,
|
||||
&self.p2pkhindex_to_p2pkhaddressbytes,
|
||||
&self.p2shindex_to_p2shaddressbytes,
|
||||
&self.p2trindex_to_p2traddressbytes,
|
||||
&self.p2wpkhindex_to_p2wpkhaddressbytes,
|
||||
&self.p2wshindex_to_p2wshaddressbytes,
|
||||
&self.txindex_to_fee,
|
||||
&self.txindex_to_height,
|
||||
&self.txindex_to_inputcount,
|
||||
&self.txindex_to_outputcount,
|
||||
&self.txindex_to_txid,
|
||||
&self.txindex_to_txversion,
|
||||
&self.txoutindex_to_addressindex,
|
||||
&self.txoutindex_to_amount,
|
||||
]
|
||||
}
|
||||
|
||||
pub fn as_mut_slice(&mut self) -> [&mut (dyn AnyBindexVec + Send + Sync); 30] {
|
||||
[
|
||||
&mut self.addressindex_to_addresstype as &mut (dyn AnyBindexVec + Send + Sync),
|
||||
&mut self.addressindex_to_addresstypeindex,
|
||||
&mut self.height_to_blockhash,
|
||||
&mut self.height_to_date,
|
||||
&mut self.height_to_totalfees, // <-
|
||||
&mut self.height_to_first_addressindex,
|
||||
&mut self.height_to_first_txindex,
|
||||
&mut self.height_to_first_txoutindex,
|
||||
&mut self.height_to_inputcount, // <-
|
||||
&mut self.height_to_last_addressindex,
|
||||
&mut self.height_to_last_txindex,
|
||||
&mut self.height_to_last_txoutindex,
|
||||
&mut self.height_to_outputcount, // <-
|
||||
&mut self.height_to_timestamp,
|
||||
&mut self.height_to_txcount, // <-
|
||||
&mut self.p2pk33index_to_p2pk33addressbytes,
|
||||
&mut self.p2pk65index_to_p2pk65addressbytes,
|
||||
&mut self.p2pkhindex_to_p2pkhaddressbytes,
|
||||
&mut self.p2shindex_to_p2shaddressbytes,
|
||||
&mut self.p2trindex_to_p2traddressbytes,
|
||||
&mut self.p2wpkhindex_to_p2wpkhaddressbytes,
|
||||
&mut self.p2wshindex_to_p2wshaddressbytes,
|
||||
&mut self.txindex_to_fee, // <-
|
||||
&mut self.txindex_to_height,
|
||||
&mut self.txindex_to_inputcount, // <-
|
||||
&mut self.txindex_to_outputcount, // <-
|
||||
&mut self.txindex_to_txid,
|
||||
&mut self.txindex_to_txversion,
|
||||
&mut self.txoutindex_to_addressindex,
|
||||
&mut self.txoutindex_to_amount,
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,30 +1,25 @@
|
||||
use derive_deref::{Deref, DerefMut};
|
||||
use std::{fs, io, path::Path};
|
||||
|
||||
use super::SliceExtended;
|
||||
use storable_vec::UnsafeSizedSerDe;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deref, DerefMut, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Version(u8);
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Version(u32);
|
||||
|
||||
impl From<u8> for Version {
|
||||
fn from(value: u8) -> Self {
|
||||
impl Version {
|
||||
pub fn write(&self, path: &Path) -> Result<(), io::Error> {
|
||||
fs::write(path, self.unsafe_as_slice())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u32> for Version {
|
||||
fn from(value: u32) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<Slice> for Version {
|
||||
impl TryFrom<&Path> for Version {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from(value: Slice) -> Result<Self, Self::Error> {
|
||||
Self::try_from(&value[..])
|
||||
}
|
||||
}
|
||||
impl TryFrom<&[u8]> for Version {
|
||||
type Error = color_eyre::Report;
|
||||
fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
|
||||
Ok(Self::from(value.read_be_u8()?))
|
||||
}
|
||||
}
|
||||
impl From<Version> for Slice {
|
||||
fn from(value: Version) -> Self {
|
||||
value.to_be_bytes().into()
|
||||
fn try_from(value: &Path) -> Result<Self, Self::Error> {
|
||||
Ok(Self::unsafe_try_from_slice(fs::read(value)?.as_slice())?.to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "snkrj"
|
||||
description = "A simple wrapper around Sanakirja's database that acts as a very fast on disk BTreeMap"
|
||||
description = "A very simple wrapper around Sanakirja"
|
||||
version = "0.1.1"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/kibo-money/kibo/tree/main/src/crates/snkrj"
|
||||
|
||||
+141
-52
@@ -1,11 +1,13 @@
|
||||
// https://docs.rs/sanakirja/latest/sanakirja/index.html
|
||||
// https://pijul.org/posts/2021-02-06-rethinking-sanakirja/
|
||||
|
||||
use core::panic;
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fmt::Debug,
|
||||
fs, io, mem,
|
||||
path::PathBuf,
|
||||
fs::{self, File},
|
||||
io, mem,
|
||||
path::{Path, PathBuf},
|
||||
result::Result,
|
||||
};
|
||||
|
||||
@@ -19,12 +21,13 @@ pub use sanakirja::*;
|
||||
///
|
||||
pub struct Database<Key, Value>
|
||||
where
|
||||
Key: Ord + Clone + Debug + Storable,
|
||||
Value: Storable + PartialEq,
|
||||
Key: DatabaseKey,
|
||||
Value: DatabaseValue,
|
||||
{
|
||||
path: PathBuf,
|
||||
pathbuf: PathBuf,
|
||||
puts: BTreeMap<Key, Value>,
|
||||
dels: BTreeSet<Key>,
|
||||
len: usize,
|
||||
db: Db_<Key, Value, page::Page<Key, Value>>,
|
||||
txn: MutTxn<Env, ()>,
|
||||
}
|
||||
@@ -32,14 +35,24 @@ where
|
||||
const ROOT_DB: usize = 0;
|
||||
const PAGE_SIZE: u64 = 4096;
|
||||
|
||||
pub type UnitDatabase = Database<(), ()>;
|
||||
|
||||
const DEFRAGMENT_RATIO_THRESHOLD: f64 = 0.5;
|
||||
|
||||
impl<Key, Value> Database<Key, Value>
|
||||
where
|
||||
Key: Ord + Clone + Debug + Storable,
|
||||
Value: Storable + PartialEq,
|
||||
Key: DatabaseKey,
|
||||
Value: DatabaseValue,
|
||||
{
|
||||
const KEY_SIZE: usize = size_of::<Key>();
|
||||
const VALUE_SIZE: usize = size_of::<Value>();
|
||||
const KEY_AND_VALUE_SIZE: usize = Self::KEY_SIZE + Self::VALUE_SIZE;
|
||||
|
||||
/// Open a database without a lock file where only one instance is safe to open.
|
||||
pub fn open(path: PathBuf) -> Result<Self, Error> {
|
||||
let env = unsafe { Env::new_nolock(&path, PAGE_SIZE, 1)? };
|
||||
pub fn open(pathbuf: PathBuf) -> Result<Self, Error> {
|
||||
fs::create_dir_all(&pathbuf)?;
|
||||
|
||||
let env = unsafe { Env::new_nolock(Self::path_sanakirja_(&pathbuf), PAGE_SIZE, 1)? };
|
||||
|
||||
let mut txn = Env::mut_txn_begin(env)?;
|
||||
|
||||
@@ -48,7 +61,8 @@ where
|
||||
.unwrap_or_else(|| unsafe { btree::create_db_(&mut txn).unwrap() });
|
||||
|
||||
Ok(Self {
|
||||
path,
|
||||
len: Self::read_length_(&pathbuf),
|
||||
pathbuf,
|
||||
puts: BTreeMap::default(),
|
||||
dels: BTreeSet::default(),
|
||||
db,
|
||||
@@ -56,6 +70,37 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
pub fn path_sanakirja(&self) -> PathBuf {
|
||||
Self::path_sanakirja_(&self.pathbuf)
|
||||
}
|
||||
fn path_sanakirja_(path: &Path) -> PathBuf {
|
||||
path.join("sanakirja")
|
||||
}
|
||||
|
||||
pub fn read_length(&self) -> usize {
|
||||
Self::read_length_(&self.pathbuf)
|
||||
}
|
||||
pub fn read_length_(path: &Path) -> usize {
|
||||
fs::read(Self::path_length(path))
|
||||
.map(|v| {
|
||||
let mut buf = [0_u8; 8];
|
||||
v.iter().enumerate().take(8).for_each(|(i, b)| {
|
||||
buf[i] = *b;
|
||||
});
|
||||
usize::from_le_bytes(buf)
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
pub fn write_length(&self) -> Result<(), io::Error> {
|
||||
Self::write_length_(&self.pathbuf, self.len)
|
||||
}
|
||||
pub fn write_length_(path: &Path, len: usize) -> Result<(), io::Error> {
|
||||
fs::write(Self::path_length(path), len.to_le_bytes())
|
||||
}
|
||||
fn path_length(path: &Path) -> PathBuf {
|
||||
path.join("length")
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get(&self, key: &Key) -> Option<&Value> {
|
||||
if let Some(cached_put) = self.get_from_ram(key) {
|
||||
@@ -100,6 +145,7 @@ where
|
||||
/// Insert without removing the key to the dels tree, so be sure that it hasn't added to the delete set
|
||||
#[inline]
|
||||
pub fn insert_to_ram(&mut self, key: Key, value: Value) -> Option<Value> {
|
||||
self.len += 1;
|
||||
self.puts.insert(key, value)
|
||||
}
|
||||
|
||||
@@ -111,9 +157,9 @@ where
|
||||
|
||||
#[inline]
|
||||
pub fn remove(&mut self, key: &Key) -> Option<Value> {
|
||||
self.remove_from_ram(key).or_else(|| {
|
||||
self.remove_later_from_disk(key);
|
||||
|
||||
self.len -= 1;
|
||||
self.puts.remove(key).or_else(|| {
|
||||
self.dels.insert(key.clone());
|
||||
None
|
||||
})
|
||||
}
|
||||
@@ -121,20 +167,17 @@ where
|
||||
/// Get only from the uncommited tree (ram) without checking the database (disk)
|
||||
#[inline]
|
||||
pub fn remove_from_ram(&mut self, key: &Key) -> Option<Value> {
|
||||
self.len -= 1;
|
||||
self.puts.remove(key)
|
||||
}
|
||||
|
||||
/// Add the key only to the dels tree without checking if it's present in the puts tree, only use if you are positive that you neither added nor updated an entry with this key
|
||||
#[inline]
|
||||
pub fn remove_later_from_disk(&mut self, key: &Key) {
|
||||
self.len -= 1;
|
||||
self.dels.insert(key.clone());
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.iter_disk().next().is_none()
|
||||
}
|
||||
|
||||
/// Iterate over key/value pairs from the uncommited tree (ram)
|
||||
#[inline]
|
||||
pub fn iter_ram(&self) -> std::collections::btree_map::Iter<'_, Key, Value> {
|
||||
@@ -156,18 +199,12 @@ where
|
||||
}
|
||||
|
||||
/// Collect a **clone** of all uncommited key/value pairs (ram)
|
||||
pub fn collect_ram(&self) -> BTreeMap<Key, Value>
|
||||
where
|
||||
Value: Clone,
|
||||
{
|
||||
pub fn collect_ram(&self) -> BTreeMap<Key, Value> {
|
||||
self.puts.clone()
|
||||
}
|
||||
|
||||
/// Collect a **clone** of all key/value pairs from the database (disk)
|
||||
pub fn collect_disk(&self) -> BTreeMap<Key, Value>
|
||||
where
|
||||
Value: Clone,
|
||||
{
|
||||
pub fn collect_disk(&self) -> BTreeMap<Key, Value> {
|
||||
self.iter_disk()
|
||||
.map(|r| r.unwrap())
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
@@ -176,54 +213,68 @@ where
|
||||
|
||||
#[inline]
|
||||
pub fn len(&self) -> usize {
|
||||
self.iter_ram_then_disk().count()
|
||||
self.len
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AnyDatabase {
|
||||
#[allow(unused)]
|
||||
fn export(self, defragment: bool) -> Result<(), Error>;
|
||||
fn boxed_export(self: Box<Self>, defragment: bool) -> Result<(), Error>;
|
||||
#[allow(unused)]
|
||||
fn destroy(self) -> io::Result<()>;
|
||||
}
|
||||
#[inline]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len == 0
|
||||
}
|
||||
|
||||
impl<Key, Value> AnyDatabase for Database<Key, Value>
|
||||
where
|
||||
Key: Ord + Clone + Debug + Storable,
|
||||
Value: Storable + PartialEq + Clone,
|
||||
{
|
||||
/// Flush all puts and dels from the ram to disk with an option to defragment the database to save some disk space
|
||||
///
|
||||
/// /!\ Do not kill the program while this function is runnning /!\
|
||||
fn export(self, defragment: bool) -> Result<(), Error> {
|
||||
Box::new(self).boxed_export(defragment)
|
||||
// pub fn export(self) -> Result<(), Error> {
|
||||
// self.boxed().boxed_export()
|
||||
// }
|
||||
|
||||
// pub fn boxed(self) -> Box<Self> {
|
||||
// Box::new(self)
|
||||
// }
|
||||
|
||||
pub fn get_file_size_to_data_ratio(&self) -> Result<f64, Error> {
|
||||
let data_bytes = (self.len() * Self::KEY_AND_VALUE_SIZE) as f64;
|
||||
let file_bytes = File::open(&self.pathbuf)?.metadata()?.len() as f64;
|
||||
Ok(file_bytes / data_bytes)
|
||||
}
|
||||
|
||||
/// Flush all puts and dels from the ram to disk with an option to defragment the database to save some disk space
|
||||
///
|
||||
/// /!\ Do not kill the program while this function is runnning /!\
|
||||
fn boxed_export(mut self: Box<Self>, defragment: bool) -> Result<(), Error> {
|
||||
pub fn export(mut self) -> Result<(), Error> {
|
||||
let defragment = self.get_file_size_to_data_ratio()? >= DEFRAGMENT_RATIO_THRESHOLD;
|
||||
|
||||
if defragment {
|
||||
let mut btree = self.as_ref().collect_disk();
|
||||
let mut btree = self.collect_disk();
|
||||
|
||||
let path = self.path.to_owned();
|
||||
let disk_len = btree.len();
|
||||
let dels_len = self.dels.len();
|
||||
let puts_len = self.puts.len();
|
||||
|
||||
let path = self.pathbuf.to_owned();
|
||||
self.dels.iter().for_each(|key| {
|
||||
btree.remove(key);
|
||||
});
|
||||
btree.append(&mut self.puts);
|
||||
|
||||
let len = btree.len();
|
||||
|
||||
if len != self.len {
|
||||
dbg!(len, self.len, path, disk_len, dels_len, puts_len);
|
||||
panic!("Len should be the same");
|
||||
}
|
||||
|
||||
self.destroy()?;
|
||||
|
||||
*self = Self::open(path).unwrap();
|
||||
self = Self::open(path).unwrap();
|
||||
|
||||
if !self.is_empty() {
|
||||
panic!()
|
||||
}
|
||||
|
||||
self.len = len;
|
||||
self.puts = btree;
|
||||
}
|
||||
|
||||
self.write_length()?;
|
||||
|
||||
if self.dels.is_empty() && self.puts.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -247,11 +298,49 @@ where
|
||||
self.txn.commit()
|
||||
}
|
||||
|
||||
fn destroy(self) -> io::Result<()> {
|
||||
let path = self.path.to_owned();
|
||||
pub fn destroy(self) -> io::Result<()> {
|
||||
let path = self.pathbuf.to_owned();
|
||||
|
||||
drop(self);
|
||||
|
||||
fs::remove_file(&path)
|
||||
fs::remove_dir_all(&path)
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AnyDatabase {
|
||||
fn export(self) -> Result<(), Error>;
|
||||
// fn boxed_export(self: Box<Self>) -> Result<(), Error>;
|
||||
fn destroy(self) -> io::Result<()>;
|
||||
}
|
||||
|
||||
impl<Key, Value> AnyDatabase for Database<Key, Value>
|
||||
where
|
||||
Key: DatabaseKey,
|
||||
Value: DatabaseValue,
|
||||
{
|
||||
fn export(self) -> Result<(), Error> {
|
||||
self.export()
|
||||
}
|
||||
|
||||
// fn boxed_export(self: Box<Self>) -> Result<(), Error> {
|
||||
// self.boxed_export()
|
||||
// }
|
||||
|
||||
fn destroy(self) -> io::Result<()> {
|
||||
self.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
pub trait DatabaseKey
|
||||
where
|
||||
Self: Ord + Clone + Debug + Storable + Send + Sync,
|
||||
{
|
||||
}
|
||||
impl<T> DatabaseKey for T where T: Ord + Clone + Debug + Storable + Send + Sync {}
|
||||
|
||||
pub trait DatabaseValue
|
||||
where
|
||||
Self: Clone + Storable + PartialEq + Send + Sync,
|
||||
{
|
||||
}
|
||||
impl<T> DatabaseValue for T where T: Clone + Storable + PartialEq + Send + Sync {}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use snkrj::{AnyDatabase, Database};
|
||||
use snkrj::Database;
|
||||
|
||||
fn main() {
|
||||
let path = std::env::temp_dir().join("./db");
|
||||
@@ -8,7 +8,7 @@ fn main() {
|
||||
|
||||
let mut database: Database<i32, i32> = Database::open(path.clone()).unwrap();
|
||||
database.insert(64, 128);
|
||||
database.export(false).unwrap();
|
||||
database.export().unwrap();
|
||||
|
||||
let mut database: Database<i32, i32> = Database::open(path).unwrap();
|
||||
database.insert(1, 2);
|
||||
@@ -25,5 +25,5 @@ fn main() {
|
||||
database.iter_ram_then_disk().for_each(|pair| {
|
||||
println!("{:?}", pair);
|
||||
});
|
||||
database.export(false).unwrap();
|
||||
database.export().unwrap();
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
/v
|
||||
Generated
-221
@@ -2,118 +2,12 @@
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "addr2line"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
|
||||
dependencies = [
|
||||
"gimli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "adler"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.71"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d"
|
||||
dependencies = [
|
||||
"addr2line",
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"miniz_oxide",
|
||||
"object",
|
||||
"rustc-demangle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b"
|
||||
dependencies = [
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "color-eyre"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55146f5e46f237f7423d74111267d4597b59b0dad0ffaf7303bce9945d843ad5"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"color-spantrace",
|
||||
"eyre",
|
||||
"indenter",
|
||||
"once_cell",
|
||||
"owo-colors",
|
||||
"tracing-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "color-spantrace"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd6be1b2a7e382e2b98b43b2adcca6bb0e465af0bdd38123873ae61eb17a72c2"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"owo-colors",
|
||||
"tracing-core",
|
||||
"tracing-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "eyre"
|
||||
version = "0.6.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec"
|
||||
dependencies = [
|
||||
"indenter",
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gimli"
|
||||
version = "0.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
|
||||
|
||||
[[package]]
|
||||
name = "indenter"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.169"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
||||
|
||||
[[package]]
|
||||
name = "memmap2"
|
||||
version = "0.9.5"
|
||||
@@ -123,124 +17,9 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"
|
||||
dependencies = [
|
||||
"adler",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.32.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.20.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
|
||||
|
||||
[[package]]
|
||||
name = "owo-colors"
|
||||
version = "3.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
|
||||
|
||||
[[package]]
|
||||
name = "sharded-slab"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "storable_vec"
|
||||
version = "0.1.2"
|
||||
dependencies = [
|
||||
"color-eyre",
|
||||
"memmap2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
|
||||
dependencies = [
|
||||
"pin-project-lite",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-core"
|
||||
version = "0.1.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"valuable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-error"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db"
|
||||
dependencies = [
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-subscriber"
|
||||
version = "0.3.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
|
||||
dependencies = [
|
||||
"sharded-slab",
|
||||
"thread_local",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
|
||||
|
||||
@@ -8,5 +8,4 @@ categories = ["database"]
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
color-eyre = "0.6.3"
|
||||
memmap2 = "0.9.5"
|
||||
|
||||
@@ -7,6 +7,7 @@ A very small, fast, efficient and simple storable `vec` which uses `mmap2` for s
|
||||
- [x] Get (Rayon compatible)
|
||||
- [x] Push
|
||||
- [ ] Update
|
||||
- [ ] Insert
|
||||
- [ ] Remove
|
||||
|
||||
## Example
|
||||
|
||||
@@ -1,21 +1,19 @@
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
fmt::Debug,
|
||||
fmt::{self, Debug},
|
||||
fs::{File, OpenOptions},
|
||||
io::{self, Write},
|
||||
marker::PhantomData,
|
||||
mem,
|
||||
ops::{Deref, DerefMut, Range},
|
||||
ops::Range,
|
||||
path::Path,
|
||||
sync::OnceLock,
|
||||
};
|
||||
|
||||
use color_eyre::eyre::{eyre, ContextCompat};
|
||||
|
||||
use memmap2::{Mmap, MmapOptions};
|
||||
|
||||
///
|
||||
/// A Push only vec stored on disk using Mmap
|
||||
/// A very small, fast, efficient and simple storable Vec
|
||||
///
|
||||
/// Reads (imports of Mmap) are lazy
|
||||
///
|
||||
@@ -23,17 +21,24 @@ use memmap2::{Mmap, MmapOptions};
|
||||
///
|
||||
/// The file isn't portable for speed reasons (TODO: but could be ?)
|
||||
///
|
||||
/// If you don't call `.flush()` it just acts as a normal Vec
|
||||
///
|
||||
#[derive(Debug)]
|
||||
pub struct StorableVec<I, T> {
|
||||
file: File,
|
||||
mmaps: VecLazyMmap,
|
||||
cache: Vec<OnceLock<Box<Mmap>>>, // Boxed to reduce the size of the lock (24 > 16)
|
||||
disk_len: usize,
|
||||
cache: Vec<T>,
|
||||
pushed: Vec<T>,
|
||||
// updated: BTreeMap<usize, T>,
|
||||
// inserted: BTreeMap<usize, T>,
|
||||
// removed: BTreeSet<usize>,
|
||||
phantom: PhantomData<I>,
|
||||
}
|
||||
|
||||
/// In bytes
|
||||
const MAX_PAGE_SIZE: usize = 4096;
|
||||
const ONE_MB: usize = 1024 * 1024;
|
||||
const MAX_CACHE_SIZE: usize = 50 * ONE_MB;
|
||||
|
||||
impl<I, T> StorableVec<I, T>
|
||||
where
|
||||
@@ -45,26 +50,31 @@ where
|
||||
pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE;
|
||||
/// In bytes
|
||||
pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE;
|
||||
pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
|
||||
|
||||
pub fn import(path: &Path) -> color_eyre::Result<Self> {
|
||||
pub fn import(path: &Path) -> Result<Self, io::Error> {
|
||||
let file = Self::open_file(path)?;
|
||||
|
||||
let mut this = Self {
|
||||
disk_len: Self::byte_index_to_index(file.metadata()?.len() as usize),
|
||||
file,
|
||||
mmaps: VecLazyMmap::default(),
|
||||
cache: vec![],
|
||||
pushed: vec![],
|
||||
// updated: BTreeMap::new(),
|
||||
// inserted: BTreeMap::new(),
|
||||
// removed: BTreeSet::new(),
|
||||
phantom: PhantomData,
|
||||
};
|
||||
|
||||
this.reset_mmaps();
|
||||
this.reset_cache();
|
||||
|
||||
Ok(this)
|
||||
}
|
||||
|
||||
fn reset_mmaps(&mut self) {
|
||||
self.mmaps
|
||||
.reset((self.disk_len as f64 / Self::PER_PAGE as f64).ceil() as usize);
|
||||
pub fn reset_cache(&mut self) {
|
||||
let len = (self.disk_len as f64 / Self::PER_PAGE as f64).ceil() as usize;
|
||||
self.cache.clear();
|
||||
self.cache.resize_with(len, Default::default);
|
||||
}
|
||||
|
||||
fn open_file(path: &Path) -> Result<File, io::Error> {
|
||||
@@ -97,63 +107,85 @@ where
|
||||
byte_index / Self::SIZE
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
#[inline]
|
||||
pub fn get(&self, index: I) -> color_eyre::Result<Option<&T>> {
|
||||
self._get(index.into())
|
||||
}
|
||||
pub fn _get(&self, index: usize) -> color_eyre::Result<Option<&T>> {
|
||||
if self.disk_len == 0 || index > self.disk_len - 1 {
|
||||
Ok(self.cache.get(index - self.disk_len))
|
||||
} else {
|
||||
let mmap_index = Self::index_to_mmap_index(index);
|
||||
|
||||
let mmap = self
|
||||
.mmaps
|
||||
.get(mmap_index)
|
||||
.context("Expect mmap to be open")?
|
||||
.get_or_load(
|
||||
MAX_PAGE_SIZE,
|
||||
(mmap_index * Self::PAGE_SIZE) as u64,
|
||||
&self.file,
|
||||
);
|
||||
|
||||
let range = Self::index_to_range(index);
|
||||
let src = &mmap[range];
|
||||
|
||||
let (prefix, shorts, suffix) = unsafe { src.align_to::<T>() };
|
||||
|
||||
if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() {
|
||||
dbg!(&src, &prefix, &shorts, &suffix);
|
||||
return Err(eyre!("align_to issue"));
|
||||
fn index_to_pushed_index(&self, index: usize) -> Result<Option<usize>> {
|
||||
if index >= self.disk_len {
|
||||
let index = index - self.disk_len;
|
||||
if index >= self.pushed.len() {
|
||||
Err(Error::IndexTooHigh)
|
||||
} else {
|
||||
Ok(Some(index))
|
||||
}
|
||||
|
||||
Ok(Some(&shorts[0]))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn first(&self) -> color_eyre::Result<Option<&T>> {
|
||||
self._get(0)
|
||||
#[inline]
|
||||
pub fn get(&self, index: I) -> Result<Option<&T>> {
|
||||
self.get_(index.into())
|
||||
}
|
||||
pub fn get_(&self, index: usize) -> Result<Option<&T>> {
|
||||
match self.index_to_pushed_index(index) {
|
||||
Ok(index) => {
|
||||
if let Some(index) = index {
|
||||
return Ok(self.pushed.get(index));
|
||||
}
|
||||
}
|
||||
Err(Error::IndexTooHigh) => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
|
||||
// if !self.updated.is_empty() {
|
||||
// if let Some(v) = self.updated.get(&index) {
|
||||
// return Ok(Some(v));
|
||||
// }
|
||||
// }
|
||||
|
||||
let mmap_index = Self::index_to_mmap_index(index);
|
||||
|
||||
let mmap = &**self
|
||||
.cache
|
||||
.get(mmap_index)
|
||||
.ok_or(Error::MmapsVecIsTooSmall)?
|
||||
.get_or_init(|| {
|
||||
Box::new(unsafe {
|
||||
MmapOptions::new()
|
||||
.len(MAX_PAGE_SIZE)
|
||||
.offset((mmap_index * Self::PAGE_SIZE) as u64)
|
||||
.map(&self.file)
|
||||
.unwrap()
|
||||
})
|
||||
});
|
||||
|
||||
let range = Self::index_to_range(index);
|
||||
let src = &mmap[range];
|
||||
|
||||
Ok(Some(T::unsafe_try_from_slice(src)?))
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn last(&self) -> color_eyre::Result<Option<&T>> {
|
||||
pub fn first(&self) -> Result<Option<&T>> {
|
||||
self.get_(0)
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn last(&self) -> Result<Option<&T>> {
|
||||
let len = self.len();
|
||||
if len == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
self._get(len - 1)
|
||||
self.get_(len - 1)
|
||||
}
|
||||
|
||||
pub fn push(&mut self, value: T) {
|
||||
self.cache.push(value)
|
||||
self.pushed.push(value)
|
||||
}
|
||||
|
||||
pub fn push_if_needed(&mut self, index: I, value: T) -> color_eyre::Result<()> {
|
||||
self._push_if_needed(index.into(), value)
|
||||
pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> {
|
||||
self.push_if_needed_(index.into(), value)
|
||||
}
|
||||
pub fn _push_if_needed(&mut self, index: usize, value: T) -> color_eyre::Result<()> {
|
||||
pub fn push_if_needed_(&mut self, index: usize, value: T) -> Result<()> {
|
||||
let len = self.len();
|
||||
match len.cmp(&index) {
|
||||
Ordering::Greater => Ok(()),
|
||||
@@ -161,34 +193,81 @@ where
|
||||
self.push(value);
|
||||
Ok(())
|
||||
}
|
||||
Ordering::Less => {
|
||||
dbg!(std::any::type_name::<I>(), std::any::type_name::<T>());
|
||||
dbg!(len, index, value);
|
||||
Err(eyre!("Index is too high"))
|
||||
}
|
||||
Ordering::Less => Err(Error::IndexTooHigh),
|
||||
}
|
||||
}
|
||||
|
||||
// pub fn update(&mut self, index: I, value: T) -> Result<()> {
|
||||
// self._update(index.into(), value)
|
||||
// }
|
||||
// pub fn update_(&mut self, index: usize, value: T) -> Result<()> {
|
||||
// if let Some(index) = self.index_to_pushed_index(index) {
|
||||
// self.pushed[index] = value;
|
||||
// } else {
|
||||
// self.updated.insert(index, value);
|
||||
// }
|
||||
// Ok(())
|
||||
// }
|
||||
|
||||
// pub fn fetch_update(&mut self, index: I, value: T) -> Result<T>
|
||||
// where
|
||||
// T: Clone,
|
||||
// {
|
||||
// self._fetch_update(index.into(), value)
|
||||
// }
|
||||
// pub fn fetch_update_(&mut self, index: usize, value: T) -> Result<T>
|
||||
// where
|
||||
// T: Clone,
|
||||
// {
|
||||
// let prev_opt = self.updated.insert(index, value);
|
||||
// if let Some(prev) = prev_opt {
|
||||
// Ok(prev)
|
||||
// } else {
|
||||
// Ok(self
|
||||
// ._get(index)?
|
||||
// .ok_or(Error::ExpectFileToHaveIndex)?
|
||||
// .clone())
|
||||
// }
|
||||
// }
|
||||
|
||||
// pub fn remove(&mut self, index: I) {
|
||||
// self._remove(index.into())
|
||||
// }
|
||||
// pub fn remove_(&mut self, index: usize) {
|
||||
// self.removed.insert(index);
|
||||
// }
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.disk_len + self.cache.len()
|
||||
self.disk_len + self.pushed.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
pub fn has(&self, index: I) -> bool {
|
||||
self.has_(index.into())
|
||||
}
|
||||
pub fn has_(&self, index: usize) -> bool {
|
||||
index < self.len()
|
||||
}
|
||||
|
||||
pub fn hasnt(&self, index: I) -> bool {
|
||||
self.hasnt_(index.into())
|
||||
}
|
||||
pub fn hasnt_(&self, index: usize) -> bool {
|
||||
!self.has_(index)
|
||||
}
|
||||
|
||||
pub fn flush(&mut self) -> io::Result<()> {
|
||||
self.disk_len += self.cache.len();
|
||||
self.reset_mmaps();
|
||||
self.disk_len += self.pushed.len();
|
||||
self.reset_cache();
|
||||
|
||||
let mut bytes: Vec<u8> = vec![];
|
||||
|
||||
mem::take(&mut self.cache).into_iter().for_each(|v| {
|
||||
let data: *const T = &v;
|
||||
let data: *const u8 = data as *const u8;
|
||||
let slice = unsafe { std::slice::from_raw_parts(data, Self::SIZE) };
|
||||
bytes.extend_from_slice(slice)
|
||||
});
|
||||
mem::take(&mut self.pushed)
|
||||
.into_iter()
|
||||
.for_each(|v| bytes.extend_from_slice(v.unsafe_as_slice()));
|
||||
|
||||
self.file.write_all(&bytes)
|
||||
}
|
||||
@@ -197,6 +276,7 @@ where
|
||||
pub trait AnyStorableVec {
|
||||
fn len(&self) -> usize;
|
||||
fn is_empty(&self) -> bool;
|
||||
fn reset_cache(&mut self);
|
||||
fn flush(&mut self) -> io::Result<()>;
|
||||
}
|
||||
|
||||
@@ -213,44 +293,60 @@ where
|
||||
self.is_empty()
|
||||
}
|
||||
|
||||
fn reset_cache(&mut self) {
|
||||
self.reset_cache();
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.flush()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct VecLazyMmap(Vec<LazyMmap>);
|
||||
impl VecLazyMmap {
|
||||
pub fn reset(&mut self, len: usize) {
|
||||
self.0.clear();
|
||||
self.0.resize_with(len, Default::default);
|
||||
}
|
||||
}
|
||||
impl Deref for VecLazyMmap {
|
||||
type Target = Vec<LazyMmap>;
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
impl DerefMut for VecLazyMmap {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
pub trait UnsafeSizedSerDe
|
||||
where
|
||||
Self: Sized,
|
||||
{
|
||||
const SIZE: usize = size_of::<Self>();
|
||||
|
||||
/// Box to reduce the size, would be 24 instead
|
||||
#[derive(Debug, Default)]
|
||||
struct LazyMmap(OnceLock<Box<Mmap>>);
|
||||
impl LazyMmap {
|
||||
pub fn get_or_load(&self, len: usize, offset: u64, file: &File) -> &Mmap {
|
||||
self.0.get_or_init(|| {
|
||||
Box::new(unsafe {
|
||||
MmapOptions::new()
|
||||
.len(len)
|
||||
.offset(offset)
|
||||
.map(file)
|
||||
.unwrap()
|
||||
})
|
||||
})
|
||||
fn unsafe_try_from_slice(slice: &[u8]) -> Result<&Self> {
|
||||
let (prefix, shorts, suffix) = unsafe { slice.align_to::<Self>() };
|
||||
|
||||
if !prefix.is_empty() || shorts.len() != 1 || !suffix.is_empty() {
|
||||
// dbg!(&slice, &prefix, &shorts, &suffix);
|
||||
return Err(Error::FailedToAlignToSelf);
|
||||
}
|
||||
|
||||
Ok(&shorts[0])
|
||||
}
|
||||
|
||||
fn unsafe_as_slice(&self) -> &[u8] {
|
||||
let data: *const Self = self;
|
||||
let data: *const u8 = data as *const u8;
|
||||
unsafe { std::slice::from_raw_parts(data, Self::SIZE) }
|
||||
}
|
||||
}
|
||||
impl<T> UnsafeSizedSerDe for T {}
|
||||
|
||||
pub type Result<T, E = Error> = std::result::Result<T, E>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
MmapsVecIsTooSmall,
|
||||
FailedToAlignToSelf,
|
||||
IndexTooHigh,
|
||||
ExpectFileToHaveIndex,
|
||||
ExpectVecToHaveIndex,
|
||||
}
|
||||
impl fmt::Display for Error {
|
||||
// This trait requires `fmt` with this exact signature.
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
match self {
|
||||
Error::MmapsVecIsTooSmall => write!(f, "Mmaps vec is too small"),
|
||||
Error::FailedToAlignToSelf => write!(f, "Failed to align_to for T"),
|
||||
Error::IndexTooHigh => write!(f, "Index too high"),
|
||||
Error::ExpectFileToHaveIndex => write!(f, "Expect file to have index"),
|
||||
Error::ExpectVecToHaveIndex => write!(f, "Expect vec to have index"),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl std::error::Error for Error {}
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
use std::path::Path;
|
||||
|
||||
use storable_vec::{AnyStorableVec, StorableVec};
|
||||
|
||||
fn main() -> color_eyre::Result<()> {
|
||||
color_eyre::install()?;
|
||||
use storable_vec::StorableVec;
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
{
|
||||
let mut vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
|
||||
|
||||
vec.push(21);
|
||||
dbg!(vec.get(0)?); // 21
|
||||
vec.push(0);
|
||||
vec.push(1);
|
||||
vec.push(2);
|
||||
dbg!(vec.get(0)?); // Some(0)
|
||||
dbg!(vec.get(21)?); // None
|
||||
|
||||
vec.flush()?;
|
||||
}
|
||||
@@ -17,7 +18,7 @@ fn main() -> color_eyre::Result<()> {
|
||||
{
|
||||
let vec: StorableVec<usize, u32> = StorableVec::import(Path::new("./v"))?;
|
||||
|
||||
dbg!(vec.get(0)?); // 21
|
||||
dbg!(vec.get(0)?); // 0
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
+40
-32
@@ -120,33 +120,37 @@ impl Binance {
|
||||
.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(|value| {
|
||||
.map(|value| -> color_eyre::Result<_> {
|
||||
// [timestamp, open, high, low, close, volume, ...]
|
||||
let array = value.as_array().unwrap();
|
||||
let array = value.as_array().context("Expect to be array")?;
|
||||
|
||||
let timestamp = (array.first().unwrap().as_u64().unwrap() / 1_000) as u32;
|
||||
let timestamp = (array
|
||||
.first()
|
||||
.context("Expect to have first")?
|
||||
.as_u64()
|
||||
.context("Expect to be convertible to u64")?
|
||||
/ 1_000) as u32;
|
||||
|
||||
let get_f32 = |index: usize| {
|
||||
array
|
||||
let get_f32 = |index: usize| -> color_eyre::Result<f32> {
|
||||
Ok(array
|
||||
.get(index)
|
||||
.unwrap()
|
||||
.context("Expect to have index")?
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.parse::<f32>()
|
||||
.unwrap()
|
||||
.context("Expect to have &str")?
|
||||
.parse::<f32>()?)
|
||||
};
|
||||
|
||||
(
|
||||
Ok((
|
||||
timestamp,
|
||||
OHLC {
|
||||
open: get_f32(1),
|
||||
high: get_f32(2),
|
||||
low: get_f32(3),
|
||||
close: get_f32(4),
|
||||
open: get_f32(1)?,
|
||||
high: get_f32(2)?,
|
||||
low: get_f32(3)?,
|
||||
close: get_f32(4)?,
|
||||
},
|
||||
)
|
||||
))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>())
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()?)
|
||||
},
|
||||
30,
|
||||
10,
|
||||
@@ -167,36 +171,40 @@ impl Binance {
|
||||
.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(|value| {
|
||||
.map(|value| -> color_eyre::Result<_> {
|
||||
// [timestamp, open, high, low, close, volume, ...]
|
||||
let array = value.as_array().unwrap();
|
||||
let array = value.as_array().context("Expect to be array")?;
|
||||
|
||||
let date = Timestamp::from(
|
||||
(array.first().unwrap().as_u64().unwrap() / 1_000) as u32,
|
||||
(array
|
||||
.first()
|
||||
.context("Expect to have first")?
|
||||
.as_u64()
|
||||
.context("Expect to be convertible to u64")?
|
||||
/ 1_000) as u32,
|
||||
)
|
||||
.to_date();
|
||||
|
||||
let get_f32 = |index: usize| {
|
||||
array
|
||||
let get_f32 = |index: usize| -> color_eyre::Result<f32> {
|
||||
Ok(array
|
||||
.get(index)
|
||||
.unwrap()
|
||||
.context("Expect to have index")?
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.parse::<f32>()
|
||||
.unwrap()
|
||||
.context("Expect to have &str")?
|
||||
.parse::<f32>()?)
|
||||
};
|
||||
|
||||
(
|
||||
Ok((
|
||||
date,
|
||||
OHLC {
|
||||
open: get_f32(1),
|
||||
high: get_f32(2),
|
||||
low: get_f32(3),
|
||||
close: get_f32(4),
|
||||
open: get_f32(1)?,
|
||||
high: get_f32(2)?,
|
||||
low: get_f32(3)?,
|
||||
close: get_f32(4)?,
|
||||
},
|
||||
)
|
||||
))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>())
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()?)
|
||||
},
|
||||
30,
|
||||
10,
|
||||
|
||||
+23
-17
@@ -2,7 +2,6 @@ use std::{collections::BTreeMap, str::FromStr};
|
||||
|
||||
use chrono::NaiveDate;
|
||||
use color_eyre::eyre::ContextCompat;
|
||||
use itertools::Itertools;
|
||||
use log::info;
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -40,7 +39,7 @@ impl Kibo {
|
||||
))?
|
||||
.json()?;
|
||||
|
||||
Ok(body
|
||||
let vec = body
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.get("dataset")
|
||||
@@ -53,7 +52,9 @@ impl Kibo {
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(Self::value_to_ohlc)
|
||||
.collect_vec())
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
Ok(vec)
|
||||
},
|
||||
30,
|
||||
RETRIES,
|
||||
@@ -85,28 +86,33 @@ impl Kibo {
|
||||
.as_object()
|
||||
.context("Expect to be an object")?
|
||||
.iter()
|
||||
.map(|(serialized_date, value)| {
|
||||
let date = Date::wrap(NaiveDate::from_str(serialized_date).unwrap());
|
||||
|
||||
(date, Self::value_to_ohlc(value))
|
||||
.map(|(serialized_date, value)| -> color_eyre::Result<_> {
|
||||
let date = Date::wrap(NaiveDate::from_str(serialized_date)?);
|
||||
Ok((date, Self::value_to_ohlc(value)?))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>())
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()?)
|
||||
},
|
||||
30,
|
||||
RETRIES,
|
||||
)
|
||||
}
|
||||
|
||||
fn value_to_ohlc(value: &Value) -> OHLC {
|
||||
let ohlc = value.as_object().unwrap();
|
||||
fn value_to_ohlc(value: &Value) -> color_eyre::Result<OHLC> {
|
||||
let ohlc = value.as_object().context("Expect as_object to work")?;
|
||||
|
||||
let get_value = |key: &str| ohlc.get(key).unwrap().as_f64().unwrap() as f32;
|
||||
let get_value = |key: &str| -> color_eyre::Result<f32> {
|
||||
Ok(ohlc
|
||||
.get(key)
|
||||
.context("Expect get key to work")?
|
||||
.as_f64()
|
||||
.context("Expect as_f64 to work")? as f32)
|
||||
};
|
||||
|
||||
OHLC {
|
||||
open: get_value("open"),
|
||||
high: get_value("high"),
|
||||
low: get_value("low"),
|
||||
close: get_value("close"),
|
||||
}
|
||||
Ok(OHLC {
|
||||
open: get_value("open")?,
|
||||
high: get_value("high")?,
|
||||
low: get_value("low")?,
|
||||
close: get_value("close")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+43
-33
@@ -34,32 +34,36 @@ impl Kraken {
|
||||
.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(|value| {
|
||||
let array = value.as_array().unwrap();
|
||||
.map(|value| -> color_eyre::Result<_> {
|
||||
let array = value.as_array().context("Expect as_array to work")?;
|
||||
|
||||
let timestamp = array.first().unwrap().as_u64().unwrap() as u32;
|
||||
let timestamp = array
|
||||
.first()
|
||||
.context("Expect first to work")?
|
||||
.as_u64()
|
||||
.expect("Expect as_u64 to work")
|
||||
as u32;
|
||||
|
||||
let get_f32 = |index: usize| {
|
||||
array
|
||||
let get_f32 = |index: usize| -> color_eyre::Result<f32> {
|
||||
Ok(array
|
||||
.get(index)
|
||||
.unwrap()
|
||||
.context("Expect get index to work")?
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.parse::<f32>()
|
||||
.unwrap()
|
||||
.context("Expect as_str to work")?
|
||||
.parse::<f32>()?)
|
||||
};
|
||||
|
||||
(
|
||||
Ok((
|
||||
timestamp,
|
||||
OHLC {
|
||||
open: get_f32(1),
|
||||
high: get_f32(2),
|
||||
low: get_f32(3),
|
||||
close: get_f32(4),
|
||||
open: get_f32(1)?,
|
||||
high: get_f32(2)?,
|
||||
low: get_f32(3)?,
|
||||
close: get_f32(4)?,
|
||||
},
|
||||
)
|
||||
))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>())
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()?)
|
||||
},
|
||||
30,
|
||||
10,
|
||||
@@ -88,33 +92,39 @@ impl Kraken {
|
||||
.as_array()
|
||||
.context("Expect to be an array")?
|
||||
.iter()
|
||||
.map(|value| {
|
||||
let array = value.as_array().unwrap();
|
||||
.map(|value| -> color_eyre::Result<_> {
|
||||
let array = value.as_array().context("Expect as_array to work")?;
|
||||
|
||||
let date = Timestamp::from(array.first().unwrap().as_u64().unwrap() as u32)
|
||||
.to_date();
|
||||
|
||||
let get_f32 = |index: usize| {
|
||||
let date = Timestamp::from(
|
||||
array
|
||||
.first()
|
||||
.context("Expect first to work")?
|
||||
.as_u64()
|
||||
.context("Expect as_u64 to work")?
|
||||
as u32,
|
||||
)
|
||||
.to_date();
|
||||
|
||||
let get_f32 = |index: usize| -> color_eyre::Result<f32> {
|
||||
Ok(array
|
||||
.get(index)
|
||||
.unwrap()
|
||||
.context("Expect get index to work")?
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.parse::<f32>()
|
||||
.unwrap()
|
||||
.context("Expect as_str to work")?
|
||||
.parse::<f32>()?)
|
||||
};
|
||||
|
||||
(
|
||||
Ok((
|
||||
date,
|
||||
OHLC {
|
||||
open: get_f32(1),
|
||||
high: get_f32(2),
|
||||
low: get_f32(3),
|
||||
close: get_f32(4),
|
||||
open: get_f32(1)?,
|
||||
high: get_f32(2)?,
|
||||
low: get_f32(3)?,
|
||||
close: get_f32(4)?,
|
||||
},
|
||||
)
|
||||
))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>())
|
||||
.collect::<Result<BTreeMap<_, _>, _>>()?)
|
||||
},
|
||||
30,
|
||||
10,
|
||||
|
||||
Reference in New Issue
Block a user