Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-20 23:04:46 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: datasets compression via zstd

Browse Source
This commit is contained in:
k
2024-08-05 00:44:46 +02:00
parent 9067c28d24
commit c646d6dc60
36 changed files with 544 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
CHANGELOG.md
View File

@@ -6,9 +6,15 @@
### Parser ### Parser
- Change the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind` - Changed the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind`
- Use the Bitcoin RPC server for various calls instead of running commands and then parsing the JSON from the output - Added datasets compression thanks to [zstd](https://crates.io/crates/zstd) to reduce disk usage
- Updated the config, run with `-h` to see - Use the Bitcoin RPC server for various calls instead of running cli commands and then parsing the JSON from the output
- Updated the config, run with `-h` to see possible args
- Moved outputs from `/target/outputs` to `/out` to allow to run commands like `cargo clean` without side effects
### Server
- Updated the code to support compressed binaries
## v. 0.3.0 | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26 ## v. 0.3.0 | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26

8
README.md
View File

@@ -16,6 +16,12 @@ To promote even more transparency and trust in the network, this project is comm
This project is in a very early stage. The web app will have bugs, the API might break and the data can definitely to be false or slightly false. This project is in a very early stage. The web app will have bugs, the API might break and the data can definitely to be false or slightly false.
## Donations
The project is a lot of work and being worked on full-time. It doesn't have any ads and solely relies on donations. If you find this project useful, any sat would really help make it even better and would be really appreciated.
You can donate on the project's [Geyser Fund](https://geyser.fund/project/satonomics/).
## Instances ## Instances
Web App: Web App:
@@ -88,7 +94,7 @@ _Maybe_:
Aka: Previous iterations Aka: Previous iterations
The initial idea was totally different yet morphed over time into what it is today: a fully FOSS self-hostable on-chain data generator The initial idea was totally different yet morphed over time into what it is today: a fully FOSS self-hostable on-chain data generator.
- https://github.com/drgarlic/satonomics - https://github.com/drgarlic/satonomics
- https://github.com/drgarlic/satonomics-parser - https://github.com/drgarlic/satonomics-parser

3
app/src/app/components/background.tsx
View File

@@ -42,7 +42,8 @@ const texts = [
"absolute scarcity", "absolute scarcity",
"time is scarce", "time is scarce",
"ride or die", "ride or die",
"cyberpunk", "cypherpunk",
"we like the coin",
]; ];
export function Background({ export function Background({

2
app/src/app/components/strip/components/anchorGit.tsx
View File

@@ -5,7 +5,7 @@ export function AnchorGit() {
<Anchor <Anchor
title="Git" title="Git"
icon={() => IconTablerBrandGithubFilled} icon={() => IconTablerBrandGithubFilled}
href="https://github.com/satonomics-org/satonomics" href="https://github.com/satonomics-org"
/> />
); );
} }

1
parser/.gitignore vendored
View File

@@ -19,3 +19,4 @@ benches
parser.log parser.log
config.toml config.toml
*\ copy* *\ copy*
/.log

224
parser/Cargo.lock generated
View File

@@ -48,7 +48,7 @@ checksum = "fe233a377643e0fc1a56421d7c90acdec45c291b30345eb9f08e8d0ddce5a4ab"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 2.0.72",
] ]
[[package]] [[package]]
@@ -289,9 +289,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "2.5.0" version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
@@ -301,9 +301,9 @@ checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa"
[[package]] [[package]]
name = "bytemuck" name = "bytemuck"
version = "1.16.1" version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6"
[[package]] [[package]]
name = "byteorder" name = "byteorder"
@@ -322,6 +322,11 @@ name = "cc"
version = "1.0.99" version = "1.0.99"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695"
dependencies = [
"jobserver",
"libc",
"once_cell",
]
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
@@ -341,14 +346,14 @@ dependencies = [
"num-traits", "num-traits",
"serde", "serde",
"wasm-bindgen", "wasm-bindgen",
"windows-targets 0.52.4", "windows-targets 0.52.6",
] ]
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.5.11" version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc"
dependencies = [ dependencies = [
"clap_builder", "clap_builder",
"clap_derive", "clap_derive",
@@ -356,9 +361,9 @@ dependencies = [
[[package]] [[package]]
name = "clap_builder" name = "clap_builder"
version = "4.5.11" version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99"
dependencies = [ dependencies = [
"anstream", "anstream",
"anstyle", "anstyle",
@@ -368,21 +373,21 @@ dependencies = [
[[package]] [[package]]
name = "clap_derive" name = "clap_derive"
version = "4.5.11" version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
dependencies = [ dependencies = [
"heck", "heck",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 2.0.72",
] ]
[[package]] [[package]]
name = "clap_lex" name = "clap_lex"
version = "0.7.0" version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70"
[[package]] [[package]]
name = "color-eyre" name = "color-eyre"
@@ -448,9 +453,9 @@ dependencies = [
[[package]] [[package]]
name = "crossbeam-channel" name = "crossbeam-channel"
version = "0.5.12" version = "0.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
dependencies = [ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
@@ -485,9 +490,9 @@ dependencies = [
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.8.19" version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]] [[package]]
name = "ctor" name = "ctor"
@@ -532,7 +537,7 @@ checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 2.0.72",
] ]
[[package]] [[package]]
@@ -693,9 +698,9 @@ dependencies = [
[[package]] [[package]]
name = "getrandom" name = "getrandom"
version = "0.2.14" version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"libc", "libc",
@@ -729,9 +734,9 @@ dependencies = [
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.14.3" version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]] [[package]]
name = "heck" name = "heck"
@@ -924,9 +929,9 @@ dependencies = [
[[package]] [[package]]
name = "inferno" name = "inferno"
version = "0.11.20" version = "0.11.21"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c77a3ae7d4761b9c64d2c030f70746ceb8cfba32dce0325a56792e0a4816c31" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88"
dependencies = [ dependencies = [
"ahash", "ahash",
"clap", "clap",
@@ -992,6 +997,15 @@ version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
[[package]]
name = "jobserver"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.69" version = "0.3.69"
@@ -1043,15 +1057,15 @@ dependencies = [
[[package]] [[package]]
name = "log" name = "log"
version = "0.4.21" version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.7.1" version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]] [[package]]
name = "memmap2" name = "memmap2"
@@ -1140,9 +1154,9 @@ dependencies = [
[[package]] [[package]]
name = "num-traits" name = "num-traits"
version = "0.2.18" version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [ dependencies = [
"autocfg", "autocfg",
] ]
@@ -1168,7 +1182,7 @@ version = "0.10.64"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f"
dependencies = [ dependencies = [
"bitflags 2.5.0", "bitflags 2.6.0",
"cfg-if", "cfg-if",
"foreign-types", "foreign-types",
"libc", "libc",
@@ -1185,7 +1199,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 2.0.72",
] ]
[[package]] [[package]]
@@ -1208,9 +1222,9 @@ dependencies = [
[[package]] [[package]]
name = "ordered-float" name = "ordered-float"
version = "4.2.1" version = "4.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19ff2cf528c6c03d9ed653d6c4ce1dc0582dc4af309790ad92f07c1cd551b0be" checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6"
dependencies = [ dependencies = [
"num-traits", "num-traits",
] ]
@@ -1256,7 +1270,7 @@ dependencies = [
"libc", "libc",
"redox_syscall 0.5.2", "redox_syscall 0.5.2",
"smallvec", "smallvec",
"windows-targets 0.52.4", "windows-targets 0.52.6",
] ]
[[package]] [[package]]
@@ -1281,6 +1295,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"toml", "toml",
"zstd",
] ]
[[package]] [[package]]
@@ -1306,7 +1321,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 2.0.72",
] ]
[[package]] [[package]]
@@ -1338,9 +1353,9 @@ dependencies = [
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.85" version = "1.0.86"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
dependencies = [ dependencies = [
"unicode-ident", "unicode-ident",
] ]
@@ -1428,7 +1443,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd"
dependencies = [ dependencies = [
"bitflags 2.5.0", "bitflags 2.6.0",
] ]
[[package]] [[package]]
@@ -1477,9 +1492,9 @@ dependencies = [
[[package]] [[package]]
name = "rgb" name = "rgb"
version = "0.8.44" version = "0.8.37"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aee83dc281d5a3200d37b299acd13b81066ea126a7f16f0eae70fc9aed241d9" checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8"
dependencies = [ dependencies = [
"bytemuck", "bytemuck",
] ]
@@ -1511,7 +1526,7 @@ version = "0.38.32"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89"
dependencies = [ dependencies = [
"bitflags 2.5.0", "bitflags 2.6.0",
"errno", "errno",
"libc", "libc",
"linux-raw-sys", "linux-raw-sys",
@@ -1661,7 +1676,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 2.0.72",
] ]
[[package]] [[package]]
@@ -1768,9 +1783,9 @@ dependencies = [
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.66" version = "2.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@@ -1833,7 +1848,7 @@ checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 2.0.72",
] ]
[[package]] [[package]]
@@ -1913,9 +1928,9 @@ dependencies = [
[[package]] [[package]]
name = "toml" name = "toml"
version = "0.8.16" version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c" checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
dependencies = [ dependencies = [
"serde", "serde",
"serde_spanned", "serde_spanned",
@@ -1925,18 +1940,18 @@ dependencies = [
[[package]] [[package]]
name = "toml_datetime" name = "toml_datetime"
version = "0.6.7" version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
dependencies = [ dependencies = [
"serde", "serde",
] ]
[[package]] [[package]]
name = "toml_edit" name = "toml_edit"
version = "0.22.17" version = "0.22.20"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16" checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"
dependencies = [ dependencies = [
"indexmap", "indexmap",
"serde", "serde",
@@ -2085,9 +2100,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]] [[package]]
name = "version_check" name = "version_check"
version = "0.9.4" version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]] [[package]]
name = "virtue" name = "virtue"
@@ -2131,7 +2146,7 @@ dependencies = [
"once_cell", "once_cell",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 2.0.72",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@@ -2165,7 +2180,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 2.0.72",
"wasm-bindgen-backend", "wasm-bindgen-backend",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@@ -2214,7 +2229,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
dependencies = [ dependencies = [
"windows-targets 0.52.4", "windows-targets 0.52.6",
] ]
[[package]] [[package]]
@@ -2232,7 +2247,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [ dependencies = [
"windows-targets 0.52.4", "windows-targets 0.52.6",
] ]
[[package]] [[package]]
@@ -2252,17 +2267,18 @@ dependencies = [
[[package]] [[package]]
name = "windows-targets" name = "windows-targets"
version = "0.52.4" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [ dependencies = [
"windows_aarch64_gnullvm 0.52.4", "windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.4", "windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.4", "windows_i686_gnu 0.52.6",
"windows_i686_msvc 0.52.4", "windows_i686_gnullvm",
"windows_x86_64_gnu 0.52.4", "windows_i686_msvc 0.52.6",
"windows_x86_64_gnullvm 0.52.4", "windows_x86_64_gnu 0.52.6",
"windows_x86_64_msvc 0.52.4", "windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
] ]
[[package]] [[package]]
@@ -2273,9 +2289,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
[[package]] [[package]]
name = "windows_aarch64_gnullvm" name = "windows_aarch64_gnullvm"
version = "0.52.4" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]] [[package]]
name = "windows_aarch64_msvc" name = "windows_aarch64_msvc"
@@ -2285,9 +2301,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
[[package]] [[package]]
name = "windows_aarch64_msvc" name = "windows_aarch64_msvc"
version = "0.52.4" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]] [[package]]
name = "windows_i686_gnu" name = "windows_i686_gnu"
@@ -2297,9 +2313,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
[[package]] [[package]]
name = "windows_i686_gnu" name = "windows_i686_gnu"
version = "0.52.4" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]] [[package]]
name = "windows_i686_msvc" name = "windows_i686_msvc"
@@ -2309,9 +2331,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
[[package]] [[package]]
name = "windows_i686_msvc" name = "windows_i686_msvc"
version = "0.52.4" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]] [[package]]
name = "windows_x86_64_gnu" name = "windows_x86_64_gnu"
@@ -2321,9 +2343,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
[[package]] [[package]]
name = "windows_x86_64_gnu" name = "windows_x86_64_gnu"
version = "0.52.4" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]] [[package]]
name = "windows_x86_64_gnullvm" name = "windows_x86_64_gnullvm"
@@ -2333,9 +2355,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
[[package]] [[package]]
name = "windows_x86_64_gnullvm" name = "windows_x86_64_gnullvm"
version = "0.52.4" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]] [[package]]
name = "windows_x86_64_msvc" name = "windows_x86_64_msvc"
@@ -2345,15 +2367,15 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]] [[package]]
name = "windows_x86_64_msvc" name = "windows_x86_64_msvc"
version = "0.52.4" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]] [[package]]
name = "winnow" name = "winnow"
version = "0.6.13" version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]
@@ -2395,7 +2417,7 @@ checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 2.0.72",
] ]
[[package]] [[package]]
@@ -2406,7 +2428,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.66", "syn 2.0.72",
] ]
[[package]] [[package]]
@@ -2414,3 +2436,31 @@ name = "zeroize"
version = "1.8.1" version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
[[package]]
name = "zstd"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.13+zstd.1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
dependencies = [
"cc",
"pkg-config",
]

11
parser/Cargo.toml
View File

@@ -14,16 +14,17 @@ bincode = { git = "https://github.com/bincode-org/bincode.git" }
biter = "0.1.0" biter = "0.1.0"
bitcoin_hashes = { version = "0.14.0" } bitcoin_hashes = { version = "0.14.0" }
chrono = { version = "0.4.38", features = ["serde"] } chrono = { version = "0.4.38", features = ["serde"] }
clap = { version = "4.5.11", features = ["derive"] } clap = { version = "4.5.13", features = ["derive"] }
color-eyre = "0.6.3" color-eyre = "0.6.3"
derive_deref = "1.1.1" derive_deref = "1.1.1"
inferno = "0.11.20" inferno = "0.11.21"
itertools = "0.13.0" itertools = "0.13.0"
memory-stats = "1.2.0" memory-stats = "1.2.0"
ordered-float = "4.2.1" ordered-float = "4.2.2"
rayon = "1.10.0" rayon = "1.10.0"
reqwest = { version = "0.12.5", features = ["blocking", "json"] } reqwest = { version = "0.12.5", features = ["blocking", "json"] }
sanakirja = "1.4.2" sanakirja = "1.4.2"
serde = { version = "1.0.204", features = ["derive"] } serde = { version = "1.0.204", features = ["derive"] }
serde_json = "1.0.120" serde_json = "1.0.122"
toml = "0.8.16" toml = "0.8.19"
zstd = "0.13.2"

5
parser/src/actions/iter_blocks.rs
View File

@@ -11,6 +11,7 @@ use crate::{
create_rpc, create_rpc,
databases::Databases, databases::Databases,
datasets::{AllDatasets, ComputeData}, datasets::{AllDatasets, ComputeData},
io::OUTPUTS_FOLDER_PATH,
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates}, states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
structs::{Date, DateData, MapKey}, structs::{Date, DateData, MapKey},
utils::{generate_allocation_files, log, time}, utils::{generate_allocation_files, log, time},
@@ -22,8 +23,6 @@ pub fn iter_blocks(
rpc: &biter::bitcoincore_rpc::Client, rpc: &biter::bitcoincore_rpc::Client,
approx_block_count: usize, approx_block_count: usize,
) -> color_eyre::Result<()> { ) -> color_eyre::Result<()> {
let export_dir = "./target/outputs";
let should_insert = true; let should_insert = true;
let should_export = true; let should_export = true;
let study_ram_usage = false; let study_ram_usage = false;
@@ -56,7 +55,7 @@ pub fn iter_blocks(
let block_receiver = biter::new( let block_receiver = biter::new(
config.datadir.as_ref().unwrap(), config.datadir.as_ref().unwrap(),
export_dir, OUTPUTS_FOLDER_PATH,
Some(height.to_usize()), Some(height.to_usize()),
None, None,
create_rpc(config).unwrap(), create_rpc(config).unwrap(),

12
parser/src/databases/metadata.rs
View File

@@ -1,14 +1,16 @@
use allocative::Allocative; use allocative::Allocative;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use std::{ use std::{
fmt::Debug, fmt::Debug,
fs, io, fs, io,
ops::{Deref, DerefMut}, ops::{Deref, DerefMut},
path::Path,
}; };
use crate::{ use crate::{
io::Binary,
structs::{Counter, Date, Height}, structs::{Counter, Date, Height},
Serialization,
}; };
#[derive(Default, Debug, Encode, Decode, Allocative)] #[derive(Default, Debug, Encode, Decode, Allocative)]
@@ -73,7 +75,7 @@ impl Metadata {
} }
} }
#[derive(Default, Debug, Encode, Decode, Allocative)] #[derive(Default, Debug, Encode, Decode, Serialize, Deserialize, Allocative)]
pub struct MetadataData { pub struct MetadataData {
pub serial: usize, pub serial: usize,
pub len: Counter, pub len: Counter,
@@ -88,17 +90,17 @@ impl MetadataData {
fn full_path(folder_path: &str) -> String { fn full_path(folder_path: &str) -> String {
let name = Self::name(); let name = Self::name();
format!("{folder_path}/{name}.bin") format!("{folder_path}/{name}")
} }
pub fn import(path: &str) -> color_eyre::Result<Self> { pub fn import(path: &str) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?; fs::create_dir_all(path)?;
Binary::import(&Self::full_path(path)) Serialization::Binary.import(Path::new(&Self::full_path(path)))
} }
pub fn export(&self, path: &str) -> color_eyre::Result<()> { pub fn export(&self, path: &str) -> color_eyre::Result<()> {
Binary::export(&Self::full_path(path), self) Serialization::Binary.export(Path::new(&Self::full_path(path)), self)
} }
pub fn reset(&mut self, path: &str) -> color_eyre::Result<(), io::Error> { pub fn reset(&mut self, path: &str) -> color_eyre::Result<(), io::Error> {

4
parser/src/datasets/mod.rs
View File

@@ -1,4 +1,4 @@
use std::{collections::BTreeMap, fs, ops::RangeInclusive}; use std::{collections::BTreeMap, fs, ops::RangeInclusive, path::Path};
use allocative::Allocative; use allocative::Allocative;
@@ -282,7 +282,7 @@ impl AllDatasets {
fs::create_dir_all(server_inputs_path)?; fs::create_dir_all(server_inputs_path)?;
Json::export( Json::export(
&format!("{server_inputs_path}/disk_path_to_type.json"), Path::new(&format!("{server_inputs_path}/disk_path_to_type.json")),
&path_to_type, &path_to_type,
)?; )?;

104
parser/src/io/binary.rs
View File

@@ -1,15 +1,39 @@
use std::{ use std::{
fmt::Debug, fmt::Debug,
fs::File, fs::{self, File},
io::{BufReader, BufWriter}, io::{BufReader, BufWriter, Cursor},
path::Path,
}; };
use bincode::{config, decode_from_std_read, encode_into_std_write, Decode, Encode}; use bincode::{
config, decode_from_slice, decode_from_std_read, encode_into_std_write, Decode, Encode,
};
use zstd::decode_all;
const ZST_EXTENSION: &str = "zst";
pub const BIN_EXTENSION: &str = "bin";
pub const COMPRESSED_BIN_EXTENSION: &str = "bin.zst";
enum BinaryType {
Raw,
Compressed,
}
pub struct Binary; pub struct Binary;
impl Binary { impl Binary {
pub fn import<T>(path: &str) -> color_eyre::Result<T> pub fn import<T>(path: &Path) -> color_eyre::Result<T>
where
T: Decode,
{
match Self::type_from_path(path) {
BinaryType::Compressed => Self::import_compressed(path),
BinaryType::Raw => Self::import_raw(path),
}
}
fn import_raw<T>(path: &Path) -> color_eyre::Result<T>
where where
T: Decode, T: Decode,
{ {
@@ -24,7 +48,34 @@ impl Binary {
Ok(decoded) Ok(decoded)
} }
pub fn export<T>(path: &str, value: &T) -> color_eyre::Result<()> fn import_compressed<T>(path: &Path) -> color_eyre::Result<T>
where
T: Decode,
{
let file = File::open(path).unwrap();
let reader = BufReader::new(file);
let decompressed = decode_all(reader).unwrap();
let config = config::standard();
let decoded = decode_from_slice::<T, _>(&decompressed, config).unwrap().0;
Ok(decoded)
}
pub fn export<T>(path: &Path, value: &T) -> color_eyre::Result<()>
where
T: Debug + Encode,
{
match Self::type_from_path(path) {
BinaryType::Compressed => Self::export_compressed(path, value),
BinaryType::Raw => Self::export_raw(path, value),
}
}
fn export_raw<T>(path: &Path, value: &T) -> color_eyre::Result<()>
where where
T: Debug + Encode, T: Debug + Encode,
{ {
@@ -40,4 +91,47 @@ impl Binary {
Ok(()) Ok(())
} }
fn export_compressed<T>(path: &Path, value: &T) -> color_eyre::Result<()>
where
T: Debug + Encode,
{
let config = config::standard();
let encoded = bincode::encode_to_vec(value, config).unwrap();
let cursor = Cursor::new(encoded);
let compressed = zstd::encode_all(cursor, 0).unwrap();
fs::write(path, compressed).unwrap();
Ok(())
}
pub fn has_correct_extension(path: &Path) -> bool {
let path = path.to_str().unwrap();
path.ends_with(BIN_EXTENSION) || path.ends_with(COMPRESSED_BIN_EXTENSION)
}
fn type_from_path(path: &Path) -> BinaryType {
let extension = path.extension();
if extension.is_none() {
panic!("Should have extension");
}
if !Self::has_correct_extension(path) {
dbg!(path);
panic!("Wrong extension")
}
let extension = extension.unwrap();
if extension == ZST_EXTENSION {
BinaryType::Compressed
} else {
BinaryType::Raw
}
}
} }

2
parser/src/io/consts.rs
View File

@@ -1,2 +1,2 @@
pub const INPUTS_FOLDER_PATH: &str = "./in"; pub const INPUTS_FOLDER_PATH: &str = "./in";
pub const OUTPUTS_FOLDER_PATH: &str = "./target/outputs"; pub const OUTPUTS_FOLDER_PATH: &str = "./out";

23
parser/src/io/json.rs
View File

@@ -1,17 +1,25 @@
use std::{ use std::{
fs::File, fs::File,
io::{BufReader, BufWriter}, io::{BufReader, BufWriter},
path::Path,
}; };
use serde::{de::DeserializeOwned, Serialize}; use serde::{de::DeserializeOwned, Serialize};
pub struct Json; pub struct Json;
pub const JSON_EXTENSION: &str = "json";
pub const HAR_EXTENSION: &str = "har";
impl Json { impl Json {
pub fn import<T>(path: &str) -> color_eyre::Result<T> pub fn import<T>(path: &Path) -> color_eyre::Result<T>
where where
T: DeserializeOwned, T: DeserializeOwned,
{ {
if !Self::has_correct_extension(path) {
panic!("Wrong extension");
}
let file = File::open(path)?; let file = File::open(path)?;
let reader = BufReader::new(file); let reader = BufReader::new(file);
@@ -19,10 +27,15 @@ impl Json {
Ok(serde_json::from_reader(reader)?) Ok(serde_json::from_reader(reader)?)
} }
pub fn export<T>(path: &str, value: &T) -> color_eyre::Result<()> pub fn export<T>(path: &Path, value: &T) -> color_eyre::Result<()>
where where
T: Serialize, T: Serialize,
{ {
if !Self::has_correct_extension(path) {
dbg!(path);
panic!("Wrong extension");
}
let file = File::create(path).unwrap_or_else(|_| { let file = File::create(path).unwrap_or_else(|_| {
dbg!(&path); dbg!(&path);
panic!("No such file or directory") panic!("No such file or directory")
@@ -34,4 +47,10 @@ impl Json {
Ok(()) Ok(())
} }
#[inline(always)]
pub fn has_correct_extension(path: &Path) -> bool {
let path = path.to_str().unwrap();
path.ends_with(JSON_EXTENSION) || path.ends_with(HAR_EXTENSION)
}
} }

103
parser/src/io/serialization.rs
View File

@@ -1,4 +1,4 @@
use std::fmt::Debug; use std::{fmt::Debug, fs, path::Path};
use allocative::Allocative; use allocative::Allocative;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
@@ -6,6 +6,8 @@ use serde::{de::DeserializeOwned, Serialize};
use crate::io::{Binary, Json}; use crate::io::{Binary, Json};
use super::{BIN_EXTENSION, COMPRESSED_BIN_EXTENSION, HAR_EXTENSION, JSON_EXTENSION};
#[derive(PartialEq, PartialOrd, Ord, Eq, Debug, Clone, Copy, Default, Allocative)] #[derive(PartialEq, PartialOrd, Ord, Eq, Debug, Clone, Copy, Default, Allocative)]
pub enum Serialization { pub enum Serialization {
#[default] #[default]
@@ -14,42 +16,105 @@ pub enum Serialization {
} }
impl Serialization { impl Serialization {
pub fn to_extension(&self) -> &str { pub fn is_serializable(&self, path: &Path) -> bool {
let path = path.to_str().unwrap();
match self { match self {
Self::Binary => "bin", Self::Binary => {
Self::Json => "json", path.ends_with(BIN_EXTENSION) || path.ends_with(COMPRESSED_BIN_EXTENSION)
}
Self::Json => path.ends_with(JSON_EXTENSION) || path.ends_with(HAR_EXTENSION),
} }
} }
pub fn from_extension(extension: &str) -> Self { pub fn from_path(path: &Path) -> Self {
match extension { let path = path.to_str().unwrap();
"bin" => Self::Binary, if path.ends_with(BIN_EXTENSION) || path.ends_with(COMPRESSED_BIN_EXTENSION) {
"json" => Self::Json, Self::Binary
_ => panic!("Extension \"{extension}\" isn't supported"), } else if path.ends_with(JSON_EXTENSION) || path.ends_with(HAR_EXTENSION) {
Self::Json
} else {
panic!("Extension \"{path}\" isn't supported")
} }
} }
pub fn append_extension(&self, path: &str) -> String { pub fn import<T>(&self, path: &Path) -> color_eyre::Result<T>
format!("{path}.{}", self.to_extension())
}
pub fn import<T>(&self, path: &str) -> color_eyre::Result<T>
where where
T: Debug + DeserializeOwned + Decode, T: Debug + DeserializeOwned + Decode,
{ {
match self { match self {
Serialization::Binary => Binary::import(path), Serialization::Binary => {
Serialization::Json => Json::import(path), if self.is_serializable(path) {
Binary::import(path)
} else {
let path = path.to_str().unwrap();
let bin_path_str = format!("{path}.{BIN_EXTENSION}");
let bin_path = Path::new(&bin_path_str);
if bin_path.exists() {
return Binary::import(bin_path);
}
let compressed_bin_path_str = format!("{path}.{COMPRESSED_BIN_EXTENSION}");
let compressed_bin_path = Path::new(&compressed_bin_path_str);
if compressed_bin_path.exists() {
return Binary::import(compressed_bin_path);
}
panic!("Wrong path")
}
}
Serialization::Json => {
if self.is_serializable(path) {
Json::import(path)
} else {
let path = path.to_str().unwrap();
let json_path_str = format!("{path}.{JSON_EXTENSION}");
let json_path = Path::new(&json_path_str);
if json_path.exists() {
return Json::import(json_path);
}
panic!("Wrong path")
}
}
} }
} }
pub fn export<T>(&self, path: &str, value: &T) -> color_eyre::Result<()> pub fn export<T>(&self, path: &Path, value: &T) -> color_eyre::Result<()>
where where
T: Debug + Serialize + Encode, T: Debug + Serialize + Encode,
{ {
match self { match self {
Serialization::Binary => Binary::export(path, value), Serialization::Binary => {
Serialization::Json => Json::export(path, value), if self.is_serializable(path) {
Binary::export(path, value)
} else {
let path = path.to_str().unwrap();
let res = Binary::export(
Path::new(&format!("{}.{COMPRESSED_BIN_EXTENSION}", path,)),
value,
);
if res.is_ok() {
let _ = fs::remove_file(Path::new(&format!("{}.{BIN_EXTENSION}", path)));
}
res
}
}
Serialization::Json => {
if self.is_serializable(path) {
Json::export(path, value)
} else {
Json::export(
Path::new(&format!("{}.{JSON_EXTENSION}", path.to_str().unwrap())),
value,
)
}
}
} }
} }
} }

2
parser/src/lib.rs
View File

@@ -15,5 +15,5 @@ pub use crate::{
Config, Date, DateMap, Height, HeightMap, MapChunkId, SerializedBTreeMap, SerializedVec, Config, Date, DateMap, Height, HeightMap, MapChunkId, SerializedBTreeMap, SerializedVec,
HEIGHT_MAP_CHUNK_SIZE, HEIGHT_MAP_CHUNK_SIZE,
}, },
utils::{create_rpc, log}, utils::{create_rpc, log, reset_logs},
}; };

4
parser/src/main.rs
View File

@@ -1,11 +1,13 @@
use std::{thread::sleep, time::Duration}; use std::{thread::sleep, time::Duration};
use biter::bitcoincore_rpc::RpcApi; use biter::bitcoincore_rpc::RpcApi;
use parser::{create_rpc, iter_blocks, log, Config}; use parser::{create_rpc, iter_blocks, log, reset_logs, Config};
fn main() -> color_eyre::Result<()> { fn main() -> color_eyre::Result<()> {
color_eyre::install()?; color_eyre::install()?;
reset_logs();
let config = Config::import(); let config = Config::import();
let rpc = create_rpc(&config).unwrap(); let rpc = create_rpc(&config).unwrap();

3
parser/src/price/binance.rs
View File

@@ -23,8 +23,7 @@ impl Binance {
let path_binance_har = Path::new(INPUTS_FOLDER_PATH).join("binance.har"); let path_binance_har = Path::new(INPUTS_FOLDER_PATH).join("binance.har");
let json: BTreeMap<String, Value> = let json: BTreeMap<String, Value> = Json::import(&path_binance_har).unwrap_or_default();
Json::import(path_binance_har.to_str().unwrap()).unwrap_or_default();
Ok(json Ok(json
.get("log") .get("log")

13
parser/src/states/_trait.rs
View File

@@ -1,13 +1,14 @@
use std::{fmt::Debug, fs, io}; use std::{fmt::Debug, fs, io, path::Path};
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use serde::{de::DeserializeOwned, Serialize};
use crate::io::{Binary, OUTPUTS_FOLDER_PATH}; use crate::{io::OUTPUTS_FOLDER_PATH, Serialization};
// https://github.com/djkoloski/rust_serialization_benchmark // https://github.com/djkoloski/rust_serialization_benchmark
pub trait AnyState pub trait AnyState
where where
Self: Debug + Encode + Decode, Self: Debug + Encode + Decode + Serialize + DeserializeOwned,
{ {
fn name<'a>() -> &'a str; fn name<'a>() -> &'a str;
@@ -24,7 +25,7 @@ where
let folder_path = Self::folder_path(); let folder_path = Self::folder_path();
format!("{folder_path}/{name}.bin") format!("{folder_path}/{name}")
} }
fn reset(&mut self) -> color_eyre::Result<(), io::Error> { fn reset(&mut self) -> color_eyre::Result<(), io::Error> {
@@ -36,11 +37,11 @@ where
fn import() -> color_eyre::Result<Self> { fn import() -> color_eyre::Result<Self> {
Self::create_dir_all()?; Self::create_dir_all()?;
Binary::import(&Self::full_path()) Serialization::Binary.import(Path::new(&Self::full_path()))
} }
fn export(&self) -> color_eyre::Result<()> { fn export(&self) -> color_eyre::Result<()> {
Binary::export(&Self::full_path(), self) Serialization::Binary.export(Path::new(&Self::full_path()), self)
} }
fn clear(&mut self); fn clear(&mut self);

3
parser/src/states/counters.rs
View File

@@ -1,11 +1,12 @@
use allocative::Allocative; use allocative::Allocative;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use crate::structs::Counter; use crate::structs::Counter;
use super::AnyState; use super::AnyState;
#[derive(Default, Debug, Encode, Decode, Allocative)] #[derive(Default, Debug, Encode, Decode, Serialize, Deserialize, Allocative)]
pub struct Counters { pub struct Counters {
pub op_return_addresses: Counter, pub op_return_addresses: Counter,
pub push_only_addresses: Counter, pub push_only_addresses: Counter,

3
parser/src/states/date_data_vec.rs
View File

@@ -1,12 +1,13 @@
use allocative::Allocative; use allocative::Allocative;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use derive_deref::{Deref, DerefMut}; use derive_deref::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
use crate::structs::{BlockData, BlockPath, DateData}; use crate::structs::{BlockData, BlockPath, DateData};
use super::AnyState; use super::AnyState;
#[derive(Default, Deref, DerefMut, Debug, Encode, Decode, Allocative)] #[derive(Default, Deref, DerefMut, Debug, Serialize, Deserialize, Encode, Decode, Allocative)]
pub struct DateDataVec(Vec<DateData>); pub struct DateDataVec(Vec<DateData>);
impl DateDataVec { impl DateDataVec {

3
parser/src/structs/block_data.rs
View File

@@ -1,9 +1,10 @@
use allocative::Allocative; use allocative::Allocative;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use super::{Amount, Height, Price}; use super::{Amount, Height, Price};
#[derive(Debug, Encode, Decode, Allocative)] #[derive(Debug, Serialize, Deserialize, Encode, Decode, Allocative)]
pub struct BlockData { pub struct BlockData {
pub height: Height, pub height: Height,
pub price: Price, pub price: Price,

5
parser/src/structs/counter.rs
View File

@@ -1,8 +1,11 @@
use allocative::Allocative; use allocative::Allocative;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use derive_deref::{Deref, DerefMut}; use derive_deref::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
#[derive(Debug, Deref, DerefMut, Default, Clone, Copy, Encode, Decode, Allocative)] #[derive(
Debug, Deref, DerefMut, Default, Clone, Copy, Encode, Decode, Serialize, Deserialize, Allocative,
)]
pub struct Counter(u32); pub struct Counter(u32);
impl Counter { impl Counter {

3
parser/src/structs/date_data.rs
View File

@@ -1,9 +1,10 @@
use allocative::Allocative; use allocative::Allocative;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use super::{BlockData, BlockPath, Date}; use super::{BlockData, BlockPath, Date};
#[derive(Debug, Encode, Decode, Allocative)] #[derive(Debug, Serialize, Deserialize, Encode, Decode, Allocative)]
pub struct DateData { pub struct DateData {
pub date: Date, pub date: Date,
pub blocks: Vec<BlockData>, pub blocks: Vec<BlockData>,

16
parser/src/structs/date_map_chunk_id.rs
View File

@@ -1,3 +1,5 @@
use std::path::Path;
use allocative::Allocative; use allocative::Allocative;
use chrono::Datelike; use chrono::Datelike;
@@ -19,8 +21,18 @@ impl MapChunkId for DateMapChunkId {
self.0.to_string() self.0.to_string()
} }
fn from_name(name: &str) -> Self { fn from_path(path: &Path) -> Self {
Self(name.parse::<i32>().unwrap()) Self(
path.file_name()
.unwrap()
.to_str()
.unwrap()
.split(".")
.next()
.unwrap()
.parse::<i32>()
.unwrap(),
)
} }
fn to_usize(self) -> usize { fn to_usize(self) -> usize {

32
parser/src/structs/generic_map.rs
View File

@@ -63,7 +63,7 @@ where
Self: Ord + Debug + Copy + Clone, Self: Ord + Debug + Copy + Clone,
{ {
fn to_name(&self) -> String; fn to_name(&self) -> String;
fn from_name(name: &str) -> Self; fn from_path(path: &Path) -> Self;
fn to_usize(self) -> usize; fn to_usize(self) -> usize;
fn from_usize(id: usize) -> Self; fn from_usize(id: usize) -> Self;
} }
@@ -124,7 +124,7 @@ where
let path_last = { let path_last = {
if export_last { if export_last {
Some(serialization.append_extension(&format!("{path}/last"))) Some(format!("{path}/last"))
} else { } else {
None None
} }
@@ -188,22 +188,16 @@ where
fs::read_dir(path) fs::read_dir(path)
.unwrap() .unwrap()
.map(|entry| entry.unwrap().path()) .map(|entry| entry.unwrap().path())
.filter(|path| { .filter(|path| serialization.is_serializable(path))
let extension = path.extension().unwrap().to_str().unwrap();
path.is_file() && extension == serialization.to_extension()
})
.map(|path| { .map(|path| {
let chunk_id = ChunkId::from_name(path.file_stem().unwrap().to_str().unwrap()); let chunk_id = ChunkId::from_path(&path);
(chunk_id, path) (chunk_id, path)
}) })
.collect() .collect()
} }
fn import(&self, path: &Path) -> color_eyre::Result<Serialized> { fn import(&self, path: &Path) -> color_eyre::Result<Serialized> {
self.serialization self.serialization.import::<Serialized>(path)
.import::<Serialized>(path.to_str().unwrap())
} }
pub fn insert(&mut self, key: Key, value: Value) -> Value { pub fn insert(&mut self, key: Key, value: Value) -> Value {
@@ -309,10 +303,7 @@ where
panic!(); panic!();
}); });
let serialized = self let serialized = self.serialization.import::<Serialized>(path).unwrap();
.serialization
.import::<Serialized>(path.to_str().unwrap())
.unwrap();
self.imported.insert(*chunk_id, serialized); self.imported.insert(*chunk_id, serialized);
} }
@@ -334,23 +325,18 @@ where
unreachable!() unreachable!()
} }
let path = self.serialization.append_extension(&format!(
"{}/{}",
self.path_all,
chunk_id.to_name()
));
let serialized = self.imported.get(chunk_id).unwrap_or_else(|| { let serialized = self.imported.get(chunk_id).unwrap_or_else(|| {
dbg!(&self.path_all, chunk_id, &self.imported); dbg!(&self.path_all, chunk_id, &self.imported);
panic!(); panic!();
}); });
self.serialization.export(&path, serialized)?; let path = format!("{}/{}", self.path_all, chunk_id.to_name());
self.serialization.export(Path::new(&path), serialized)?;
if index == len - 1 { if index == len - 1 {
if let Some(path_last) = self.path_last.as_ref() { if let Some(path_last) = self.path_last.as_ref() {
self.serialization self.serialization
.export(path_last, serialized.last().unwrap())?; .export(Path::new(path_last), serialized.last().unwrap())?;
} }
} }

17
parser/src/structs/height_map_chunk_id.rs
View File

@@ -1,3 +1,5 @@
use std::path::Path;
use allocative::Allocative; use allocative::Allocative;
use derive_deref::{Deref, DerefMut}; use derive_deref::{Deref, DerefMut};
@@ -26,9 +28,20 @@ impl MapChunkId for HeightMapChunkId {
format!("{start}..{end}") format!("{start}..{end}")
} }
fn from_name(name: &str) -> Self { fn from_path(path: &Path) -> Self {
Self(Height::new( Self(Height::new(
name.split("..").next().unwrap().parse::<u32>().unwrap(), path.file_name()
.unwrap()
.to_str()
.unwrap()
.split("..")
.next()
.unwrap()
.parse::<u32>()
.unwrap_or_else(|_| {
dbg!(path);
panic!()
}),
)) ))
} }

15
parser/src/structs/price.rs
View File

@@ -2,11 +2,24 @@ use std::ops::{Add, AddAssign, Div, Mul, Sub, SubAssign};
use allocative::Allocative; use allocative::Allocative;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use super::Amount; use super::Amount;
#[derive( #[derive(
Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Allocative, Debug,
Default,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Serialize,
Deserialize,
Encode,
Decode,
Allocative,
)] )]
pub struct Price(u64); pub struct Price(u64);

13
parser/src/utils/log.rs
View File

@@ -1,8 +1,17 @@
use std::{fs::OpenOptions, io::Write}; use std::{
fs::{self, OpenOptions},
io::Write,
};
use chrono::Local; use chrono::Local;
use color_eyre::owo_colors::OwoColorize; use color_eyre::owo_colors::OwoColorize;
const LOG_PATH: &str = "./.log";
pub fn reset_logs() {
let _ = fs::remove_file(LOG_PATH);
}
#[inline(always)] #[inline(always)]
pub fn log(str: &str) { pub fn log(str: &str) {
let date_time = format!("{}", Local::now().format("%Y-%m-%d %H:%M:%S -")); let date_time = format!("{}", Local::now().format("%Y-%m-%d %H:%M:%S -"));
@@ -13,7 +22,7 @@ pub fn log(str: &str) {
let mut file = OpenOptions::new() let mut file = OpenOptions::new()
.create(true) .create(true)
.append(true) .append(true)
.open("./parser.log") .open(LOG_PATH)
.unwrap(); .unwrap();
if let Err(e) = writeln!(file, "{} {}", date_time, line) { if let Err(e) = writeln!(file, "{} {}", date_time, line) {

2
server/.gitignore vendored
View File

@@ -1,4 +1,4 @@
/target /target
.DS_Store .DS_Store
/parser.log /.log
/in /in

49
server/Cargo.lock generated
View File

@@ -477,9 +477,9 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.5.11" version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc"
dependencies = [ dependencies = [
"clap_builder", "clap_builder",
"clap_derive", "clap_derive",
@@ -487,9 +487,9 @@ dependencies = [
[[package]] [[package]]
name = "clap_builder" name = "clap_builder"
version = "4.5.11" version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99"
dependencies = [ dependencies = [
"anstream", "anstream",
"anstyle", "anstyle",
@@ -499,9 +499,9 @@ dependencies = [
[[package]] [[package]]
name = "clap_derive" name = "clap_derive"
version = "4.5.11" version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
dependencies = [ dependencies = [
"heck", "heck",
"proc-macro2", "proc-macro2",
@@ -1081,9 +1081,9 @@ dependencies = [
[[package]] [[package]]
name = "inferno" name = "inferno"
version = "0.11.20" version = "0.11.21"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c77a3ae7d4761b9c64d2c030f70746ceb8cfba32dce0325a56792e0a4816c31" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88"
dependencies = [ dependencies = [
"ahash", "ahash",
"clap", "clap",
@@ -1380,9 +1380,9 @@ dependencies = [
[[package]] [[package]]
name = "ordered-float" name = "ordered-float"
version = "4.2.1" version = "4.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19ff2cf528c6c03d9ed653d6c4ce1dc0582dc4af309790ad92f07c1cd551b0be" checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6"
dependencies = [ dependencies = [
"num-traits", "num-traits",
] ]
@@ -1463,6 +1463,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"toml", "toml",
"zstd",
] ]
[[package]] [[package]]
@@ -2187,9 +2188,9 @@ dependencies = [
[[package]] [[package]]
name = "toml" name = "toml"
version = "0.8.16" version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c" checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
dependencies = [ dependencies = [
"serde", "serde",
"serde_spanned", "serde_spanned",
@@ -2199,18 +2200,18 @@ dependencies = [
[[package]] [[package]]
name = "toml_datetime" name = "toml_datetime"
version = "0.6.7" version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
dependencies = [ dependencies = [
"serde", "serde",
] ]
[[package]] [[package]]
name = "toml_edit" name = "toml_edit"
version = "0.22.17" version = "0.22.20"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16" checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"
dependencies = [ dependencies = [
"indexmap", "indexmap",
"serde", "serde",
@@ -2645,9 +2646,9 @@ checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8"
[[package]] [[package]]
name = "winnow" name = "winnow"
version = "0.6.13" version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]
@@ -2711,27 +2712,27 @@ checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
[[package]] [[package]]
name = "zstd" name = "zstd"
version = "0.13.0" version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9"
dependencies = [ dependencies = [
"zstd-safe", "zstd-safe",
] ]
[[package]] [[package]]
name = "zstd-safe" name = "zstd-safe"
version = "7.0.0" version = "7.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"
dependencies = [ dependencies = [
"zstd-sys", "zstd-sys",
] ]
[[package]] [[package]]
name = "zstd-sys" name = "zstd-sys"
version = "2.0.9+zstd.1.5.5" version = "2.0.13+zstd.1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
dependencies = [ dependencies = [
"cc", "cc",
"pkg-config", "pkg-config",

4
server/run.sh
View File

@@ -1,5 +1,7 @@
if cargo watch --help &> /dev/null; then if cargo watch --help &> /dev/null; then
cargo watch --no-vcs-ignores -w "./src" -w "./run.sh" -w "./in/datasets_len.txt" -x "run -r" TRIGGER="./in/datasets_len.txt"
echo "0" > $TRIGGER
cargo watch --no-vcs-ignores -w "./src" -w "./run.sh" -w "$TRIGGER" -x "run -r"
else else
cargo run -r cargo run -r
fi fi

20
server/src/handler.rs
View File

@@ -109,16 +109,16 @@ fn _file_handler(
let type_name = route.values_type.split("::").last().unwrap(); let type_name = route.values_type.split("::").last().unwrap();
let value = match type_name { let value = match type_name {
"u8" => typed_value_to_response::<u8>(kind, &route.file_path, chunk)?, "u8" => typed_value_to_response::<u8>(kind, &route, chunk)?,
"u16" => typed_value_to_response::<u16>(kind, &route.file_path, chunk)?, "u16" => typed_value_to_response::<u16>(kind, &route, chunk)?,
"u32" => typed_value_to_response::<u32>(kind, &route.file_path, chunk)?, "u32" => typed_value_to_response::<u32>(kind, &route, chunk)?,
"u64" => typed_value_to_response::<u64>(kind, &route.file_path, chunk)?, "u64" => typed_value_to_response::<u64>(kind, &route, chunk)?,
"usize" => typed_value_to_response::<usize>(kind, &route.file_path, chunk)?, "usize" => typed_value_to_response::<usize>(kind, &route, chunk)?,
"f32" => typed_value_to_response::<f32>(kind, &route.file_path, chunk)?, "f32" => typed_value_to_response::<f32>(kind, &route, chunk)?,
"f64" => typed_value_to_response::<f64>(kind, &route.file_path, chunk)?, "f64" => typed_value_to_response::<f64>(kind, &route, chunk)?,
"OHLC" => typed_value_to_response::<OHLC>(kind, &route.file_path, chunk)?, "OHLC" => typed_value_to_response::<OHLC>(kind, &route, chunk)?,
"Date" => typed_value_to_response::<Date>(kind, &route.file_path, chunk)?, "Date" => typed_value_to_response::<Date>(kind, &route, chunk)?,
"Height" => typed_value_to_response::<Height>(kind, &route.file_path, chunk)?, "Height" => typed_value_to_response::<Height>(kind, &route, chunk)?,
_ => panic!("Incompatible type: {type_name}"), _ => panic!("Incompatible type: {type_name}"),
}; };

16
server/src/imports.rs
View File

@@ -1,27 +1,25 @@
use std::fmt::Debug; use std::{fmt::Debug, path::Path};
use bincode::Decode; use bincode::Decode;
use parser::{Date, Serialization, SerializedBTreeMap, SerializedVec}; use parser::{Date, SerializedBTreeMap, SerializedVec};
use serde::{de::DeserializeOwned, Serialize}; use serde::{de::DeserializeOwned, Serialize};
pub fn import_map<T>(relative_path: &str) -> color_eyre::Result<SerializedBTreeMap<Date, T>> use crate::routes::Route;
pub fn import_map<T>(route: &Route) -> color_eyre::Result
where where
T: Serialize + Debug + DeserializeOwned + Decode, T: Serialize + Debug + DeserializeOwned + Decode,
{ {
Serialization::from_extension(relative_path.split('.').last().unwrap()).import(relative_path)
} }
pub fn import_vec<T>(relative_path: &str) -> color_eyre::Result<SerializedVec<T>> pub fn import_vec<T>(route: &Route) -> color_eyre::Result
where where
T: Serialize + Debug + DeserializeOwned + Decode, T: Serialize + Debug + DeserializeOwned + Decode,
{ {
Serialization::from_extension(relative_path.split('.').last().unwrap()).import(relative_path)
} }
pub fn import_value<T>(relative_path: &str) -> color_eyre::Result<T> pub fn import_value<T>(route: &Route) -> color_eyre::Result<T>
where where
T: Serialize + Debug + DeserializeOwned + Decode, T: Serialize + Debug + DeserializeOwned + Decode,
{ {
Serialization::from_extension(relative_path.split('.').last().unwrap())
.import::<T>(relative_path)
} }

5
server/src/main.rs
View File

@@ -1,7 +1,7 @@
use std::sync::Arc; use std::sync::Arc;
use axum::{extract::State, http::HeaderMap, response::Response, routing::get, serve, Router}; use axum::{extract::State, http::HeaderMap, response::Response, routing::get, serve, Router};
use parser::log; use parser::{log, reset_logs};
use reqwest::header::HOST; use reqwest::header::HOST;
use response::generic_to_reponse; use response::generic_to_reponse;
use routes::Routes; use routes::Routes;
@@ -12,7 +12,6 @@ use tower_http::compression::CompressionLayer;
mod chunk; mod chunk;
mod handler; mod handler;
mod headers; mod headers;
mod imports;
mod kind; mod kind;
mod paths; mod paths;
mod response; mod response;
@@ -36,6 +35,8 @@ pub struct AppState {
async fn main() -> color_eyre::Result<()> { async fn main() -> color_eyre::Result<()> {
color_eyre::install()?; color_eyre::install()?;
reset_logs();
let routes = Routes::build(); let routes = Routes::build();
routes.generate_dts_file(); routes.generate_dts_file();

27
server/src/response.rs
View File

@@ -1,15 +1,16 @@
use std::fmt::Debug; use std::{fmt::Debug, path::Path};
use axum::response::{IntoResponse, Json, Response}; use axum::response::{IntoResponse, Json, Response};
use bincode::Decode; use bincode::Decode;
use parser::{Date, SerializedBTreeMap, SerializedVec};
use serde::de::DeserializeOwned; use serde::de::DeserializeOwned;
use serde::Serialize; use serde::Serialize;
use crate::{ use crate::{
chunk::Chunk, chunk::Chunk,
headers::{add_cache_control_to_headers, add_cors_to_headers, add_json_type_to_headers}, headers::{add_cache_control_to_headers, add_cors_to_headers, add_json_type_to_headers},
imports::{import_map, import_value, import_vec},
kind::Kind, kind::Kind,
routes::Route,
}; };
#[derive(Serialize)] #[derive(Serialize)]
@@ -24,16 +25,30 @@ where
pub fn typed_value_to_response<T>( pub fn typed_value_to_response<T>(
kind: Kind, kind: Kind,
relative_path: &str, route: &Route,
chunk: Option<Chunk>, chunk: Option<Chunk>,
) -> color_eyre::Result<Response> ) -> color_eyre::Result<Response>
where where
T: Serialize + Debug + DeserializeOwned + Decode, T: Serialize + Debug + DeserializeOwned + Decode,
{ {
Ok(match kind { Ok(match kind {
Kind::Date => dataset_to_response(import_map::<T>(relative_path)?, chunk.unwrap()), Kind::Date => dataset_to_response(
Kind::Height => dataset_to_response(import_vec::<T>(relative_path)?, chunk.unwrap()), route
Kind::Last => value_to_response(import_value::<T>(relative_path)?), .serialization
.import::<SerializedBTreeMap<Date, T>>(Path::new(&route.file_path))?,
chunk.unwrap(),
),
Kind::Height => dataset_to_response(
route
.serialization
.import::<SerializedVec<T>>(Path::new(&route.file_path))?,
chunk.unwrap(),
),
Kind::Last => value_to_response(
route
.serialization
.import::<T>(Path::new(&route.file_path))?,
),
}) })
} }

24
server/src/routes.rs
View File

@@ -1,6 +1,7 @@
use std::{ use std::{
collections::{BTreeMap, HashMap}, collections::{BTreeMap, HashMap},
fs, fs,
path::Path,
}; };
use derive_deref::{Deref, DerefMut}; use derive_deref::{Deref, DerefMut};
@@ -26,31 +27,22 @@ const APP_TYPES_PATH: &str = "../app/src/types";
impl Routes { impl Routes {
pub fn build() -> Self { pub fn build() -> Self {
let path_to_type: BTreeMap<String, String> = let path_to_type: BTreeMap<String, String> =
Json::import(&format!("{INPUTS_PATH}/disk_path_to_type.json")).unwrap(); Json::import(Path::new(&format!("{INPUTS_PATH}/disk_path_to_type.json"))).unwrap();
let mut routes = Routes::default(); let mut routes = Routes::default();
path_to_type.into_iter().for_each(|(key, value)| { path_to_type.into_iter().for_each(|(key, value)| {
let mut split_key = key.split('/').collect_vec(); let mut split_key = key.split('/').collect_vec();
let last = split_key.pop().unwrap().to_owned();
let mut split_last = split_key.pop().unwrap().split('.').rev().collect_vec();
let last = split_last.pop().unwrap().to_owned();
let mut skip = 2; let mut skip = 2;
let serialization = split_last.pop().map_or_else( let mut serialization = Serialization::Binary;
|| {
if *split_key.get(1).unwrap() == "price" {
skip = 1;
Serialization::Json if *split_key.get(1).unwrap() == "price" {
} else { skip = 1;
Serialization::Binary serialization = Serialization::Json;
} }
},
Serialization::from_extension,
);
let split_key = split_key.iter().skip(skip).collect_vec(); let split_key = split_key.iter().skip(skip).collect_vec();