diff --git a/CHANGELOG.md b/CHANGELOG.md index d0706c448..a3fbf27f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,15 @@ ### Parser -- Change the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind` -- Use the Bitcoin RPC server for various calls instead of running commands and then parsing the JSON from the output -- Updated the config, run with `-h` to see +- Changed the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind` +- Added datasets compression thanks to [zstd](https://crates.io/crates/zstd) to reduce disk usage +- Use the Bitcoin RPC server for various calls instead of running cli commands and then parsing the JSON from the output +- Updated the config, run with `-h` to see possible args +- Moved outputs from `/target/outputs` to `/out` to allow to run commands like `cargo clean` without side effects + +### Server + +- Updated the code to support compressed binaries ## v. 0.3.0 | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26 diff --git a/README.md b/README.md index 2b9d08319..91f927a16 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,12 @@ To promote even more transparency and trust in the network, this project is comm This project is in a very early stage. The web app will have bugs, the API might break and the data can definitely to be false or slightly false. +## Donations + +The project is a lot of work and being worked on full-time. It doesn't have any ads and solely relies on donations. If you find this project useful, any sat would really help make it even better and would be really appreciated. + +You can donate on the project's [Geyser Fund](https://geyser.fund/project/satonomics/). + ## Instances Web App: @@ -88,7 +94,7 @@ _Maybe_: Aka: Previous iterations -The initial idea was totally different yet morphed over time into what it is today: a fully FOSS self-hostable on-chain data generator +The initial idea was totally different yet morphed over time into what it is today: a fully FOSS self-hostable on-chain data generator. - https://github.com/drgarlic/satonomics - https://github.com/drgarlic/satonomics-parser diff --git a/app/src/app/components/background.tsx b/app/src/app/components/background.tsx index b2fac03c7..a4c701d4a 100644 --- a/app/src/app/components/background.tsx +++ b/app/src/app/components/background.tsx @@ -42,7 +42,8 @@ const texts = [ "absolute scarcity", "time is scarce", "ride or die", - "cyberpunk", + "cypherpunk", + "we like the coin", ]; export function Background({ diff --git a/app/src/app/components/strip/components/anchorGit.tsx b/app/src/app/components/strip/components/anchorGit.tsx index f96cd9b38..643646457 100644 --- a/app/src/app/components/strip/components/anchorGit.tsx +++ b/app/src/app/components/strip/components/anchorGit.tsx @@ -5,7 +5,7 @@ export function AnchorGit() { IconTablerBrandGithubFilled} - href="https://github.com/satonomics-org/satonomics" + href="https://github.com/satonomics-org" /> ); } diff --git a/parser/.gitignore b/parser/.gitignore index 47cec896f..fc8724640 100644 --- a/parser/.gitignore +++ b/parser/.gitignore @@ -19,3 +19,4 @@ benches parser.log config.toml *\ copy* +/.log diff --git a/parser/Cargo.lock b/parser/Cargo.lock index 372620352..f30b0ec99 100644 --- a/parser/Cargo.lock +++ b/parser/Cargo.lock @@ -48,7 +48,7 @@ checksum = "fe233a377643e0fc1a56421d7c90acdec45c291b30345eb9f08e8d0ddce5a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.72", ] [[package]] @@ -289,9 +289,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bumpalo" @@ -301,9 +301,9 @@ checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" [[package]] name = "bytemuck" -version = "1.16.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" +checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" [[package]] name = "byteorder" @@ -322,6 +322,11 @@ name = "cc" version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] [[package]] name = "cfg-if" @@ -341,14 +346,14 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.4", + "windows-targets 0.52.6", ] [[package]] name = "clap" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" +checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc" dependencies = [ "clap_builder", "clap_derive", @@ -356,9 +361,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" +checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99" dependencies = [ "anstream", "anstyle", @@ -368,21 +373,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.72", ] [[package]] name = "clap_lex" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" [[package]] name = "color-eyre" @@ -448,9 +453,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ "crossbeam-utils", ] @@ -485,9 +490,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "ctor" @@ -532,7 +537,7 @@ checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.72", ] [[package]] @@ -693,9 +698,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -729,9 +734,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "heck" @@ -924,9 +929,9 @@ dependencies = [ [[package]] name = "inferno" -version = "0.11.20" +version = "0.11.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c77a3ae7d4761b9c64d2c030f70746ceb8cfba32dce0325a56792e0a4816c31" +checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", "clap", @@ -992,6 +997,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.69" @@ -1043,15 +1057,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" @@ -1140,9 +1154,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] @@ -1168,7 +1182,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "foreign-types", "libc", @@ -1185,7 +1199,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.72", ] [[package]] @@ -1208,9 +1222,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.2.1" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ff2cf528c6c03d9ed653d6c4ce1dc0582dc4af309790ad92f07c1cd551b0be" +checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6" dependencies = [ "num-traits", ] @@ -1256,7 +1270,7 @@ dependencies = [ "libc", "redox_syscall 0.5.2", "smallvec", - "windows-targets 0.52.4", + "windows-targets 0.52.6", ] [[package]] @@ -1281,6 +1295,7 @@ dependencies = [ "serde", "serde_json", "toml", + "zstd", ] [[package]] @@ -1306,7 +1321,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.72", ] [[package]] @@ -1338,9 +1353,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -1428,7 +1443,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -1477,9 +1492,9 @@ dependencies = [ [[package]] name = "rgb" -version = "0.8.44" +version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aee83dc281d5a3200d37b299acd13b81066ea126a7f16f0eae70fc9aed241d9" +checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8" dependencies = [ "bytemuck", ] @@ -1511,7 +1526,7 @@ version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -1661,7 +1676,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.72", ] [[package]] @@ -1768,9 +1783,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.66" +version = "2.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" dependencies = [ "proc-macro2", "quote", @@ -1833,7 +1848,7 @@ checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.72", ] [[package]] @@ -1913,9 +1928,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ "serde", "serde_spanned", @@ -1925,18 +1940,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.17" +version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16" +checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ "indexmap", "serde", @@ -2085,9 +2100,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "virtue" @@ -2131,7 +2146,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.72", "wasm-bindgen-shared", ] @@ -2165,7 +2180,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.72", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2214,7 +2229,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.6", ] [[package]] @@ -2232,7 +2247,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.6", ] [[package]] @@ -2252,17 +2267,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.4", - "windows_aarch64_msvc 0.52.4", - "windows_i686_gnu 0.52.4", - "windows_i686_msvc 0.52.4", - "windows_x86_64_gnu 0.52.4", - "windows_x86_64_gnullvm 0.52.4", - "windows_x86_64_msvc 0.52.4", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -2273,9 +2289,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -2285,9 +2301,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -2297,9 +2313,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -2309,9 +2331,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -2321,9 +2343,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -2333,9 +2355,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -2345,15 +2367,15 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.4" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.13" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" dependencies = [ "memchr", ] @@ -2395,7 +2417,7 @@ checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.72", ] [[package]] @@ -2406,7 +2428,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.72", ] [[package]] @@ -2414,3 +2436,31 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/parser/Cargo.toml b/parser/Cargo.toml index f73b137d0..2df89367b 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -14,16 +14,17 @@ bincode = { git = "https://github.com/bincode-org/bincode.git" } biter = "0.1.0" bitcoin_hashes = { version = "0.14.0" } chrono = { version = "0.4.38", features = ["serde"] } -clap = { version = "4.5.11", features = ["derive"] } +clap = { version = "4.5.13", features = ["derive"] } color-eyre = "0.6.3" derive_deref = "1.1.1" -inferno = "0.11.20" +inferno = "0.11.21" itertools = "0.13.0" memory-stats = "1.2.0" -ordered-float = "4.2.1" +ordered-float = "4.2.2" rayon = "1.10.0" reqwest = { version = "0.12.5", features = ["blocking", "json"] } sanakirja = "1.4.2" serde = { version = "1.0.204", features = ["derive"] } -serde_json = "1.0.120" -toml = "0.8.16" +serde_json = "1.0.122" +toml = "0.8.19" +zstd = "0.13.2" diff --git a/parser/src/actions/iter_blocks.rs b/parser/src/actions/iter_blocks.rs index 8ada00963..0068580cc 100644 --- a/parser/src/actions/iter_blocks.rs +++ b/parser/src/actions/iter_blocks.rs @@ -11,6 +11,7 @@ use crate::{ create_rpc, databases::Databases, datasets::{AllDatasets, ComputeData}, + io::OUTPUTS_FOLDER_PATH, states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates}, structs::{Date, DateData, MapKey}, utils::{generate_allocation_files, log, time}, @@ -22,8 +23,6 @@ pub fn iter_blocks( rpc: &biter::bitcoincore_rpc::Client, approx_block_count: usize, ) -> color_eyre::Result<()> { - let export_dir = "./target/outputs"; - let should_insert = true; let should_export = true; let study_ram_usage = false; @@ -56,7 +55,7 @@ pub fn iter_blocks( let block_receiver = biter::new( config.datadir.as_ref().unwrap(), - export_dir, + OUTPUTS_FOLDER_PATH, Some(height.to_usize()), None, create_rpc(config).unwrap(), diff --git a/parser/src/databases/metadata.rs b/parser/src/databases/metadata.rs index f1d41de5e..6cf56ceb0 100644 --- a/parser/src/databases/metadata.rs +++ b/parser/src/databases/metadata.rs @@ -1,14 +1,16 @@ use allocative::Allocative; use bincode::{Decode, Encode}; +use serde::{Deserialize, Serialize}; use std::{ fmt::Debug, fs, io, ops::{Deref, DerefMut}, + path::Path, }; use crate::{ - io::Binary, structs::{Counter, Date, Height}, + Serialization, }; #[derive(Default, Debug, Encode, Decode, Allocative)] @@ -73,7 +75,7 @@ impl Metadata { } } -#[derive(Default, Debug, Encode, Decode, Allocative)] +#[derive(Default, Debug, Encode, Decode, Serialize, Deserialize, Allocative)] pub struct MetadataData { pub serial: usize, pub len: Counter, @@ -88,17 +90,17 @@ impl MetadataData { fn full_path(folder_path: &str) -> String { let name = Self::name(); - format!("{folder_path}/{name}.bin") + format!("{folder_path}/{name}") } pub fn import(path: &str) -> color_eyre::Result { fs::create_dir_all(path)?; - Binary::import(&Self::full_path(path)) + Serialization::Binary.import(Path::new(&Self::full_path(path))) } pub fn export(&self, path: &str) -> color_eyre::Result<()> { - Binary::export(&Self::full_path(path), self) + Serialization::Binary.export(Path::new(&Self::full_path(path)), self) } pub fn reset(&mut self, path: &str) -> color_eyre::Result<(), io::Error> { diff --git a/parser/src/datasets/mod.rs b/parser/src/datasets/mod.rs index eaa8f5f1b..171aa5087 100644 --- a/parser/src/datasets/mod.rs +++ b/parser/src/datasets/mod.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, fs, ops::RangeInclusive}; +use std::{collections::BTreeMap, fs, ops::RangeInclusive, path::Path}; use allocative::Allocative; @@ -282,7 +282,7 @@ impl AllDatasets { fs::create_dir_all(server_inputs_path)?; Json::export( - &format!("{server_inputs_path}/disk_path_to_type.json"), + Path::new(&format!("{server_inputs_path}/disk_path_to_type.json")), &path_to_type, )?; diff --git a/parser/src/io/binary.rs b/parser/src/io/binary.rs index abef7e160..9a8acab2d 100644 --- a/parser/src/io/binary.rs +++ b/parser/src/io/binary.rs @@ -1,15 +1,39 @@ use std::{ fmt::Debug, - fs::File, - io::{BufReader, BufWriter}, + fs::{self, File}, + io::{BufReader, BufWriter, Cursor}, + path::Path, }; -use bincode::{config, decode_from_std_read, encode_into_std_write, Decode, Encode}; +use bincode::{ + config, decode_from_slice, decode_from_std_read, encode_into_std_write, Decode, Encode, +}; +use zstd::decode_all; + +const ZST_EXTENSION: &str = "zst"; + +pub const BIN_EXTENSION: &str = "bin"; +pub const COMPRESSED_BIN_EXTENSION: &str = "bin.zst"; + +enum BinaryType { + Raw, + Compressed, +} pub struct Binary; impl Binary { - pub fn import(path: &str) -> color_eyre::Result + pub fn import(path: &Path) -> color_eyre::Result + where + T: Decode, + { + match Self::type_from_path(path) { + BinaryType::Compressed => Self::import_compressed(path), + BinaryType::Raw => Self::import_raw(path), + } + } + + fn import_raw(path: &Path) -> color_eyre::Result where T: Decode, { @@ -24,7 +48,34 @@ impl Binary { Ok(decoded) } - pub fn export(path: &str, value: &T) -> color_eyre::Result<()> + fn import_compressed(path: &Path) -> color_eyre::Result + where + T: Decode, + { + let file = File::open(path).unwrap(); + + let reader = BufReader::new(file); + + let decompressed = decode_all(reader).unwrap(); + + let config = config::standard(); + + let decoded = decode_from_slice::(&decompressed, config).unwrap().0; + + Ok(decoded) + } + + pub fn export(path: &Path, value: &T) -> color_eyre::Result<()> + where + T: Debug + Encode, + { + match Self::type_from_path(path) { + BinaryType::Compressed => Self::export_compressed(path, value), + BinaryType::Raw => Self::export_raw(path, value), + } + } + + fn export_raw(path: &Path, value: &T) -> color_eyre::Result<()> where T: Debug + Encode, { @@ -40,4 +91,47 @@ impl Binary { Ok(()) } + + fn export_compressed(path: &Path, value: &T) -> color_eyre::Result<()> + where + T: Debug + Encode, + { + let config = config::standard(); + + let encoded = bincode::encode_to_vec(value, config).unwrap(); + + let cursor = Cursor::new(encoded); + + let compressed = zstd::encode_all(cursor, 0).unwrap(); + + fs::write(path, compressed).unwrap(); + + Ok(()) + } + + pub fn has_correct_extension(path: &Path) -> bool { + let path = path.to_str().unwrap(); + path.ends_with(BIN_EXTENSION) || path.ends_with(COMPRESSED_BIN_EXTENSION) + } + + fn type_from_path(path: &Path) -> BinaryType { + let extension = path.extension(); + + if extension.is_none() { + panic!("Should have extension"); + } + + if !Self::has_correct_extension(path) { + dbg!(path); + panic!("Wrong extension") + } + + let extension = extension.unwrap(); + + if extension == ZST_EXTENSION { + BinaryType::Compressed + } else { + BinaryType::Raw + } + } } diff --git a/parser/src/io/consts.rs b/parser/src/io/consts.rs index 62a50ceb6..750f7edb0 100644 --- a/parser/src/io/consts.rs +++ b/parser/src/io/consts.rs @@ -1,2 +1,2 @@ pub const INPUTS_FOLDER_PATH: &str = "./in"; -pub const OUTPUTS_FOLDER_PATH: &str = "./target/outputs"; +pub const OUTPUTS_FOLDER_PATH: &str = "./out"; diff --git a/parser/src/io/json.rs b/parser/src/io/json.rs index 9b9e94f73..6d81fe580 100644 --- a/parser/src/io/json.rs +++ b/parser/src/io/json.rs @@ -1,17 +1,25 @@ use std::{ fs::File, io::{BufReader, BufWriter}, + path::Path, }; use serde::{de::DeserializeOwned, Serialize}; pub struct Json; +pub const JSON_EXTENSION: &str = "json"; +pub const HAR_EXTENSION: &str = "har"; + impl Json { - pub fn import(path: &str) -> color_eyre::Result + pub fn import(path: &Path) -> color_eyre::Result where T: DeserializeOwned, { + if !Self::has_correct_extension(path) { + panic!("Wrong extension"); + } + let file = File::open(path)?; let reader = BufReader::new(file); @@ -19,10 +27,15 @@ impl Json { Ok(serde_json::from_reader(reader)?) } - pub fn export(path: &str, value: &T) -> color_eyre::Result<()> + pub fn export(path: &Path, value: &T) -> color_eyre::Result<()> where T: Serialize, { + if !Self::has_correct_extension(path) { + dbg!(path); + panic!("Wrong extension"); + } + let file = File::create(path).unwrap_or_else(|_| { dbg!(&path); panic!("No such file or directory") @@ -34,4 +47,10 @@ impl Json { Ok(()) } + + #[inline(always)] + pub fn has_correct_extension(path: &Path) -> bool { + let path = path.to_str().unwrap(); + path.ends_with(JSON_EXTENSION) || path.ends_with(HAR_EXTENSION) + } } diff --git a/parser/src/io/serialization.rs b/parser/src/io/serialization.rs index 57c2d7434..8b2e29bb2 100644 --- a/parser/src/io/serialization.rs +++ b/parser/src/io/serialization.rs @@ -1,4 +1,4 @@ -use std::fmt::Debug; +use std::{fmt::Debug, fs, path::Path}; use allocative::Allocative; use bincode::{Decode, Encode}; @@ -6,6 +6,8 @@ use serde::{de::DeserializeOwned, Serialize}; use crate::io::{Binary, Json}; +use super::{BIN_EXTENSION, COMPRESSED_BIN_EXTENSION, HAR_EXTENSION, JSON_EXTENSION}; + #[derive(PartialEq, PartialOrd, Ord, Eq, Debug, Clone, Copy, Default, Allocative)] pub enum Serialization { #[default] @@ -14,42 +16,105 @@ pub enum Serialization { } impl Serialization { - pub fn to_extension(&self) -> &str { + pub fn is_serializable(&self, path: &Path) -> bool { + let path = path.to_str().unwrap(); match self { - Self::Binary => "bin", - Self::Json => "json", + Self::Binary => { + path.ends_with(BIN_EXTENSION) || path.ends_with(COMPRESSED_BIN_EXTENSION) + } + Self::Json => path.ends_with(JSON_EXTENSION) || path.ends_with(HAR_EXTENSION), } } - pub fn from_extension(extension: &str) -> Self { - match extension { - "bin" => Self::Binary, - "json" => Self::Json, - _ => panic!("Extension \"{extension}\" isn't supported"), + pub fn from_path(path: &Path) -> Self { + let path = path.to_str().unwrap(); + if path.ends_with(BIN_EXTENSION) || path.ends_with(COMPRESSED_BIN_EXTENSION) { + Self::Binary + } else if path.ends_with(JSON_EXTENSION) || path.ends_with(HAR_EXTENSION) { + Self::Json + } else { + panic!("Extension \"{path}\" isn't supported") } } - pub fn append_extension(&self, path: &str) -> String { - format!("{path}.{}", self.to_extension()) - } - - pub fn import(&self, path: &str) -> color_eyre::Result + pub fn import(&self, path: &Path) -> color_eyre::Result where T: Debug + DeserializeOwned + Decode, { match self { - Serialization::Binary => Binary::import(path), - Serialization::Json => Json::import(path), + Serialization::Binary => { + if self.is_serializable(path) { + Binary::import(path) + } else { + let path = path.to_str().unwrap(); + let bin_path_str = format!("{path}.{BIN_EXTENSION}"); + let bin_path = Path::new(&bin_path_str); + + if bin_path.exists() { + return Binary::import(bin_path); + } + + let compressed_bin_path_str = format!("{path}.{COMPRESSED_BIN_EXTENSION}"); + let compressed_bin_path = Path::new(&compressed_bin_path_str); + + if compressed_bin_path.exists() { + return Binary::import(compressed_bin_path); + } + + panic!("Wrong path") + } + } + Serialization::Json => { + if self.is_serializable(path) { + Json::import(path) + } else { + let path = path.to_str().unwrap(); + let json_path_str = format!("{path}.{JSON_EXTENSION}"); + let json_path = Path::new(&json_path_str); + + if json_path.exists() { + return Json::import(json_path); + } + + panic!("Wrong path") + } + } } } - pub fn export(&self, path: &str, value: &T) -> color_eyre::Result<()> + pub fn export(&self, path: &Path, value: &T) -> color_eyre::Result<()> where T: Debug + Serialize + Encode, { match self { - Serialization::Binary => Binary::export(path, value), - Serialization::Json => Json::export(path, value), + Serialization::Binary => { + if self.is_serializable(path) { + Binary::export(path, value) + } else { + let path = path.to_str().unwrap(); + + let res = Binary::export( + Path::new(&format!("{}.{COMPRESSED_BIN_EXTENSION}", path,)), + value, + ); + + if res.is_ok() { + let _ = fs::remove_file(Path::new(&format!("{}.{BIN_EXTENSION}", path))); + } + + res + } + } + Serialization::Json => { + if self.is_serializable(path) { + Json::export(path, value) + } else { + Json::export( + Path::new(&format!("{}.{JSON_EXTENSION}", path.to_str().unwrap())), + value, + ) + } + } } } } diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 8fcc22535..73826726b 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -15,5 +15,5 @@ pub use crate::{ Config, Date, DateMap, Height, HeightMap, MapChunkId, SerializedBTreeMap, SerializedVec, HEIGHT_MAP_CHUNK_SIZE, }, - utils::{create_rpc, log}, + utils::{create_rpc, log, reset_logs}, }; diff --git a/parser/src/main.rs b/parser/src/main.rs index 52d271bc5..5e925bde3 100644 --- a/parser/src/main.rs +++ b/parser/src/main.rs @@ -1,11 +1,13 @@ use std::{thread::sleep, time::Duration}; use biter::bitcoincore_rpc::RpcApi; -use parser::{create_rpc, iter_blocks, log, Config}; +use parser::{create_rpc, iter_blocks, log, reset_logs, Config}; fn main() -> color_eyre::Result<()> { color_eyre::install()?; + reset_logs(); + let config = Config::import(); let rpc = create_rpc(&config).unwrap(); diff --git a/parser/src/price/binance.rs b/parser/src/price/binance.rs index 4ef3e9e80..4dbfe6297 100644 --- a/parser/src/price/binance.rs +++ b/parser/src/price/binance.rs @@ -23,8 +23,7 @@ impl Binance { let path_binance_har = Path::new(INPUTS_FOLDER_PATH).join("binance.har"); - let json: BTreeMap = - Json::import(path_binance_har.to_str().unwrap()).unwrap_or_default(); + let json: BTreeMap = Json::import(&path_binance_har).unwrap_or_default(); Ok(json .get("log") diff --git a/parser/src/states/_trait.rs b/parser/src/states/_trait.rs index 11ee4b5b8..2b82fbd94 100644 --- a/parser/src/states/_trait.rs +++ b/parser/src/states/_trait.rs @@ -1,13 +1,14 @@ -use std::{fmt::Debug, fs, io}; +use std::{fmt::Debug, fs, io, path::Path}; use bincode::{Decode, Encode}; +use serde::{de::DeserializeOwned, Serialize}; -use crate::io::{Binary, OUTPUTS_FOLDER_PATH}; +use crate::{io::OUTPUTS_FOLDER_PATH, Serialization}; // https://github.com/djkoloski/rust_serialization_benchmark pub trait AnyState where - Self: Debug + Encode + Decode, + Self: Debug + Encode + Decode + Serialize + DeserializeOwned, { fn name<'a>() -> &'a str; @@ -24,7 +25,7 @@ where let folder_path = Self::folder_path(); - format!("{folder_path}/{name}.bin") + format!("{folder_path}/{name}") } fn reset(&mut self) -> color_eyre::Result<(), io::Error> { @@ -36,11 +37,11 @@ where fn import() -> color_eyre::Result { Self::create_dir_all()?; - Binary::import(&Self::full_path()) + Serialization::Binary.import(Path::new(&Self::full_path())) } fn export(&self) -> color_eyre::Result<()> { - Binary::export(&Self::full_path(), self) + Serialization::Binary.export(Path::new(&Self::full_path()), self) } fn clear(&mut self); diff --git a/parser/src/states/counters.rs b/parser/src/states/counters.rs index 5494d3fa9..4e369e376 100644 --- a/parser/src/states/counters.rs +++ b/parser/src/states/counters.rs @@ -1,11 +1,12 @@ use allocative::Allocative; use bincode::{Decode, Encode}; +use serde::{Deserialize, Serialize}; use crate::structs::Counter; use super::AnyState; -#[derive(Default, Debug, Encode, Decode, Allocative)] +#[derive(Default, Debug, Encode, Decode, Serialize, Deserialize, Allocative)] pub struct Counters { pub op_return_addresses: Counter, pub push_only_addresses: Counter, diff --git a/parser/src/states/date_data_vec.rs b/parser/src/states/date_data_vec.rs index e0ca14561..7e939f443 100644 --- a/parser/src/states/date_data_vec.rs +++ b/parser/src/states/date_data_vec.rs @@ -1,12 +1,13 @@ use allocative::Allocative; use bincode::{Decode, Encode}; use derive_deref::{Deref, DerefMut}; +use serde::{Deserialize, Serialize}; use crate::structs::{BlockData, BlockPath, DateData}; use super::AnyState; -#[derive(Default, Deref, DerefMut, Debug, Encode, Decode, Allocative)] +#[derive(Default, Deref, DerefMut, Debug, Serialize, Deserialize, Encode, Decode, Allocative)] pub struct DateDataVec(Vec); impl DateDataVec { diff --git a/parser/src/structs/block_data.rs b/parser/src/structs/block_data.rs index 00b96bb3b..e19d26f1b 100644 --- a/parser/src/structs/block_data.rs +++ b/parser/src/structs/block_data.rs @@ -1,9 +1,10 @@ use allocative::Allocative; use bincode::{Decode, Encode}; +use serde::{Deserialize, Serialize}; use super::{Amount, Height, Price}; -#[derive(Debug, Encode, Decode, Allocative)] +#[derive(Debug, Serialize, Deserialize, Encode, Decode, Allocative)] pub struct BlockData { pub height: Height, pub price: Price, diff --git a/parser/src/structs/counter.rs b/parser/src/structs/counter.rs index f45f99312..67acd0168 100644 --- a/parser/src/structs/counter.rs +++ b/parser/src/structs/counter.rs @@ -1,8 +1,11 @@ use allocative::Allocative; use bincode::{Decode, Encode}; use derive_deref::{Deref, DerefMut}; +use serde::{Deserialize, Serialize}; -#[derive(Debug, Deref, DerefMut, Default, Clone, Copy, Encode, Decode, Allocative)] +#[derive( + Debug, Deref, DerefMut, Default, Clone, Copy, Encode, Decode, Serialize, Deserialize, Allocative, +)] pub struct Counter(u32); impl Counter { diff --git a/parser/src/structs/date_data.rs b/parser/src/structs/date_data.rs index 521ff6fe3..b92cec89f 100644 --- a/parser/src/structs/date_data.rs +++ b/parser/src/structs/date_data.rs @@ -1,9 +1,10 @@ use allocative::Allocative; use bincode::{Decode, Encode}; +use serde::{Deserialize, Serialize}; use super::{BlockData, BlockPath, Date}; -#[derive(Debug, Encode, Decode, Allocative)] +#[derive(Debug, Serialize, Deserialize, Encode, Decode, Allocative)] pub struct DateData { pub date: Date, pub blocks: Vec, diff --git a/parser/src/structs/date_map_chunk_id.rs b/parser/src/structs/date_map_chunk_id.rs index 7592754a7..4fe77d300 100644 --- a/parser/src/structs/date_map_chunk_id.rs +++ b/parser/src/structs/date_map_chunk_id.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use allocative::Allocative; use chrono::Datelike; @@ -19,8 +21,18 @@ impl MapChunkId for DateMapChunkId { self.0.to_string() } - fn from_name(name: &str) -> Self { - Self(name.parse::().unwrap()) + fn from_path(path: &Path) -> Self { + Self( + path.file_name() + .unwrap() + .to_str() + .unwrap() + .split(".") + .next() + .unwrap() + .parse::() + .unwrap(), + ) } fn to_usize(self) -> usize { diff --git a/parser/src/structs/generic_map.rs b/parser/src/structs/generic_map.rs index 10716952b..9e2efda15 100644 --- a/parser/src/structs/generic_map.rs +++ b/parser/src/structs/generic_map.rs @@ -63,7 +63,7 @@ where Self: Ord + Debug + Copy + Clone, { fn to_name(&self) -> String; - fn from_name(name: &str) -> Self; + fn from_path(path: &Path) -> Self; fn to_usize(self) -> usize; fn from_usize(id: usize) -> Self; } @@ -124,7 +124,7 @@ where let path_last = { if export_last { - Some(serialization.append_extension(&format!("{path}/last"))) + Some(format!("{path}/last")) } else { None } @@ -188,22 +188,16 @@ where fs::read_dir(path) .unwrap() .map(|entry| entry.unwrap().path()) - .filter(|path| { - let extension = path.extension().unwrap().to_str().unwrap(); - - path.is_file() && extension == serialization.to_extension() - }) + .filter(|path| serialization.is_serializable(path)) .map(|path| { - let chunk_id = ChunkId::from_name(path.file_stem().unwrap().to_str().unwrap()); - + let chunk_id = ChunkId::from_path(&path); (chunk_id, path) }) .collect() } fn import(&self, path: &Path) -> color_eyre::Result { - self.serialization - .import::(path.to_str().unwrap()) + self.serialization.import::(path) } pub fn insert(&mut self, key: Key, value: Value) -> Value { @@ -309,10 +303,7 @@ where panic!(); }); - let serialized = self - .serialization - .import::(path.to_str().unwrap()) - .unwrap(); + let serialized = self.serialization.import::(path).unwrap(); self.imported.insert(*chunk_id, serialized); } @@ -334,23 +325,18 @@ where unreachable!() } - let path = self.serialization.append_extension(&format!( - "{}/{}", - self.path_all, - chunk_id.to_name() - )); - let serialized = self.imported.get(chunk_id).unwrap_or_else(|| { dbg!(&self.path_all, chunk_id, &self.imported); panic!(); }); - self.serialization.export(&path, serialized)?; + let path = format!("{}/{}", self.path_all, chunk_id.to_name()); + self.serialization.export(Path::new(&path), serialized)?; if index == len - 1 { if let Some(path_last) = self.path_last.as_ref() { self.serialization - .export(path_last, serialized.last().unwrap())?; + .export(Path::new(path_last), serialized.last().unwrap())?; } } diff --git a/parser/src/structs/height_map_chunk_id.rs b/parser/src/structs/height_map_chunk_id.rs index c0bb9c536..37d34bc36 100644 --- a/parser/src/structs/height_map_chunk_id.rs +++ b/parser/src/structs/height_map_chunk_id.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use allocative::Allocative; use derive_deref::{Deref, DerefMut}; @@ -26,9 +28,20 @@ impl MapChunkId for HeightMapChunkId { format!("{start}..{end}") } - fn from_name(name: &str) -> Self { + fn from_path(path: &Path) -> Self { Self(Height::new( - name.split("..").next().unwrap().parse::().unwrap(), + path.file_name() + .unwrap() + .to_str() + .unwrap() + .split("..") + .next() + .unwrap() + .parse::() + .unwrap_or_else(|_| { + dbg!(path); + panic!() + }), )) } diff --git a/parser/src/structs/price.rs b/parser/src/structs/price.rs index 6d4e64eec..4f1400d99 100644 --- a/parser/src/structs/price.rs +++ b/parser/src/structs/price.rs @@ -2,11 +2,24 @@ use std::ops::{Add, AddAssign, Div, Mul, Sub, SubAssign}; use allocative::Allocative; use bincode::{Decode, Encode}; +use serde::{Deserialize, Serialize}; use super::Amount; #[derive( - Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Allocative, + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + Encode, + Decode, + Allocative, )] pub struct Price(u64); diff --git a/parser/src/utils/log.rs b/parser/src/utils/log.rs index c9695d5bc..d6593f75f 100644 --- a/parser/src/utils/log.rs +++ b/parser/src/utils/log.rs @@ -1,8 +1,17 @@ -use std::{fs::OpenOptions, io::Write}; +use std::{ + fs::{self, OpenOptions}, + io::Write, +}; use chrono::Local; use color_eyre::owo_colors::OwoColorize; +const LOG_PATH: &str = "./.log"; + +pub fn reset_logs() { + let _ = fs::remove_file(LOG_PATH); +} + #[inline(always)] pub fn log(str: &str) { let date_time = format!("{}", Local::now().format("%Y-%m-%d %H:%M:%S -")); @@ -13,7 +22,7 @@ pub fn log(str: &str) { let mut file = OpenOptions::new() .create(true) .append(true) - .open("./parser.log") + .open(LOG_PATH) .unwrap(); if let Err(e) = writeln!(file, "{} {}", date_time, line) { diff --git a/server/.gitignore b/server/.gitignore index 9788728f4..3bd2c3517 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -1,4 +1,4 @@ /target .DS_Store -/parser.log +/.log /in diff --git a/server/Cargo.lock b/server/Cargo.lock index 6101727b9..d2b6342af 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -477,9 +477,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" +checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc" dependencies = [ "clap_builder", "clap_derive", @@ -487,9 +487,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" +checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99" dependencies = [ "anstream", "anstyle", @@ -499,9 +499,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" dependencies = [ "heck", "proc-macro2", @@ -1081,9 +1081,9 @@ dependencies = [ [[package]] name = "inferno" -version = "0.11.20" +version = "0.11.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c77a3ae7d4761b9c64d2c030f70746ceb8cfba32dce0325a56792e0a4816c31" +checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", "clap", @@ -1380,9 +1380,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.2.1" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ff2cf528c6c03d9ed653d6c4ce1dc0582dc4af309790ad92f07c1cd551b0be" +checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6" dependencies = [ "num-traits", ] @@ -1463,6 +1463,7 @@ dependencies = [ "serde", "serde_json", "toml", + "zstd", ] [[package]] @@ -2187,9 +2188,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ "serde", "serde_spanned", @@ -2199,18 +2200,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.17" +version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16" +checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ "indexmap", "serde", @@ -2645,9 +2646,9 @@ checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] name = "winnow" -version = "0.6.13" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" dependencies = [ "memchr", ] @@ -2711,27 +2712,27 @@ checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zstd" -version = "0.13.0" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.0.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/server/run.sh b/server/run.sh index 754d9c1b1..b63e33bd3 100755 --- a/server/run.sh +++ b/server/run.sh @@ -1,5 +1,7 @@ if cargo watch --help &> /dev/null; then - cargo watch --no-vcs-ignores -w "./src" -w "./run.sh" -w "./in/datasets_len.txt" -x "run -r" + TRIGGER="./in/datasets_len.txt" + echo "0" > $TRIGGER + cargo watch --no-vcs-ignores -w "./src" -w "./run.sh" -w "$TRIGGER" -x "run -r" else cargo run -r fi diff --git a/server/src/handler.rs b/server/src/handler.rs index 32ad39dfb..60ef1927f 100644 --- a/server/src/handler.rs +++ b/server/src/handler.rs @@ -109,16 +109,16 @@ fn _file_handler( let type_name = route.values_type.split("::").last().unwrap(); let value = match type_name { - "u8" => typed_value_to_response::(kind, &route.file_path, chunk)?, - "u16" => typed_value_to_response::(kind, &route.file_path, chunk)?, - "u32" => typed_value_to_response::(kind, &route.file_path, chunk)?, - "u64" => typed_value_to_response::(kind, &route.file_path, chunk)?, - "usize" => typed_value_to_response::(kind, &route.file_path, chunk)?, - "f32" => typed_value_to_response::(kind, &route.file_path, chunk)?, - "f64" => typed_value_to_response::(kind, &route.file_path, chunk)?, - "OHLC" => typed_value_to_response::(kind, &route.file_path, chunk)?, - "Date" => typed_value_to_response::(kind, &route.file_path, chunk)?, - "Height" => typed_value_to_response::(kind, &route.file_path, chunk)?, + "u8" => typed_value_to_response::(kind, &route, chunk)?, + "u16" => typed_value_to_response::(kind, &route, chunk)?, + "u32" => typed_value_to_response::(kind, &route, chunk)?, + "u64" => typed_value_to_response::(kind, &route, chunk)?, + "usize" => typed_value_to_response::(kind, &route, chunk)?, + "f32" => typed_value_to_response::(kind, &route, chunk)?, + "f64" => typed_value_to_response::(kind, &route, chunk)?, + "OHLC" => typed_value_to_response::(kind, &route, chunk)?, + "Date" => typed_value_to_response::(kind, &route, chunk)?, + "Height" => typed_value_to_response::(kind, &route, chunk)?, _ => panic!("Incompatible type: {type_name}"), }; diff --git a/server/src/imports.rs b/server/src/imports.rs index f389520ab..e40fa5130 100644 --- a/server/src/imports.rs +++ b/server/src/imports.rs @@ -1,27 +1,25 @@ -use std::fmt::Debug; +use std::{fmt::Debug, path::Path}; use bincode::Decode; -use parser::{Date, Serialization, SerializedBTreeMap, SerializedVec}; +use parser::{Date, SerializedBTreeMap, SerializedVec}; use serde::{de::DeserializeOwned, Serialize}; -pub fn import_map(relative_path: &str) -> color_eyre::Result> +use crate::routes::Route; + +pub fn import_map(route: &Route) -> color_eyre::Result where T: Serialize + Debug + DeserializeOwned + Decode, { - Serialization::from_extension(relative_path.split('.').last().unwrap()).import(relative_path) } -pub fn import_vec(relative_path: &str) -> color_eyre::Result> +pub fn import_vec(route: &Route) -> color_eyre::Result where T: Serialize + Debug + DeserializeOwned + Decode, { - Serialization::from_extension(relative_path.split('.').last().unwrap()).import(relative_path) } -pub fn import_value(relative_path: &str) -> color_eyre::Result +pub fn import_value(route: &Route) -> color_eyre::Result where T: Serialize + Debug + DeserializeOwned + Decode, { - Serialization::from_extension(relative_path.split('.').last().unwrap()) - .import::(relative_path) } diff --git a/server/src/main.rs b/server/src/main.rs index f9d217790..727ee5107 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use axum::{extract::State, http::HeaderMap, response::Response, routing::get, serve, Router}; -use parser::log; +use parser::{log, reset_logs}; use reqwest::header::HOST; use response::generic_to_reponse; use routes::Routes; @@ -12,7 +12,6 @@ use tower_http::compression::CompressionLayer; mod chunk; mod handler; mod headers; -mod imports; mod kind; mod paths; mod response; @@ -36,6 +35,8 @@ pub struct AppState { async fn main() -> color_eyre::Result<()> { color_eyre::install()?; + reset_logs(); + let routes = Routes::build(); routes.generate_dts_file(); diff --git a/server/src/response.rs b/server/src/response.rs index 566d55e8b..1d494ac24 100644 --- a/server/src/response.rs +++ b/server/src/response.rs @@ -1,15 +1,16 @@ -use std::fmt::Debug; +use std::{fmt::Debug, path::Path}; use axum::response::{IntoResponse, Json, Response}; use bincode::Decode; +use parser::{Date, SerializedBTreeMap, SerializedVec}; use serde::de::DeserializeOwned; use serde::Serialize; use crate::{ chunk::Chunk, headers::{add_cache_control_to_headers, add_cors_to_headers, add_json_type_to_headers}, - imports::{import_map, import_value, import_vec}, kind::Kind, + routes::Route, }; #[derive(Serialize)] @@ -24,16 +25,30 @@ where pub fn typed_value_to_response( kind: Kind, - relative_path: &str, + route: &Route, chunk: Option, ) -> color_eyre::Result where T: Serialize + Debug + DeserializeOwned + Decode, { Ok(match kind { - Kind::Date => dataset_to_response(import_map::(relative_path)?, chunk.unwrap()), - Kind::Height => dataset_to_response(import_vec::(relative_path)?, chunk.unwrap()), - Kind::Last => value_to_response(import_value::(relative_path)?), + Kind::Date => dataset_to_response( + route + .serialization + .import::>(Path::new(&route.file_path))?, + chunk.unwrap(), + ), + Kind::Height => dataset_to_response( + route + .serialization + .import::>(Path::new(&route.file_path))?, + chunk.unwrap(), + ), + Kind::Last => value_to_response( + route + .serialization + .import::(Path::new(&route.file_path))?, + ), }) } diff --git a/server/src/routes.rs b/server/src/routes.rs index 6d5b76dd5..2867a1b29 100644 --- a/server/src/routes.rs +++ b/server/src/routes.rs @@ -1,6 +1,7 @@ use std::{ collections::{BTreeMap, HashMap}, fs, + path::Path, }; use derive_deref::{Deref, DerefMut}; @@ -26,31 +27,22 @@ const APP_TYPES_PATH: &str = "../app/src/types"; impl Routes { pub fn build() -> Self { let path_to_type: BTreeMap = - Json::import(&format!("{INPUTS_PATH}/disk_path_to_type.json")).unwrap(); + Json::import(Path::new(&format!("{INPUTS_PATH}/disk_path_to_type.json"))).unwrap(); let mut routes = Routes::default(); path_to_type.into_iter().for_each(|(key, value)| { let mut split_key = key.split('/').collect_vec(); - - let mut split_last = split_key.pop().unwrap().split('.').rev().collect_vec(); - - let last = split_last.pop().unwrap().to_owned(); + let last = split_key.pop().unwrap().to_owned(); let mut skip = 2; - let serialization = split_last.pop().map_or_else( - || { - if *split_key.get(1).unwrap() == "price" { - skip = 1; + let mut serialization = Serialization::Binary; - Serialization::Json - } else { - Serialization::Binary - } - }, - Serialization::from_extension, - ); + if *split_key.get(1).unwrap() == "price" { + skip = 1; + serialization = Serialization::Json; + } let split_key = split_key.iter().skip(skip).collect_vec();