Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-20 23:04:46 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: datasets compression via zstd

Browse Source
This commit is contained in:
k
2024-08-05 00:44:46 +02:00
parent 9067c28d24
commit c646d6dc60
36 changed files with 544 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
CHANGELOG.md
View File

@@ -6,9 +6,15 @@
### Parser
- Change the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind`
- Use the Bitcoin RPC server for various calls instead of running commands and then parsing the JSON from the output
- Updated the config, run with `-h` to see
- Changed the block iterator from a custom version of [bitcoin-explorer](https://crates.io/crates/bitcoin-explorer) to the homemade [biter](https://crates.io/crates/biter) which allows the parser to run alongside `bitcoind`
- Added datasets compression thanks to [zstd](https://crates.io/crates/zstd) to reduce disk usage
- Use the Bitcoin RPC server for various calls instead of running cli commands and then parsing the JSON from the output
- Updated the config, run with `-h` to see possible args
- Moved outputs from `/target/outputs` to `/out` to allow to run commands like `cargo clean` without side effects
### Server
- Updated the code to support compressed binaries
## v. 0.3.0 | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26

8
README.md
View File

@@ -16,6 +16,12 @@ To promote even more transparency and trust in the network, this project is comm
This project is in a very early stage. The web app will have bugs, the API might break and the data can definitely to be false or slightly false.
## Donations
The project is a lot of work and being worked on full-time. It doesn't have any ads and solely relies on donations. If you find this project useful, any sat would really help make it even better and would be really appreciated.
You can donate on the project's [Geyser Fund](https://geyser.fund/project/satonomics/).
## Instances
Web App:
@@ -88,7 +94,7 @@ _Maybe_:
Aka: Previous iterations
The initial idea was totally different yet morphed over time into what it is today: a fully FOSS self-hostable on-chain data generator
The initial idea was totally different yet morphed over time into what it is today: a fully FOSS self-hostable on-chain data generator.
- https://github.com/drgarlic/satonomics
- https://github.com/drgarlic/satonomics-parser

3
app/src/app/components/background.tsx
View File

@@ -42,7 +42,8 @@ const texts = [
"absolute scarcity",
"time is scarce",
"ride or die",
"cyberpunk",
"cypherpunk",
"we like the coin",
];
export function Background({

2
app/src/app/components/strip/components/anchorGit.tsx
View File

@@ -5,7 +5,7 @@ export function AnchorGit() {
<Anchor
title="Git"
icon={() => IconTablerBrandGithubFilled}
href="https://github.com/satonomics-org/satonomics"
href="https://github.com/satonomics-org"
/>
);
}

1
parser/.gitignore vendored
View File

@@ -19,3 +19,4 @@ benches
parser.log
config.toml
*\ copy*
/.log

224
parser/Cargo.lock generated
View File

@@ -48,7 +48,7 @@ checksum = "fe233a377643e0fc1a56421d7c90acdec45c291b30345eb9f08e8d0ddce5a4ab"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.72",
]
[[package]]
@@ -289,9 +289,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.5.0"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
[[package]]
name = "bumpalo"
@@ -301,9 +301,9 @@ checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa"
[[package]]
name = "bytemuck"
version = "1.16.1"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e"
checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6"
[[package]]
name = "byteorder"
@@ -322,6 +322,11 @@ name = "cc"
version = "1.0.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695"
dependencies = [
"jobserver",
"libc",
"once_cell",
]
[[package]]
name = "cfg-if"
@@ -341,14 +346,14 @@ dependencies = [
"num-traits",
"serde",
"wasm-bindgen",
"windows-targets 0.52.4",
"windows-targets 0.52.6",
]
[[package]]
name = "clap"
version = "4.5.11"
version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3"
checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc"
dependencies = [
"clap_builder",
"clap_derive",
@@ -356,9 +361,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.11"
version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa"
checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99"
dependencies = [
"anstream",
"anstyle",
@@ -368,21 +373,21 @@ dependencies = [
[[package]]
name = "clap_derive"
version = "4.5.11"
version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e"
checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.72",
]
[[package]]
name = "clap_lex"
version = "0.7.0"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70"
[[package]]
name = "color-eyre"
@@ -448,9 +453,9 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
version = "0.5.12"
version = "0.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95"
checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
dependencies = [
"crossbeam-utils",
]
@@ -485,9 +490,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.19"
version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]]
name = "ctor"
@@ -532,7 +537,7 @@ checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.72",
]
[[package]]
@@ -693,9 +698,9 @@ dependencies = [
[[package]]
name = "getrandom"
version = "0.2.14"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
@@ -729,9 +734,9 @@ dependencies = [
[[package]]
name = "hashbrown"
version = "0.14.3"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "heck"
@@ -924,9 +929,9 @@ dependencies = [
[[package]]
name = "inferno"
version = "0.11.20"
version = "0.11.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c77a3ae7d4761b9c64d2c030f70746ceb8cfba32dce0325a56792e0a4816c31"
checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88"
dependencies = [
"ahash",
"clap",
@@ -992,6 +997,15 @@ version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
[[package]]
name = "jobserver"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
dependencies = [
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.69"
@@ -1043,15 +1057,15 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.21"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
[[package]]
name = "memchr"
version = "2.7.1"
version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "memmap2"
@@ -1140,9 +1154,9 @@ dependencies = [
[[package]]
name = "num-traits"
version = "0.2.18"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
]
@@ -1168,7 +1182,7 @@ version = "0.10.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f"
dependencies = [
"bitflags 2.5.0",
"bitflags 2.6.0",
"cfg-if",
"foreign-types",
"libc",
@@ -1185,7 +1199,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.72",
]
[[package]]
@@ -1208,9 +1222,9 @@ dependencies = [
[[package]]
name = "ordered-float"
version = "4.2.1"
version = "4.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19ff2cf528c6c03d9ed653d6c4ce1dc0582dc4af309790ad92f07c1cd551b0be"
checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6"
dependencies = [
"num-traits",
]
@@ -1256,7 +1270,7 @@ dependencies = [
"libc",
"redox_syscall 0.5.2",
"smallvec",
"windows-targets 0.52.4",
"windows-targets 0.52.6",
]
[[package]]
@@ -1281,6 +1295,7 @@ dependencies = [
"serde",
"serde_json",
"toml",
"zstd",
]
[[package]]
@@ -1306,7 +1321,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.72",
]
[[package]]
@@ -1338,9 +1353,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.85"
version = "1.0.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23"
checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
dependencies = [
"unicode-ident",
]
@@ -1428,7 +1443,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd"
dependencies = [
"bitflags 2.5.0",
"bitflags 2.6.0",
]
[[package]]
@@ -1477,9 +1492,9 @@ dependencies = [
[[package]]
name = "rgb"
version = "0.8.44"
version = "0.8.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aee83dc281d5a3200d37b299acd13b81066ea126a7f16f0eae70fc9aed241d9"
checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8"
dependencies = [
"bytemuck",
]
@@ -1511,7 +1526,7 @@ version = "0.38.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89"
dependencies = [
"bitflags 2.5.0",
"bitflags 2.6.0",
"errno",
"libc",
"linux-raw-sys",
@@ -1661,7 +1676,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.72",
]
[[package]]
@@ -1768,9 +1783,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.66"
version = "2.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5"
checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
dependencies = [
"proc-macro2",
"quote",
@@ -1833,7 +1848,7 @@ checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.72",
]
[[package]]
@@ -1913,9 +1928,9 @@ dependencies = [
[[package]]
name = "toml"
version = "0.8.16"
version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c"
checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
dependencies = [
"serde",
"serde_spanned",
@@ -1925,18 +1940,18 @@ dependencies = [
[[package]]
name = "toml_datetime"
version = "0.6.7"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db"
checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
version = "0.22.17"
version = "0.22.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16"
checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"
dependencies = [
"indexmap",
"serde",
@@ -2085,9 +2100,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
version = "0.9.4"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "virtue"
@@ -2131,7 +2146,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.72",
"wasm-bindgen-shared",
]
@@ -2165,7 +2180,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.72",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -2214,7 +2229,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
dependencies = [
"windows-targets 0.52.4",
"windows-targets 0.52.6",
]
[[package]]
@@ -2232,7 +2247,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets 0.52.4",
"windows-targets 0.52.6",
]
[[package]]
@@ -2252,17 +2267,18 @@ dependencies = [
[[package]]
name = "windows-targets"
version = "0.52.4"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm 0.52.4",
"windows_aarch64_msvc 0.52.4",
"windows_i686_gnu 0.52.4",
"windows_i686_msvc 0.52.4",
"windows_x86_64_gnu 0.52.4",
"windows_x86_64_gnullvm 0.52.4",
"windows_x86_64_msvc 0.52.4",
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
@@ -2273,9 +2289,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.4"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
@@ -2285,9 +2301,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.4"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
@@ -2297,9 +2313,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
[[package]]
name = "windows_i686_gnu"
version = "0.52.4"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
@@ -2309,9 +2331,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
[[package]]
name = "windows_i686_msvc"
version = "0.52.4"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
@@ -2321,9 +2343,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.4"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
@@ -2333,9 +2355,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.4"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
@@ -2345,15 +2367,15 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.4"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winnow"
version = "0.6.13"
version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1"
checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
dependencies = [
"memchr",
]
@@ -2395,7 +2417,7 @@ checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.72",
]
[[package]]
@@ -2406,7 +2428,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 2.0.72",
]
[[package]]
@@ -2414,3 +2436,31 @@ name = "zeroize"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
[[package]]
name = "zstd"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.13+zstd.1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
dependencies = [
"cc",
"pkg-config",
]

11
parser/Cargo.toml
View File

@@ -14,16 +14,17 @@ bincode = { git = "https://github.com/bincode-org/bincode.git" }
biter = "0.1.0"
bitcoin_hashes = { version = "0.14.0" }
chrono = { version = "0.4.38", features = ["serde"] }
clap = { version = "4.5.11", features = ["derive"] }
clap = { version = "4.5.13", features = ["derive"] }
color-eyre = "0.6.3"
derive_deref = "1.1.1"
inferno = "0.11.20"
inferno = "0.11.21"
itertools = "0.13.0"
memory-stats = "1.2.0"
ordered-float = "4.2.1"
ordered-float = "4.2.2"
rayon = "1.10.0"
reqwest = { version = "0.12.5", features = ["blocking", "json"] }
sanakirja = "1.4.2"
serde = { version = "1.0.204", features = ["derive"] }
serde_json = "1.0.120"
toml = "0.8.16"
serde_json = "1.0.122"
toml = "0.8.19"
zstd = "0.13.2"

5
parser/src/actions/iter_blocks.rs
View File

@@ -11,6 +11,7 @@ use crate::{
create_rpc,
databases::Databases,
datasets::{AllDatasets, ComputeData},
io::OUTPUTS_FOLDER_PATH,
states::{AddressCohortsDurableStates, States, UTXOCohortsDurableStates},
structs::{Date, DateData, MapKey},
utils::{generate_allocation_files, log, time},
@@ -22,8 +23,6 @@ pub fn iter_blocks(
rpc: &biter::bitcoincore_rpc::Client,
approx_block_count: usize,
) -> color_eyre::Result<()> {
let export_dir = "./target/outputs";
let should_insert = true;
let should_export = true;
let study_ram_usage = false;
@@ -56,7 +55,7 @@ pub fn iter_blocks(
let block_receiver = biter::new(
config.datadir.as_ref().unwrap(),
export_dir,
OUTPUTS_FOLDER_PATH,
Some(height.to_usize()),
None,
create_rpc(config).unwrap(),

12
parser/src/databases/metadata.rs
View File

@@ -1,14 +1,16 @@
use allocative::Allocative;
use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use std::{
fmt::Debug,
fs, io,
ops::{Deref, DerefMut},
path::Path,
};
use crate::{
io::Binary,
structs::{Counter, Date, Height},
Serialization,
};
#[derive(Default, Debug, Encode, Decode, Allocative)]
@@ -73,7 +75,7 @@ impl Metadata {
}
}
#[derive(Default, Debug, Encode, Decode, Allocative)]
#[derive(Default, Debug, Encode, Decode, Serialize, Deserialize, Allocative)]
pub struct MetadataData {
pub serial: usize,
pub len: Counter,
@@ -88,17 +90,17 @@ impl MetadataData {
fn full_path(folder_path: &str) -> String {
let name = Self::name();
format!("{folder_path}/{name}.bin")
format!("{folder_path}/{name}")
}
pub fn import(path: &str) -> color_eyre::Result<Self> {
fs::create_dir_all(path)?;
Binary::import(&Self::full_path(path))
Serialization::Binary.import(Path::new(&Self::full_path(path)))
}
pub fn export(&self, path: &str) -> color_eyre::Result<()> {
Binary::export(&Self::full_path(path), self)
Serialization::Binary.export(Path::new(&Self::full_path(path)), self)
}
pub fn reset(&mut self, path: &str) -> color_eyre::Result<(), io::Error> {

4
parser/src/datasets/mod.rs
View File

@@ -1,4 +1,4 @@
use std::{collections::BTreeMap, fs, ops::RangeInclusive};
use std::{collections::BTreeMap, fs, ops::RangeInclusive, path::Path};
use allocative::Allocative;
@@ -282,7 +282,7 @@ impl AllDatasets {
fs::create_dir_all(server_inputs_path)?;
Json::export(
&format!("{server_inputs_path}/disk_path_to_type.json"),
Path::new(&format!("{server_inputs_path}/disk_path_to_type.json")),
&path_to_type,
)?;

104
parser/src/io/binary.rs
View File

@@ -1,15 +1,39 @@
use std::{
fmt::Debug,
fs::File,
io::{BufReader, BufWriter},
fs::{self, File},
io::{BufReader, BufWriter, Cursor},
path::Path,
};
use bincode::{config, decode_from_std_read, encode_into_std_write, Decode, Encode};
use bincode::{
config, decode_from_slice, decode_from_std_read, encode_into_std_write, Decode, Encode,
};
use zstd::decode_all;
const ZST_EXTENSION: &str = "zst";
pub const BIN_EXTENSION: &str = "bin";
pub const COMPRESSED_BIN_EXTENSION: &str = "bin.zst";
enum BinaryType {
Raw,
Compressed,
}
pub struct Binary;
impl Binary {
pub fn import<T>(path: &str) -> color_eyre::Result<T>
pub fn import<T>(path: &Path) -> color_eyre::Result<T>
where
T: Decode,
{
match Self::type_from_path(path) {
BinaryType::Compressed => Self::import_compressed(path),
BinaryType::Raw => Self::import_raw(path),
}
}
fn import_raw<T>(path: &Path) -> color_eyre::Result<T>
where
T: Decode,
{
@@ -24,7 +48,34 @@ impl Binary {
Ok(decoded)
}
pub fn export<T>(path: &str, value: &T) -> color_eyre::Result<()>
fn import_compressed<T>(path: &Path) -> color_eyre::Result<T>
where
T: Decode,
{
let file = File::open(path).unwrap();
let reader = BufReader::new(file);
let decompressed = decode_all(reader).unwrap();
let config = config::standard();
let decoded = decode_from_slice::<T, _>(&decompressed, config).unwrap().0;
Ok(decoded)
}
pub fn export<T>(path: &Path, value: &T) -> color_eyre::Result<()>
where
T: Debug + Encode,
{
match Self::type_from_path(path) {
BinaryType::Compressed => Self::export_compressed(path, value),
BinaryType::Raw => Self::export_raw(path, value),
}
}
fn export_raw<T>(path: &Path, value: &T) -> color_eyre::Result<()>
where
T: Debug + Encode,
{
@@ -40,4 +91,47 @@ impl Binary {
Ok(())
}
fn export_compressed<T>(path: &Path, value: &T) -> color_eyre::Result<()>
where
T: Debug + Encode,
{
let config = config::standard();
let encoded = bincode::encode_to_vec(value, config).unwrap();
let cursor = Cursor::new(encoded);
let compressed = zstd::encode_all(cursor, 0).unwrap();
fs::write(path, compressed).unwrap();
Ok(())
}
pub fn has_correct_extension(path: &Path) -> bool {
let path = path.to_str().unwrap();
path.ends_with(BIN_EXTENSION) || path.ends_with(COMPRESSED_BIN_EXTENSION)
}
fn type_from_path(path: &Path) -> BinaryType {
let extension = path.extension();
if extension.is_none() {
panic!("Should have extension");
}
if !Self::has_correct_extension(path) {
dbg!(path);
panic!("Wrong extension")
}
let extension = extension.unwrap();
if extension == ZST_EXTENSION {
BinaryType::Compressed
} else {
BinaryType::Raw
}
}
}

2
parser/src/io/consts.rs
View File

@@ -1,2 +1,2 @@
pub const INPUTS_FOLDER_PATH: &str = "./in";
pub const OUTPUTS_FOLDER_PATH: &str = "./target/outputs";
pub const OUTPUTS_FOLDER_PATH: &str = "./out";

23
parser/src/io/json.rs
View File

@@ -1,17 +1,25 @@
use std::{
fs::File,
io::{BufReader, BufWriter},
path::Path,
};
use serde::{de::DeserializeOwned, Serialize};
pub struct Json;
pub const JSON_EXTENSION: &str = "json";
pub const HAR_EXTENSION: &str = "har";
impl Json {
pub fn import<T>(path: &str) -> color_eyre::Result<T>
pub fn import<T>(path: &Path) -> color_eyre::Result<T>
where
T: DeserializeOwned,
{
if !Self::has_correct_extension(path) {
panic!("Wrong extension");
}
let file = File::open(path)?;
let reader = BufReader::new(file);
@@ -19,10 +27,15 @@ impl Json {
Ok(serde_json::from_reader(reader)?)
}
pub fn export<T>(path: &str, value: &T) -> color_eyre::Result<()>
pub fn export<T>(path: &Path, value: &T) -> color_eyre::Result<()>
where
T: Serialize,
{
if !Self::has_correct_extension(path) {
dbg!(path);
panic!("Wrong extension");
}
let file = File::create(path).unwrap_or_else(|_| {
dbg!(&path);
panic!("No such file or directory")
@@ -34,4 +47,10 @@ impl Json {
Ok(())
}
#[inline(always)]
pub fn has_correct_extension(path: &Path) -> bool {
let path = path.to_str().unwrap();
path.ends_with(JSON_EXTENSION) || path.ends_with(HAR_EXTENSION)
}
}

103
parser/src/io/serialization.rs
View File

@@ -1,4 +1,4 @@
use std::fmt::Debug;
use std::{fmt::Debug, fs, path::Path};
use allocative::Allocative;
use bincode::{Decode, Encode};
@@ -6,6 +6,8 @@ use serde::{de::DeserializeOwned, Serialize};
use crate::io::{Binary, Json};
use super::{BIN_EXTENSION, COMPRESSED_BIN_EXTENSION, HAR_EXTENSION, JSON_EXTENSION};
#[derive(PartialEq, PartialOrd, Ord, Eq, Debug, Clone, Copy, Default, Allocative)]
pub enum Serialization {
#[default]
@@ -14,42 +16,105 @@ pub enum Serialization {
}
impl Serialization {
pub fn to_extension(&self) -> &str {
pub fn is_serializable(&self, path: &Path) -> bool {
let path = path.to_str().unwrap();
match self {
Self::Binary => "bin",
Self::Json => "json",
Self::Binary => {
path.ends_with(BIN_EXTENSION) || path.ends_with(COMPRESSED_BIN_EXTENSION)
}
Self::Json => path.ends_with(JSON_EXTENSION) || path.ends_with(HAR_EXTENSION),
}
}
pub fn from_extension(extension: &str) -> Self {
match extension {
"bin" => Self::Binary,
"json" => Self::Json,
_ => panic!("Extension \"{extension}\" isn't supported"),
pub fn from_path(path: &Path) -> Self {
let path = path.to_str().unwrap();
if path.ends_with(BIN_EXTENSION) || path.ends_with(COMPRESSED_BIN_EXTENSION) {
Self::Binary
} else if path.ends_with(JSON_EXTENSION) || path.ends_with(HAR_EXTENSION) {
Self::Json
} else {
panic!("Extension \"{path}\" isn't supported")
}
}
pub fn append_extension(&self, path: &str) -> String {
format!("{path}.{}", self.to_extension())
}
pub fn import<T>(&self, path: &str) -> color_eyre::Result<T>
pub fn import<T>(&self, path: &Path) -> color_eyre::Result<T>
where
T: Debug + DeserializeOwned + Decode,
{
match self {
Serialization::Binary => Binary::import(path),
Serialization::Json => Json::import(path),
Serialization::Binary => {
if self.is_serializable(path) {
Binary::import(path)
} else {
let path = path.to_str().unwrap();
let bin_path_str = format!("{path}.{BIN_EXTENSION}");
let bin_path = Path::new(&bin_path_str);
if bin_path.exists() {
return Binary::import(bin_path);
}
let compressed_bin_path_str = format!("{path}.{COMPRESSED_BIN_EXTENSION}");
let compressed_bin_path = Path::new(&compressed_bin_path_str);
if compressed_bin_path.exists() {
return Binary::import(compressed_bin_path);
}
panic!("Wrong path")
}
}
Serialization::Json => {
if self.is_serializable(path) {
Json::import(path)
} else {
let path = path.to_str().unwrap();
let json_path_str = format!("{path}.{JSON_EXTENSION}");
let json_path = Path::new(&json_path_str);
if json_path.exists() {
return Json::import(json_path);
}
panic!("Wrong path")
}
}
}
}
pub fn export<T>(&self, path: &str, value: &T) -> color_eyre::Result<()>
pub fn export<T>(&self, path: &Path, value: &T) -> color_eyre::Result<()>
where
T: Debug + Serialize + Encode,
{
match self {
Serialization::Binary => Binary::export(path, value),
Serialization::Json => Json::export(path, value),
Serialization::Binary => {
if self.is_serializable(path) {
Binary::export(path, value)
} else {
let path = path.to_str().unwrap();
let res = Binary::export(
Path::new(&format!("{}.{COMPRESSED_BIN_EXTENSION}", path,)),
value,
);
if res.is_ok() {
let _ = fs::remove_file(Path::new(&format!("{}.{BIN_EXTENSION}", path)));
}
res
}
}
Serialization::Json => {
if self.is_serializable(path) {
Json::export(path, value)
} else {
Json::export(
Path::new(&format!("{}.{JSON_EXTENSION}", path.to_str().unwrap())),
value,
)
}
}
}
}
}

2
parser/src/lib.rs
View File

@@ -15,5 +15,5 @@ pub use crate::{
Config, Date, DateMap, Height, HeightMap, MapChunkId, SerializedBTreeMap, SerializedVec,
HEIGHT_MAP_CHUNK_SIZE,
},
utils::{create_rpc, log},
utils::{create_rpc, log, reset_logs},
};

4
parser/src/main.rs
View File

@@ -1,11 +1,13 @@
use std::{thread::sleep, time::Duration};
use biter::bitcoincore_rpc::RpcApi;
use parser::{create_rpc, iter_blocks, log, Config};
use parser::{create_rpc, iter_blocks, log, reset_logs, Config};
fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
reset_logs();
let config = Config::import();
let rpc = create_rpc(&config).unwrap();

3
parser/src/price/binance.rs
View File

@@ -23,8 +23,7 @@ impl Binance {
let path_binance_har = Path::new(INPUTS_FOLDER_PATH).join("binance.har");
let json: BTreeMap<String, Value> =
Json::import(path_binance_har.to_str().unwrap()).unwrap_or_default();
let json: BTreeMap<String, Value> = Json::import(&path_binance_har).unwrap_or_default();
Ok(json
.get("log")

13
parser/src/states/_trait.rs
View File

@@ -1,13 +1,14 @@
use std::{fmt::Debug, fs, io};
use std::{fmt::Debug, fs, io, path::Path};
use bincode::{Decode, Encode};
use serde::{de::DeserializeOwned, Serialize};
use crate::io::{Binary, OUTPUTS_FOLDER_PATH};
use crate::{io::OUTPUTS_FOLDER_PATH, Serialization};
// https://github.com/djkoloski/rust_serialization_benchmark
pub trait AnyState
where
Self: Debug + Encode + Decode,
Self: Debug + Encode + Decode + Serialize + DeserializeOwned,
{
fn name<'a>() -> &'a str;
@@ -24,7 +25,7 @@ where
let folder_path = Self::folder_path();
format!("{folder_path}/{name}.bin")
format!("{folder_path}/{name}")
}
fn reset(&mut self) -> color_eyre::Result<(), io::Error> {
@@ -36,11 +37,11 @@ where
fn import() -> color_eyre::Result<Self> {
Self::create_dir_all()?;
Binary::import(&Self::full_path())
Serialization::Binary.import(Path::new(&Self::full_path()))
}
fn export(&self) -> color_eyre::Result<()> {
Binary::export(&Self::full_path(), self)
Serialization::Binary.export(Path::new(&Self::full_path()), self)
}
fn clear(&mut self);

3
parser/src/states/counters.rs
View File

@@ -1,11 +1,12 @@
use allocative::Allocative;
use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use crate::structs::Counter;
use super::AnyState;
#[derive(Default, Debug, Encode, Decode, Allocative)]
#[derive(Default, Debug, Encode, Decode, Serialize, Deserialize, Allocative)]
pub struct Counters {
pub op_return_addresses: Counter,
pub push_only_addresses: Counter,

3
parser/src/states/date_data_vec.rs
View File

@@ -1,12 +1,13 @@
use allocative::Allocative;
use bincode::{Decode, Encode};
use derive_deref::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
use crate::structs::{BlockData, BlockPath, DateData};
use super::AnyState;
#[derive(Default, Deref, DerefMut, Debug, Encode, Decode, Allocative)]
#[derive(Default, Deref, DerefMut, Debug, Serialize, Deserialize, Encode, Decode, Allocative)]
pub struct DateDataVec(Vec<DateData>);
impl DateDataVec {

3
parser/src/structs/block_data.rs
View File

@@ -1,9 +1,10 @@
use allocative::Allocative;
use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use super::{Amount, Height, Price};
#[derive(Debug, Encode, Decode, Allocative)]
#[derive(Debug, Serialize, Deserialize, Encode, Decode, Allocative)]
pub struct BlockData {
pub height: Height,
pub price: Price,

5
parser/src/structs/counter.rs
View File

@@ -1,8 +1,11 @@
use allocative::Allocative;
use bincode::{Decode, Encode};
use derive_deref::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
#[derive(Debug, Deref, DerefMut, Default, Clone, Copy, Encode, Decode, Allocative)]
#[derive(
Debug, Deref, DerefMut, Default, Clone, Copy, Encode, Decode, Serialize, Deserialize, Allocative,
)]
pub struct Counter(u32);
impl Counter {

3
parser/src/structs/date_data.rs
View File

@@ -1,9 +1,10 @@
use allocative::Allocative;
use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use super::{BlockData, BlockPath, Date};
#[derive(Debug, Encode, Decode, Allocative)]
#[derive(Debug, Serialize, Deserialize, Encode, Decode, Allocative)]
pub struct DateData {
pub date: Date,
pub blocks: Vec<BlockData>,

16
parser/src/structs/date_map_chunk_id.rs
View File

@@ -1,3 +1,5 @@
use std::path::Path;
use allocative::Allocative;
use chrono::Datelike;
@@ -19,8 +21,18 @@ impl MapChunkId for DateMapChunkId {
self.0.to_string()
}
fn from_name(name: &str) -> Self {
Self(name.parse::<i32>().unwrap())
fn from_path(path: &Path) -> Self {
Self(
path.file_name()
.unwrap()
.to_str()
.unwrap()
.split(".")
.next()
.unwrap()
.parse::<i32>()
.unwrap(),
)
}
fn to_usize(self) -> usize {

32
parser/src/structs/generic_map.rs
View File

@@ -63,7 +63,7 @@ where
Self: Ord + Debug + Copy + Clone,
{
fn to_name(&self) -> String;
fn from_name(name: &str) -> Self;
fn from_path(path: &Path) -> Self;
fn to_usize(self) -> usize;
fn from_usize(id: usize) -> Self;
}
@@ -124,7 +124,7 @@ where
let path_last = {
if export_last {
Some(serialization.append_extension(&format!("{path}/last")))
Some(format!("{path}/last"))
} else {
None
}
@@ -188,22 +188,16 @@ where
fs::read_dir(path)
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| {
let extension = path.extension().unwrap().to_str().unwrap();
path.is_file() && extension == serialization.to_extension()
})
.filter(|path| serialization.is_serializable(path))
.map(|path| {
let chunk_id = ChunkId::from_name(path.file_stem().unwrap().to_str().unwrap());
let chunk_id = ChunkId::from_path(&path);
(chunk_id, path)
})
.collect()
}
fn import(&self, path: &Path) -> color_eyre::Result<Serialized> {
self.serialization
.import::<Serialized>(path.to_str().unwrap())
self.serialization.import::<Serialized>(path)
}
pub fn insert(&mut self, key: Key, value: Value) -> Value {
@@ -309,10 +303,7 @@ where
panic!();
});
let serialized = self
.serialization
.import::<Serialized>(path.to_str().unwrap())
.unwrap();
let serialized = self.serialization.import::<Serialized>(path).unwrap();
self.imported.insert(*chunk_id, serialized);
}
@@ -334,23 +325,18 @@ where
unreachable!()
}
let path = self.serialization.append_extension(&format!(
"{}/{}",
self.path_all,
chunk_id.to_name()
));
let serialized = self.imported.get(chunk_id).unwrap_or_else(|| {
dbg!(&self.path_all, chunk_id, &self.imported);
panic!();
});
self.serialization.export(&path, serialized)?;
let path = format!("{}/{}", self.path_all, chunk_id.to_name());
self.serialization.export(Path::new(&path), serialized)?;
if index == len - 1 {
if let Some(path_last) = self.path_last.as_ref() {
self.serialization
.export(path_last, serialized.last().unwrap())?;
.export(Path::new(path_last), serialized.last().unwrap())?;
}
}

17
parser/src/structs/height_map_chunk_id.rs
View File

@@ -1,3 +1,5 @@
use std::path::Path;
use allocative::Allocative;
use derive_deref::{Deref, DerefMut};
@@ -26,9 +28,20 @@ impl MapChunkId for HeightMapChunkId {
format!("{start}..{end}")
}
fn from_name(name: &str) -> Self {
fn from_path(path: &Path) -> Self {
Self(Height::new(
name.split("..").next().unwrap().parse::<u32>().unwrap(),
path.file_name()
.unwrap()
.to_str()
.unwrap()
.split("..")
.next()
.unwrap()
.parse::<u32>()
.unwrap_or_else(|_| {
dbg!(path);
panic!()
}),
))
}

15
parser/src/structs/price.rs
View File

@@ -2,11 +2,24 @@ use std::ops::{Add, AddAssign, Div, Mul, Sub, SubAssign};
use allocative::Allocative;
use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use super::Amount;
#[derive(
Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Allocative,
Debug,
Default,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Serialize,
Deserialize,
Encode,
Decode,
Allocative,
)]
pub struct Price(u64);

13
parser/src/utils/log.rs
View File

@@ -1,8 +1,17 @@
use std::{fs::OpenOptions, io::Write};
use std::{
fs::{self, OpenOptions},
io::Write,
};
use chrono::Local;
use color_eyre::owo_colors::OwoColorize;
const LOG_PATH: &str = "./.log";
pub fn reset_logs() {
let _ = fs::remove_file(LOG_PATH);
}
#[inline(always)]
pub fn log(str: &str) {
let date_time = format!("{}", Local::now().format("%Y-%m-%d %H:%M:%S -"));
@@ -13,7 +22,7 @@ pub fn log(str: &str) {
let mut file = OpenOptions::new()
.create(true)
.append(true)
.open("./parser.log")
.open(LOG_PATH)
.unwrap();
if let Err(e) = writeln!(file, "{} {}", date_time, line) {

2
server/.gitignore vendored
View File

@@ -1,4 +1,4 @@
/target
.DS_Store
/parser.log
/.log
/in

49
server/Cargo.lock generated
View File

@@ -477,9 +477,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.11"
version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3"
checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc"
dependencies = [
"clap_builder",
"clap_derive",
@@ -487,9 +487,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.11"
version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa"
checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99"
dependencies = [
"anstream",
"anstyle",
@@ -499,9 +499,9 @@ dependencies = [
[[package]]
name = "clap_derive"
version = "4.5.11"
version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e"
checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
dependencies = [
"heck",
"proc-macro2",
@@ -1081,9 +1081,9 @@ dependencies = [
[[package]]
name = "inferno"
version = "0.11.20"
version = "0.11.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c77a3ae7d4761b9c64d2c030f70746ceb8cfba32dce0325a56792e0a4816c31"
checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88"
dependencies = [
"ahash",
"clap",
@@ -1380,9 +1380,9 @@ dependencies = [
[[package]]
name = "ordered-float"
version = "4.2.1"
version = "4.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19ff2cf528c6c03d9ed653d6c4ce1dc0582dc4af309790ad92f07c1cd551b0be"
checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6"
dependencies = [
"num-traits",
]
@@ -1463,6 +1463,7 @@ dependencies = [
"serde",
"serde_json",
"toml",
"zstd",
]
[[package]]
@@ -2187,9 +2188,9 @@ dependencies = [
[[package]]
name = "toml"
version = "0.8.16"
version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c"
checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
dependencies = [
"serde",
"serde_spanned",
@@ -2199,18 +2200,18 @@ dependencies = [
[[package]]
name = "toml_datetime"
version = "0.6.7"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db"
checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
version = "0.22.17"
version = "0.22.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16"
checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"
dependencies = [
"indexmap",
"serde",
@@ -2645,9 +2646,9 @@ checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8"
[[package]]
name = "winnow"
version = "0.6.13"
version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1"
checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
dependencies = [
"memchr",
]
@@ -2711,27 +2712,27 @@ checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
[[package]]
name = "zstd"
version = "0.13.0"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110"
checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.0.0"
version = "7.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e"
checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.9+zstd.1.5.5"
version = "2.0.13+zstd.1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656"
checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
dependencies = [
"cc",
"pkg-config",

4
server/run.sh
View File

@@ -1,5 +1,7 @@
if cargo watch --help &> /dev/null; then
cargo watch --no-vcs-ignores -w "./src" -w "./run.sh" -w "./in/datasets_len.txt" -x "run -r"
TRIGGER="./in/datasets_len.txt"
echo "0" > $TRIGGER
cargo watch --no-vcs-ignores -w "./src" -w "./run.sh" -w "$TRIGGER" -x "run -r"
else
cargo run -r
fi

20
server/src/handler.rs
View File

@@ -109,16 +109,16 @@ fn _file_handler(
let type_name = route.values_type.split("::").last().unwrap();
let value = match type_name {
"u8" => typed_value_to_response::<u8>(kind, &route.file_path, chunk)?,
"u16" => typed_value_to_response::<u16>(kind, &route.file_path, chunk)?,
"u32" => typed_value_to_response::<u32>(kind, &route.file_path, chunk)?,
"u64" => typed_value_to_response::<u64>(kind, &route.file_path, chunk)?,
"usize" => typed_value_to_response::<usize>(kind, &route.file_path, chunk)?,
"f32" => typed_value_to_response::<f32>(kind, &route.file_path, chunk)?,
"f64" => typed_value_to_response::<f64>(kind, &route.file_path, chunk)?,
"OHLC" => typed_value_to_response::<OHLC>(kind, &route.file_path, chunk)?,
"Date" => typed_value_to_response::<Date>(kind, &route.file_path, chunk)?,
"Height" => typed_value_to_response::<Height>(kind, &route.file_path, chunk)?,
"u8" => typed_value_to_response::<u8>(kind, &route, chunk)?,
"u16" => typed_value_to_response::<u16>(kind, &route, chunk)?,
"u32" => typed_value_to_response::<u32>(kind, &route, chunk)?,
"u64" => typed_value_to_response::<u64>(kind, &route, chunk)?,
"usize" => typed_value_to_response::<usize>(kind, &route, chunk)?,
"f32" => typed_value_to_response::<f32>(kind, &route, chunk)?,
"f64" => typed_value_to_response::<f64>(kind, &route, chunk)?,
"OHLC" => typed_value_to_response::<OHLC>(kind, &route, chunk)?,
"Date" => typed_value_to_response::<Date>(kind, &route, chunk)?,
"Height" => typed_value_to_response::<Height>(kind, &route, chunk)?,
_ => panic!("Incompatible type: {type_name}"),
};

16
server/src/imports.rs
View File

@@ -1,27 +1,25 @@
use std::fmt::Debug;
use std::{fmt::Debug, path::Path};
use bincode::Decode;
use parser::{Date, Serialization, SerializedBTreeMap, SerializedVec};
use parser::{Date, SerializedBTreeMap, SerializedVec};
use serde::{de::DeserializeOwned, Serialize};
pub fn import_map<T>(relative_path: &str) -> color_eyre::Result<SerializedBTreeMap<Date, T>>
use crate::routes::Route;
pub fn import_map<T>(route: &Route) -> color_eyre::Result
where
T: Serialize + Debug + DeserializeOwned + Decode,
{
Serialization::from_extension(relative_path.split('.').last().unwrap()).import(relative_path)
}
pub fn import_vec<T>(relative_path: &str) -> color_eyre::Result<SerializedVec<T>>
pub fn import_vec<T>(route: &Route) -> color_eyre::Result
where
T: Serialize + Debug + DeserializeOwned + Decode,
{
Serialization::from_extension(relative_path.split('.').last().unwrap()).import(relative_path)
}
pub fn import_value<T>(relative_path: &str) -> color_eyre::Result<T>
pub fn import_value<T>(route: &Route) -> color_eyre::Result<T>
where
T: Serialize + Debug + DeserializeOwned + Decode,
{
Serialization::from_extension(relative_path.split('.').last().unwrap())
.import::<T>(relative_path)
}

5
server/src/main.rs
View File

@@ -1,7 +1,7 @@
use std::sync::Arc;
use axum::{extract::State, http::HeaderMap, response::Response, routing::get, serve, Router};
use parser::log;
use parser::{log, reset_logs};
use reqwest::header::HOST;
use response::generic_to_reponse;
use routes::Routes;
@@ -12,7 +12,6 @@ use tower_http::compression::CompressionLayer;
mod chunk;
mod handler;
mod headers;
mod imports;
mod kind;
mod paths;
mod response;
@@ -36,6 +35,8 @@ pub struct AppState {
async fn main() -> color_eyre::Result<()> {
color_eyre::install()?;
reset_logs();
let routes = Routes::build();
routes.generate_dts_file();

27
server/src/response.rs
View File

@@ -1,15 +1,16 @@
use std::fmt::Debug;
use std::{fmt::Debug, path::Path};
use axum::response::{IntoResponse, Json, Response};
use bincode::Decode;
use parser::{Date, SerializedBTreeMap, SerializedVec};
use serde::de::DeserializeOwned;
use serde::Serialize;
use crate::{
chunk::Chunk,
headers::{add_cache_control_to_headers, add_cors_to_headers, add_json_type_to_headers},
imports::{import_map, import_value, import_vec},
kind::Kind,
routes::Route,
};
#[derive(Serialize)]
@@ -24,16 +25,30 @@ where
pub fn typed_value_to_response<T>(
kind: Kind,
relative_path: &str,
route: &Route,
chunk: Option<Chunk>,
) -> color_eyre::Result<Response>
where
T: Serialize + Debug + DeserializeOwned + Decode,
{
Ok(match kind {
Kind::Date => dataset_to_response(import_map::<T>(relative_path)?, chunk.unwrap()),
Kind::Height => dataset_to_response(import_vec::<T>(relative_path)?, chunk.unwrap()),
Kind::Last => value_to_response(import_value::<T>(relative_path)?),
Kind::Date => dataset_to_response(
route
.serialization
.import::<SerializedBTreeMap<Date, T>>(Path::new(&route.file_path))?,
chunk.unwrap(),
),
Kind::Height => dataset_to_response(
route
.serialization
.import::<SerializedVec<T>>(Path::new(&route.file_path))?,
chunk.unwrap(),
),
Kind::Last => value_to_response(
route
.serialization
.import::<T>(Path::new(&route.file_path))?,
),
})
}

24
server/src/routes.rs
View File

@@ -1,6 +1,7 @@
use std::{
collections::{BTreeMap, HashMap},
fs,
path::Path,
};
use derive_deref::{Deref, DerefMut};
@@ -26,31 +27,22 @@ const APP_TYPES_PATH: &str = "../app/src/types";
impl Routes {
pub fn build() -> Self {
let path_to_type: BTreeMap<String, String> =
Json::import(&format!("{INPUTS_PATH}/disk_path_to_type.json")).unwrap();
Json::import(Path::new(&format!("{INPUTS_PATH}/disk_path_to_type.json"))).unwrap();
let mut routes = Routes::default();
path_to_type.into_iter().for_each(|(key, value)| {
let mut split_key = key.split('/').collect_vec();
let mut split_last = split_key.pop().unwrap().split('.').rev().collect_vec();
let last = split_last.pop().unwrap().to_owned();
let last = split_key.pop().unwrap().to_owned();
let mut skip = 2;
let serialization = split_last.pop().map_or_else(
|| {
if *split_key.get(1).unwrap() == "price" {
skip = 1;
let mut serialization = Serialization::Binary;
Serialization::Json
} else {
Serialization::Binary
}
},
Serialization::from_extension,
);
if *split_key.get(1).unwrap() == "price" {
skip = 1;
serialization = Serialization::Json;
}
let split_key = split_key.iter().skip(skip).collect_vec();