release: v0.0.23

.github/workflows/release.yml

@@ -0,0 +1,291 @@
+# This file was autogenerated by dist: https://opensource.axo.dev/cargo-dist/
+#
+# Copyright 2022-2024, axodotdev
+# SPDX-License-Identifier: MIT or Apache-2.0
+#
+# CI that:
+#
+# * checks for a Git Tag that looks like a release
+# * builds artifacts with dist (archives, installers, hashes)
+# * uploads those artifacts to temporary workflow zip
+# * on success, uploads the artifacts to a GitHub Release
+#
+# Note that the GitHub Release will be created with a generated
+# title/body based on your changelogs.
+
+name: Release
+permissions:
+  "contents": "write"
+
+# This task will run whenever you push a git tag that looks like a version
+# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc.
+# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where
+# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION
+# must be a Cargo-style SemVer Version (must have at least major.minor.patch).
+#
+# If PACKAGE_NAME is specified, then the announcement will be for that
+# package (erroring out if it doesn't have the given version or isn't dist-able).
+#
+# If PACKAGE_NAME isn't specified, then the announcement will be for all
+# (dist-able) packages in the workspace with that version (this mode is
+# intended for workspaces with only one dist-able package, or with all dist-able
+# packages versioned/released in lockstep).
+#
+# If you push multiple tags at once, separate instances of this workflow will
+# spin up, creating an independent announcement for each one. However, GitHub
+# will hard limit this to 3 tags per commit, as it will assume more tags is a
+# mistake.
+#
+# If there's a prerelease-style suffix to the version, then the release(s)
+# will be marked as a prerelease.
+on:
+  pull_request:
+  push:
+    tags:
+      - '**[0-9]+.[0-9]+.[0-9]+*'
+
+jobs:
+  # Run 'dist plan' (or host) to determine what tasks we need to do
+  plan:
+    runs-on: "ubuntu-20.04"
+    outputs:
+      val: ${{ steps.plan.outputs.manifest }}
+      tag: ${{ !github.event.pull_request && github.ref_name || '' }}
+      tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }}
+      publishing: ${{ !github.event.pull_request }}
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install dist
+        # we specify bash to get pipefail; it guards against the `curl` command
+        # failing. otherwise `sh` won't catch that `curl` returned non-0
+        shell: bash
+        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.28.0/cargo-dist-installer.sh | sh"
+      - name: Cache dist
+        uses: actions/upload-artifact@v4
+        with:
+          name: cargo-dist-cache
+          path: ~/.cargo/bin/dist
+      # sure would be cool if github gave us proper conditionals...
+      # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
+      # functionality based on whether this is a pull_request, and whether it's from a fork.
+      # (PRs run on the *source* but secrets are usually on the *target* -- that's *good*
+      # but also really annoying to build CI around when it needs secrets to work right.)
+      - id: plan
+        run: |
+          dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json
+          echo "dist ran successfully"
+          cat plan-dist-manifest.json
+          echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
+      - name: "Upload dist-manifest.json"
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifacts-plan-dist-manifest
+          path: plan-dist-manifest.json
+
+  # Build and packages all the platform-specific things
+  build-local-artifacts:
+    name: build-local-artifacts (${{ join(matrix.targets, ', ') }})
+    # Let the initial task tell us to not run (currently very blunt)
+    needs:
+      - plan
+    if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }}
+    strategy:
+      fail-fast: false
+      # Target platforms/runners are computed by dist in create-release.
+      # Each member of the matrix has the following arguments:
+      #
+      # - runner: the github runner
+      # - dist-args: cli flags to pass to dist
+      # - install-dist: expression to run to install dist on the runner
+      #
+      # Typically there will be:
+      # - 1 "global" task that builds universal installers
+      # - N "local" tasks that build each platform's binaries and platform-specific installers
+      matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
+    runs-on: ${{ matrix.runner }}
+    container: ${{ matrix.container && matrix.container.image || null }}
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
+    steps:
+      - name: enable windows longpaths
+        run: |
+          git config --global core.longpaths true
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install Rust non-interactively if not already installed
+        if: ${{ matrix.container }}
+        run: |
+          if ! command -v cargo > /dev/null 2>&1; then
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+            echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+          fi
+      - name: Install dist
+        run: ${{ matrix.install_dist.run }}
+      # Get the dist-manifest
+      - name: Fetch local artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: artifacts-*
+          path: target/distrib/
+          merge-multiple: true
+      - name: Install dependencies
+        run: |
+          ${{ matrix.packages_install }}
+      - name: Build artifacts
+        run: |
+          # Actually do builds and make zips and whatnot
+          dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
+          echo "dist ran successfully"
+      - id: cargo-dist
+        name: Post-build
+        # We force bash here just because github makes it really hard to get values up
+        # to "real" actions without writing to env-vars, and writing to env-vars has
+        # inconsistent syntax between shell and powershell.
+        shell: bash
+        run: |
+          # Parse out what we just built and upload it to scratch storage
+          echo "paths<<EOF" >> "$GITHUB_OUTPUT"
+          dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT"
+          echo "EOF" >> "$GITHUB_OUTPUT"
+
+          cp dist-manifest.json "$BUILD_MANIFEST_NAME"
+      - name: "Upload artifacts"
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifacts-build-local-${{ join(matrix.targets, '_') }}
+          path: |
+            ${{ steps.cargo-dist.outputs.paths }}
+            ${{ env.BUILD_MANIFEST_NAME }}
+
+  # Build and package all the platform-agnostic(ish) things
+  build-global-artifacts:
+    needs:
+      - plan
+      - build-local-artifacts
+    runs-on: "ubuntu-20.04"
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install cached dist
+        uses: actions/download-artifact@v4
+        with:
+          name: cargo-dist-cache
+          path: ~/.cargo/bin/
+      - run: chmod +x ~/.cargo/bin/dist
+      # Get all the local artifacts for the global tasks to use (for e.g. checksums)
+      - name: Fetch local artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: artifacts-*
+          path: target/distrib/
+          merge-multiple: true
+      - id: cargo-dist
+        shell: bash
+        run: |
+          dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json
+          echo "dist ran successfully"
+
+          # Parse out what we just built and upload it to scratch storage
+          echo "paths<<EOF" >> "$GITHUB_OUTPUT"
+          jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
+          echo "EOF" >> "$GITHUB_OUTPUT"
+
+          cp dist-manifest.json "$BUILD_MANIFEST_NAME"
+      - name: "Upload artifacts"
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifacts-build-global
+          path: |
+            ${{ steps.cargo-dist.outputs.paths }}
+            ${{ env.BUILD_MANIFEST_NAME }}
+  # Determines if we should publish/announce
+  host:
+    needs:
+      - plan
+      - build-local-artifacts
+      - build-global-artifacts
+    # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine)
+    if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    runs-on: "ubuntu-20.04"
+    outputs:
+      val: ${{ steps.host.outputs.manifest }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install cached dist
+        uses: actions/download-artifact@v4
+        with:
+          name: cargo-dist-cache
+          path: ~/.cargo/bin/
+      - run: chmod +x ~/.cargo/bin/dist
+      # Fetch artifacts from scratch-storage
+      - name: Fetch artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: artifacts-*
+          path: target/distrib/
+          merge-multiple: true
+      - id: host
+        shell: bash
+        run: |
+          dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json
+          echo "artifacts uploaded and released successfully"
+          cat dist-manifest.json
+          echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
+      - name: "Upload dist-manifest.json"
+        uses: actions/upload-artifact@v4
+        with:
+          # Overwrite the previous copy
+          name: artifacts-dist-manifest
+          path: dist-manifest.json
+      # Create a GitHub Release while uploading all files to it
+      - name: "Download GitHub Artifacts"
+        uses: actions/download-artifact@v4
+        with:
+          pattern: artifacts-*
+          path: artifacts
+          merge-multiple: true
+      - name: Cleanup
+        run: |
+          # Remove the granular manifests
+          rm -f artifacts/*-dist-manifest.json
+      - name: Create GitHub Release
+        env:
+          PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}"
+          ANNOUNCEMENT_TITLE: "${{ fromJson(steps.host.outputs.manifest).announcement_title }}"
+          ANNOUNCEMENT_BODY: "${{ fromJson(steps.host.outputs.manifest).announcement_github_body }}"
+          RELEASE_COMMIT: "${{ github.sha }}"
+        run: |
+          # Write and read notes from a file to avoid quoting breaking things
+          echo "$ANNOUNCEMENT_BODY" > $RUNNER_TEMP/notes.txt
+
+          gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/*
+
+  announce:
+    needs:
+      - plan
+      - host
+    # use "always() && ..." to allow us to wait for all publish jobs while
+    # still allowing individual publish jobs to skip themselves (for prereleases).
+    # "host" however must run to completion, no skipping allowed!
+    if: ${{ always() && needs.host.result == 'success' }}
+    runs-on: "ubuntu-20.04"
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive

CHANGELOG.md

@@ -36,7 +36,7 @@ Added git tags for each version though Markdown won't display formatted on Githu
 Moved Sanakirja database wrapper to its own crate (`snkrj`) and added a robust auto defragmentation to improve disk usage without the need for user's intervention.
 Since it's not used anymore it will moved out of the repository relatively soon.
 
-# [v0.5.0](https://github.com/kibo-money/kibo/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
+# [kibo-v0.5.0](https://github.com/kibo-money/kibo/tree/eea56d394bf92c62c81da8b78b8c47ea730683f5) | [873199](https://mempool.space/block/0000000000000000000270925aa6a565be92e13164565a3f7994ca1966e48050) - 2024/12/04
 
 ![Image of the kibo Web App version 0.5.0](https://github.com/kibo-money/kibo/blob/main/_assets/v0.5.0.jpg)
 
@@ -103,7 +103,7 @@ Since it's not used anymore it will moved out of the repository relatively soon.
 
 - Moved back to this repo
 
-# [v0.4.0](https://github.com/kibo-money/kibo/tree/a64c544815d9ef785e2fc1323582f774f16b9200) | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
+# [kibo-v0.4.0](https://github.com/kibo-money/kibo/tree/a64c544815d9ef785e2fc1323582f774f16b9200) | [861950](https://mempool.space/block/00000000000000000000530d0e30ccf7deeace122dcc99f2668a06c6dad83629) - 2024/09/19
 
 ![Image of the kibo Web App version 0.4.0](https://github.com/kibo-money/kibo/blob/main/_assets/v0.4.0.jpg)
 
@@ -140,7 +140,7 @@ Since it's not used anymore it will moved out of the repository relatively soon.
 - Added serving of the website
 - Improved `Cache-Control` behavior
 
-# [v0.3.0](https://github.com/kibo-money/kibo/tree/b68b016091c45b071218fba01bac5b76e8eaf18c) | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26
+# [kibo-v0.3.0](https://github.com/kibo-money/kibo/tree/b68b016091c45b071218fba01bac5b76e8eaf18c) | [853930](https://mempool.space/block/00000000000000000002eb5e9a7950ca2d5d98bd1ed28fc9098aa630d417985d) - 2024/07/26
 
 ![Image of the Satonomics Web App version 0.3.0](https://github.com/kibo-money/kibo/blob/main/_assets/v0.3.0.jpg)
 
@@ -219,7 +219,7 @@ Since it's not used anymore it will moved out of the repository relatively soon.
   - Only run with a watcher if `cargo watch` is available
   - Removed id_to_path file in favor for only `paths.d.ts` in `app/src/types`
 
-# [v0.2.0](https://github.com/kibo-money/kibo/tree/248187889283597c5dbb806292297453c25e97b8) | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08
+# [kibo-v0.2.0](https://github.com/kibo-money/kibo/tree/248187889283597c5dbb806292297453c25e97b8) | [851286](https://mempool.space/block/0000000000000000000281ca7f1bf8c50702bfca168c7af1bdc67c977c1ac8ed) - 2024/07/08
 
 ![Image of the Satonomics Web App version 0.2.0](https://github.com/kibo-money/kibo/blob/main/_assets/v0.2.0.jpg)
 
@@ -255,7 +255,7 @@ Since it's not used anymore it will moved out of the repository relatively soon.
 
 - Fixed ulimit only being run in Mac OS instead of whenever the program is detected
 
-# [v0.1.1](https://github.com/kibo-money/kibo/tree/e55b5195a9de9aea306903c94ed63cb1720fda5f) | [849240](https://mempool.space/block/000000000000000000002b8653988655071c07bb5f7181c038f9326bc86db741) - 2024/06/24
+# [kibo-v0.1.1](https://github.com/kibo-money/kibo/tree/e55b5195a9de9aea306903c94ed63cb1720fda5f) | [849240](https://mempool.space/block/000000000000000000002b8653988655071c07bb5f7181c038f9326bc86db741) - 2024/06/24
 
 ![Image of the Satonomics Web App version 0.1.1](https://github.com/kibo-money/kibo/blob/main/_assets/v0.1.1.jpg)
 
@@ -305,10 +305,10 @@ Since it's not used anymore it will moved out of the repository relatively soon.
 
 - Deleted old price datasets and their backups
 
-# [v0.1.0](https://github.com/kibo-money/kibo/tree/a1a576d088c8f83ed32d48753a7611f70a964574) | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19
+# [kibo-v0.1.0](https://github.com/kibo-money/kibo/tree/a1a576d088c8f83ed32d48753a7611f70a964574) | [848642](https://mempool.space/block/000000000000000000020be5761d70751252219a9557f55e91ecdfb86c4e026a) - 2024/06/19
 
 ![Image of the Satonomics Web App version 0.1.0](https://github.com/kibo-money/kibo/blob/main/_assets/v0.1.0.jpg)
 
-# v0.0.1 | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20
+# kibo-v0.0.1 | [835444](https://mempool.space/block/000000000000000000009f93907a0dd83c080d5585cc7ec82c076d45f6d7c872) - 2024/03/20
 
 ![Image of the Satonomics Web App version 0.0.X](https://github.com/kibo-money/kibo/blob/main/_assets/v0.0.X.jpg)

Cargo.lock

@@ -138,6 +138,12 @@ dependencies = [
  "derive_arbitrary",
 ]
 
+[[package]]
+name = "arc-swap"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
+
 [[package]]
 name = "arrayvec"
 version = "0.7.6"
@@ -368,7 +374,7 @@ dependencies = [
 
 [[package]]
 name = "brk"
-version = "0.0.13"
+version = "0.0.23"
 dependencies = [
  "brk_cli",
  "brk_computer",
@@ -385,7 +391,7 @@ dependencies = [
 
 [[package]]
 name = "brk_cli"
-version = "0.0.13"
+version = "0.0.23"
 dependencies = [
  "brk_computer",
  "brk_core",
@@ -406,7 +412,7 @@ dependencies = [
 
 [[package]]
 name = "brk_computer"
-version = "0.0.13"
+version = "0.0.23"
 dependencies = [
  "brk_core",
  "brk_exit",
@@ -421,7 +427,7 @@ dependencies = [
 
 [[package]]
 name = "brk_core"
-version = "0.0.13"
+version = "0.0.23"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -438,7 +444,7 @@ dependencies = [
 
 [[package]]
 name = "brk_exit"
-version = "0.0.13"
+version = "0.0.23"
 dependencies = [
  "brk_logger",
  "ctrlc",
@@ -447,7 +453,7 @@ dependencies = [
 
 [[package]]
 name = "brk_fetcher"
-version = "0.0.13"
+version = "0.0.23"
 dependencies = [
  "brk_core",
  "brk_logger",
@@ -460,7 +466,7 @@ dependencies = [
 
 [[package]]
 name = "brk_indexer"
-version = "0.0.13"
+version = "0.0.23"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -479,7 +485,7 @@ dependencies = [
 
 [[package]]
 name = "brk_logger"
-version = "0.0.13"
+version = "0.0.23"
 dependencies = [
  "color-eyre",
  "env_logger",
@@ -489,7 +495,7 @@ dependencies = [
 
 [[package]]
 name = "brk_parser"
-version = "0.0.13"
+version = "0.0.23"
 dependencies = [
  "bitcoin",
  "bitcoincore-rpc",
@@ -504,7 +510,7 @@ dependencies = [
 
 [[package]]
 name = "brk_query"
-version = "0.0.13"
+version = "0.0.23"
 dependencies = [
  "brk_computer",
  "brk_indexer",
@@ -520,7 +526,7 @@ dependencies = [
 
 [[package]]
 name = "brk_server"
-version = "0.0.13"
+version = "0.0.23"
 dependencies = [
  "axum",
  "brk_computer",
@@ -541,14 +547,15 @@ dependencies = [
  "tokio",
  "tower-http",
  "tracing",
- "tracing-subscriber",
  "zip",
 ]
 
 [[package]]
 name = "brk_vec"
-version = "0.0.13"
+version = "0.0.23"
 dependencies = [
+ "arc-swap",
+ "axum",
  "memmap2",
  "rayon",
  "serde",
@@ -641,9 +648,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.17"
+version = "1.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a"
+checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362"
 dependencies = [
  "jobserver",
  "libc",
@@ -687,9 +694,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.35"
+version = "4.5.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944"
+checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -697,9 +704,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.35"
+version = "4.5.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9"
+checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5"
 dependencies = [
  "anstream",
  "anstyle",
@@ -844,9 +851,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.14"
+version = "0.5.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471"
+checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -907,9 +914,9 @@ dependencies = [
 
 [[package]]
 name = "ctrlc"
-version = "3.4.5"
+version = "3.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3"
+checksum = "697b5419f348fd5ae2478e8018cb016c00a5881c7f46c717de98ffd135a5651c"
 dependencies = [
  "nix",
  "windows-sys 0.59.0",
@@ -972,9 +979,9 @@ checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b"
 
 [[package]]
 name = "deranged"
-version = "0.4.1"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058"
+checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e"
 dependencies = [
  "powerfmt",
  "serde",
@@ -1068,9 +1075,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
 
 [[package]]
 name = "errno"
-version = "0.3.10"
+version = "0.3.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
+checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e"
 dependencies = [
  "libc",
  "windows-sys 0.59.0",
@@ -1122,7 +1129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece"
 dependencies = [
  "crc32fast",
- "miniz_oxide 0.8.5",
+ "miniz_oxide 0.8.8",
 ]
 
 [[package]]
@@ -1413,9 +1420,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.8.0"
+version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058"
+checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e"
 dependencies = [
  "equivalent",
  "hashbrown 0.15.2",
@@ -1463,9 +1470,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
 
 [[package]]
 name = "jiff"
-version = "0.2.5"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c102670231191d07d37a35af3eb77f1f0dbf7a71be51a962dcd57ea607be7260"
+checksum = "e5ad87c89110f55e4cd4dc2893a9790820206729eaf221555f742d540b0724a0"
 dependencies = [
  "jiff-static",
  "jiff-tzdb-platform",
@@ -1478,9 +1485,9 @@ dependencies = [
 
 [[package]]
 name = "jiff-static"
-version = "0.2.5"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4cdde31a9d349f1b1f51a0b3714a5940ac022976f4b49485fc04be052b183b4c"
+checksum = "d076d5b64a7e2fe6f0743f02c43ca4a6725c0f904203bfe276a5b3e793103605"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1548,9 +1555,9 @@ checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.9.3"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413"
+checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
 
 [[package]]
 name = "lock_api"
@@ -1663,18 +1670,18 @@ dependencies = [
 
 [[package]]
 name = "miniz_oxide"
-version = "0.8.5"
+version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5"
+checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a"
 dependencies = [
  "adler2",
 ]
 
 [[package]]
 name = "minreq"
-version = "2.13.3"
+version = "2.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "567496f13503d6cae8c9f961f34536850275f396307d7a6b981eef1464032f53"
+checksum = "f0d2aaba477837b46ec1289588180fabfccf0c3b1d1a0c6b1866240cd6cd5ce9"
 dependencies = [
  "log",
  "rustls",
@@ -1713,16 +1720,6 @@ version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51"
 
-[[package]]
-name = "nu-ansi-term"
-version = "0.46.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
-dependencies = [
- "overload",
- "winapi",
-]
-
 [[package]]
 name = "num-bigint"
 version = "0.4.6"
@@ -1778,12 +1775,6 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
 
-[[package]]
-name = "overload"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
-
 [[package]]
 name = "owo-colors"
 version = "3.5.0"
@@ -1798,9 +1789,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564"
 
 [[package]]
 name = "oxc"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c274a11ab2471eea5f970d943ecb7b64dc9fa21d89ac5c968fc0f48ca9971196"
+checksum = "fa680279066565502d5cc612d82dc2841499518f5430d047aff33869ec5592a5"
 dependencies = [
  "oxc_allocator",
  "oxc_ast",
@@ -1841,9 +1832,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_allocator"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fef6db9c542af8b1b0889c4fdbc4dfd0a13c5bec0f94ef39bb826084465d6b1e"
+checksum = "3f90c485c4f2781e0c8b4703a5fa205f49ecc6d496d2b88260db27d60f9f1661"
 dependencies = [
  "allocator-api2",
  "bumpalo",
@@ -1855,9 +1846,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_ast"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c269cf713bed18d74e957045d2b1cf57827333a66aad2b2c136c9f8b79940bd2"
+checksum = "2552b2248c0d320c97f865d0620946821e687a0c02200cc6aa08161862447f8d"
 dependencies = [
  "bitflags",
  "cow-utils",
@@ -1872,9 +1863,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_ast_macros"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1d8afe9d7e4aed37d6e8906250db4872f514c15a244268ce3da09ccb4017900"
+checksum = "ccd0b70b7752610fc00a22e3cbf92fa2b0a007b022a976fba75a254f6ea21f18"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1883,9 +1874,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_ast_visit"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d360f012f970b3e79f8f9863abb83b373014eb9d1a23fecd7ce33e555cc3151"
+checksum = "ef6a46d3d3158668e5d8a9231bce6f5c8abc4a818a48583a254f4aa4a9682fa0"
 dependencies = [
  "oxc_allocator",
  "oxc_ast",
@@ -1895,9 +1886,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_cfg"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17a48085dd0ca7e8f3ad946a131d8229af783be65cbf4965ff6379e432f6b625"
+checksum = "e6e0e5fa83ea1b3bbcc5496cf6acbbc46f7d147cbd342ecf968df500239c73bd"
 dependencies = [
  "bitflags",
  "itertools",
@@ -1910,9 +1901,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_codegen"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1e6d4e1c953205c62effeb61c1da684dd1668feaf21b4fe15c2b603526355f6"
+checksum = "454e75b9152f3aea41e4de64154bcb4a261a50df75746b08136b9994130f3a00"
 dependencies = [
  "bitflags",
  "cow-utils",
@@ -1931,15 +1922,15 @@ dependencies = [
 
 [[package]]
 name = "oxc_data_structures"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e2866696e6bb90151c5687a961c3ff1fa2726436869d42eea14cf2d5c663590"
+checksum = "c77ea3b4eae572b066a314edd33ccd07e0d0b148bdea635fdd1a44ff97b1a9fb"
 
 [[package]]
 name = "oxc_diagnostics"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d7800336d376a2baeafb9d2bcdc9fbef254863c33810e6d1cc4a431cd0048ef"
+checksum = "f5f38bf4f67832d521bcd99a79d4483e9b571d37c81e6b09a0a397b18931df8b"
 dependencies = [
  "cow-utils",
  "oxc-miette",
@@ -1947,9 +1938,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_ecmascript"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0edf650adfacf84768cdfeca874d8f5ffebfd108ee8a44959c0eca82b04321c6"
+checksum = "53a754f033637a086a80d7f81620ff1123f20ad70789f37873bc635ec89f0021"
 dependencies = [
  "cow-utils",
  "num-bigint",
@@ -1961,9 +1952,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_estree"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd77f6588d373c1b753588b784de380a7418f8e46776e25c4564739994b9a5a6"
+checksum = "85b7edeb47c4700027cc113d22ed899df45fda2ed5b29af2e471cad2be09244d"
 
 [[package]]
 name = "oxc_index"
@@ -1973,9 +1964,9 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392"
 
 [[package]]
 name = "oxc_mangler"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02f00ca9e544230f01d37de50ba710136763a13804303bcf70cf22d8155f9db5"
+checksum = "8a837c659bb2e340f491d2196d4565d8e75ffa19e7a8c6b35eeab67159cda546"
 dependencies = [
  "fixedbitset",
  "itertools",
@@ -1990,9 +1981,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_minifier"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8670c4452b9f78012b9ede921e40f91686ad9a1af1c40aab8ee3a5f4ef8380ef"
+checksum = "d186a202e49cebf004903e45eccecc3fc8bd10d4f6803ad73eb9eba51b4770b4"
 dependencies = [
  "cow-utils",
  "oxc_allocator",
@@ -2012,9 +2003,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_parser"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8eaebe1ac01073b2a43a8848ae7ec24fdad3813855d72bb1942908632654f94"
+checksum = "a54ce76e891cdfa2bf1b946492e94f17a5d6a5d9731e59aaa1b4ae435f8439ef"
 dependencies = [
  "bitflags",
  "cow-utils",
@@ -2035,9 +2026,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_regular_expression"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9ad8833ffba010e9f4cced1482297282728b0dc1c410cd716f206d214250f3f"
+checksum = "50d43967a36f4ce1577099e3d70927845fb9f1cbfdd57298e7fd69897c1f434d"
 dependencies = [
  "oxc_allocator",
  "oxc_ast_macros",
@@ -2051,9 +2042,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_semantic"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0523c9521e3ee63d6e0b463941e030b482977d7c323ee802eebacac8e5227bc8"
+checksum = "2c8f0eb991e07aa1aab2efd85f61879452374d406145bdc6d32681ddda8d74c8"
 dependencies = [
  "itertools",
  "oxc_allocator",
@@ -2087,9 +2078,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_span"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bcfa55b4e3de93d39528b4ac616c00918dc73b6f6828403b3c56a7dde7af23b"
+checksum = "2cdfaea137d47979b406de82e3b3c13f03b4bda4bc237c1b6fdbb8c46b436e70"
 dependencies = [
  "compact_str",
  "oxc-miette",
@@ -2100,9 +2091,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_syntax"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7bc0d1f63b254791867280db5e582d7411c112f0798eee50bfdb8cdd2ef80c93"
+checksum = "eb675497fc1051b3cec9c5cfa224ead942a830c8583394deaadcae5eb54669fa"
 dependencies = [
  "bitflags",
  "cow-utils",
@@ -2121,9 +2112,9 @@ dependencies = [
 
 [[package]]
 name = "oxc_traverse"
-version = "0.62.0"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a1f5183b3db89b3b3c7621741d5e2b9c13996dec5be4ad5e7adf86fdcfd24f0"
+checksum = "f89e6801eb9ef451e1c785fdee237050be91da172b9fbcad849f9703d37a8ae8"
 dependencies = [
  "compact_str",
  "itoa",
@@ -2213,7 +2204,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772"
 dependencies = [
  "fixedbitset",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
 ]
 
 [[package]]
@@ -2420,9 +2411,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.10"
+version = "0.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1"
+checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3"
 dependencies = [
  "bitflags",
 ]
@@ -2583,9 +2574,9 @@ dependencies = [
 
 [[package]]
 name = "self_cell"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2fdfc24bc566f839a2da4c4295b82db7d25a24253867d5c64355abb5799bdbe"
+checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"
 
 [[package]]
 name = "seq-macro"
@@ -2675,7 +2666,7 @@ dependencies = [
  "chrono",
  "hex",
  "indexmap 1.9.3",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
  "serde",
  "serde_derive",
  "serde_json",
@@ -2750,9 +2741,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
 
 [[package]]
 name = "smallvec"
-version = "1.14.0"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd"
+checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
 
 [[package]]
 name = "smawk"
@@ -2932,9 +2923,9 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.44.1"
+version = "1.44.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a"
+checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48"
 dependencies = [
  "backtrace",
  "bytes",
@@ -2999,7 +2990,7 @@ version = "0.22.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
 dependencies = [
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
  "serde",
  "serde_spanned",
  "toml_datetime",
@@ -3097,29 +3088,15 @@ dependencies = [
  "tracing-subscriber",
 ]
 
-[[package]]
-name = "tracing-log"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
-dependencies = [
- "log",
- "once_cell",
- "tracing-core",
-]
-
 [[package]]
 name = "tracing-subscriber"
 version = "0.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
 dependencies = [
- "nu-ansi-term",
  "sharded-slab",
- "smallvec",
  "thread_local",
  "tracing-core",
- "tracing-log",
 ]
 
 [[package]]
@@ -3284,28 +3261,6 @@ version = "0.25.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
 
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-
 [[package]]
 name = "windows-core"
 version = "0.61.0"
@@ -3449,9 +3404,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
-version = "0.7.4"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36"
+checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10"
 dependencies = [
  "memchr",
 ]
@@ -3522,9 +3477,9 @@ dependencies = [
 
 [[package]]
 name = "zip"
-version = "2.5.0"
+version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27c03817464f64e23f6f37574b4fdc8cf65925b5bfd2b0f2aedf959791941f88"
+checksum = "1dcb24d0152526ae49b9b96c1dcf71850ca1e0b882e4e28ed898a93c41334744"
 dependencies = [
  "aes",
  "arbitrary",
@@ -3536,7 +3491,7 @@ dependencies = [
  "flate2",
  "getrandom 0.3.2",
  "hmac",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
  "lzma-rs",
  "memchr",
  "pbkdf2",

Cargo.toml

@@ -4,7 +4,7 @@ members = ["crates/*"]
 package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
 package.license = "MIT"
 package.edition = "2024"
-package.version = "0.0.13"
+package.version = "0.0.23"
 package.repository = "https://github.com/bitcoinresearchkit/brk"
 
 [profile.release]
@@ -12,7 +12,11 @@ lto = "fat"
 codegen-units = 1
 panic = "abort"
 
+[profile.dist]
+inherits = "release"
+
 [workspace.dependencies]
+axum = "0.8.3"
 bitcoin = { version = "0.32.5", features = ["serde"] }
 bitcoincore-rpc = "0.19.0"
 brk_cli = { version = "0", path = "crates/brk_cli" }
@@ -27,15 +31,32 @@ brk_query = { version = "0", path = "crates/brk_query" }
 brk_server = { version = "0", path = "crates/brk_server" }
 brk_vec = { version = "0", path = "crates/brk_vec" }
 byteview = "0.6.1"
-clap = { version = "4.5.35", features = ["derive", "string"] }
+clap = { version = "4.5.36", features = ["derive", "string"] }
 color-eyre = "0.6.3"
 derive_deref = "1.1.1"
 fjall = "2.8.0"
-jiff = "0.2.5"
+jiff = "0.2.8"
 log = { version = "0.4.27" }
-minreq = { version = "2.13.3", features = ["https", "serde_json"] }
+minreq = { version = "2.13.4", features = ["https", "serde_json"] }
 rayon = "1.10.0"
 serde = { version = "1.0.219", features = ["derive"] }
 serde_json = { version = "1.0.140", features = ["float_roundtrip"] }
 tabled = "0.18.0"
 zerocopy = { version = "0.8.24", features = ["derive"] }
+
+[workspace.metadata.release]
+shared-version = true
+tag-name = "v{{version}}"
+pre-release-commit-message = "release: v{{version}}"
+tag-message = "release: v{{version}}"
+
+[workspace.metadata.dist]
+cargo-dist-version = "0.28.0"
+ci = "github"
+installers = []
+targets = [
+    "aarch64-apple-darwin",
+    "aarch64-unknown-linux-gnu",
+    "x86_64-apple-darwin",
+    "x86_64-unknown-linux-gnu",
+]

README.md

@@ -20,7 +20,7 @@
   <a href="https://deps.rs/crate/brk">
     <img src="https://deps.rs/crate/brk/latest/status.svg" alt="Dependency status">
   </a>
-  <a href="https://discord.gg/Cvrwpv3zEG">
+  <a href="https://discord.gg/HaR3wpH3nr">
     <img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
   </a>
   <a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
@@ -58,6 +58,7 @@ The toolkit can be used in various ways to accommodate as many needs as possible
   For more information visit: [`brk_cli`](https://crates.io/crates/brk_cli)
 - **[Crates](https://crates.io/crates/brk)** \
   Rust developers have access to a wide range crates, each built upon one another with its own specific purpose, enabling independent use and offering great flexibility.
+  PRs are welcome, especially if their goal is to introduce additional datasets.
 
 The primary goal of this project is to be fully-featured and accessible for everyone, regardless of their background or financial situation - whether that person is an enthusiast, researcher, miner, analyst, or simply curious.
 
@@ -76,7 +77,7 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
 - [`brk_parser`](https://crates.io/crates/brk_parser): A very fast Bitcoin Core block parser and iterator built on top of bitcoin-rust
 - [`brk_query`](https://crates.io/crates/brk_query): A library that finds requested datasets.
 - [`brk_server`](https://crates.io/crates/brk_server): A server that serves Bitcoin data and swappable front-ends, built on top of `brk_indexer`, `brk_fetcher` and `brk_computer`
-- [`brk_vec`](https://crates.io/crates/brk_vec): A very small, fast, efficient and simple storable Vec.
+- [`brk_vec`](https://crates.io/crates/brk_vec): A push-only, truncable, compressable, saveable Vec
 
 ## Acknowledgments
 

crates/brk_cli/Cargo.toml

@@ -26,3 +26,6 @@ toml = "0.8.20"
 [[bin]]
 name = "brk"
 path = "src/main.rs"
+
+[package.metadata.dist]
+dist = false

crates/brk_cli/README.md

@@ -20,7 +20,7 @@
   <a href="https://deps.rs/crate/brk_cli">
     <img src="https://deps.rs/crate/brk_cli/latest/status.svg" alt="Dependency status">
   </a>
-  <a href="https://discord.gg/Cvrwpv3zEG">
+  <a href="https://discord.gg/HaR3wpH3nr">
     <img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
   </a>
   <a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
@@ -59,16 +59,28 @@ To be determined
 - Unix based operating system (Mac OS or Linux)
   - Ubuntu users need to install `open-ssl` via `sudo apt install libssl-dev pkg-config`
 
-## Install
+## Download
+
+### Binaries
+
+You can find a pre-built binary for your operating system on the releases page ([link](https://github.com/bitcoinresearchkit/brk/releases/latest)).
+
+### Cargo
 
 ```bash
+# Install
 cargo install brk # or `cargo install brk_cli`, the result is the same
+
+# Update
+cargo install brk # or `cargo install-update -a` if you have `cargo-update` installed
 ```
 
-## Update
+### Source
 
 ```bash
-cargo install brk # or `cargo install-update -a` if you have `cargo-update` installed
+git clone https://github.com/bitcoinresearchkit/brk.git
+cd brk/crates/brk
+cargo run -r
 ```
 
 ## Usage

crates/brk_cli/src/run.rs

@@ -58,10 +58,28 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
             };
 
             if config.process() {
+                let wait_for_synced_node = || -> color_eyre::Result<()> {
+                    let is_synced = || -> color_eyre::Result<bool> {
+                        let info = rpc.get_blockchain_info()?;
+                        Ok(info.headers == info.blocks)
+                    };
+
+                    if !is_synced()? {
+                        info!("Waiting for node to be synced...");
+                        while !is_synced()? {
+                            sleep(Duration::from_secs(1))
+                        }
+                    }
+
+                    Ok(())
+                };
+
                 loop {
+                    wait_for_synced_node()?;
+
                     let block_count = rpc.get_block_count()?;
 
-                    info!("{block_count} blocks found.");
+                    info!("{} blocks found.", block_count + 1);
 
                     let starting_indexes = indexer.index(&parser, rpc, &exit)?;
 
@@ -272,9 +290,10 @@ impl RunConfig {
     }
 
     fn read(path: &Path) -> Self {
-        fs::read_to_string(path).map_or(RunConfig::default(), |contents| {
+        fs::read_to_string(path).map_or_else(
-            toml::from_str(&contents).unwrap_or_default()
+            |_| RunConfig::default(),
-        })
+            |contents| toml::from_str(&contents).unwrap_or_default(),
+        )
     }
 
     fn write(&self, path: &Path) -> std::io::Result<()> {

crates/brk_computer/README.md

@@ -20,7 +20,7 @@
   <a href="https://deps.rs/crate/brk_computer">
     <img src="https://deps.rs/crate/brk_computer/latest/status.svg" alt="Dependency status">
   </a>
-  <a href="https://discord.gg/Cvrwpv3zEG">
+  <a href="https://discord.gg/HaR3wpH3nr">
     <img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
   </a>
   <a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">

crates/brk_computer/src/storage/stores.rs

@@ -13,9 +13,9 @@ pub struct Stores {
 impl Stores {
     pub fn import(path: &Path) -> color_eyre::Result<Self> {
         let address_to_utxos_received =
-            Store::import(&path.join("address_to_utxos_received"), Version::ONE)?;
+            Store::import(&path.join("address_to_utxos_received"), Version::ZERO)?;
         let address_to_utxos_spent =
-            Store::import(&path.join("address_to_utxos_spent"), Version::ONE)?;
+            Store::import(&path.join("address_to_utxos_spent"), Version::ZERO)?;
 
         Ok(Self {
             address_to_utxos_received,

crates/brk_computer/src/storage/vecs/base.rs

@@ -2,38 +2,49 @@ use core::error;
 use std::{
     cmp::Ordering,
     fmt::Debug,
-    io,
+    ops::Add,
-    ops::{Add, Sub},
     path::{Path, PathBuf},
 };
 
 use brk_core::CheckedSub;
 use brk_exit::Exit;
-use brk_vec::{AnyStorableVec, Compressed, Error, Result, StoredIndex, StoredType, Version};
+use brk_vec::{
+    Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec, Value,
+    Version,
+};
+use log::info;
 
-const FLUSH_EVERY: usize = 10_000;
+const ONE_KIB: usize = 1024;
+const ONE_MIB: usize = ONE_KIB * ONE_KIB;
+const MAX_CACHE_SIZE: usize = 210 * ONE_MIB;
 
 #[derive(Debug)]
-pub struct StorableVec<I, T> {
+pub struct ComputedVec<I, T>
-    computed_version: Option<Version>,
-    vec: brk_vec::StorableVec<I, T>,
-}
-
-impl<I, T> StorableVec<I, T>
 where
     I: StoredIndex,
     T: StoredType,
 {
+    computed_version: Option<Version>,
+    inner: StoredVec<I, T>,
+}
+
+impl<I, T> ComputedVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    const SIZE_OF: usize = size_of::<T>();
+
     pub fn forced_import(
         path: &Path,
         version: Version,
         compressed: Compressed,
     ) -> brk_vec::Result<Self> {
-        let vec = brk_vec::StorableVec::forced_import(path, version, compressed)?;
+        let inner = StoredVec::forced_import(path, version, compressed)?;
 
         Ok(Self {
             computed_version: None,
-            vec,
+            inner,
         })
     }
 
@@ -42,7 +53,7 @@ where
             return Ok(());
         }
         exit.block();
-        self.vec.truncate_if_needed(index)?;
+        self.inner.truncate_if_needed(index)?;
         exit.release();
         Ok(())
     }
@@ -57,102 +68,118 @@ where
                 if ord == Ordering::Greater {
                     self.safe_truncate_if_needed(index, exit)?;
                 }
-                self.vec.push(value);
+                self.inner.push(value);
             }
         }
 
-        if self.vec.pushed_len() >= FLUSH_EVERY {
+        if self.inner.pushed_len() * Self::SIZE_OF >= MAX_CACHE_SIZE {
-            Ok(self.safe_flush(exit)?)
+            self.safe_flush(exit)
         } else {
             Ok(())
         }
     }
 
-    pub fn safe_flush(&mut self, exit: &Exit) -> io::Result<()> {
+    pub fn safe_flush(&mut self, exit: &Exit) -> Result<()> {
         if exit.triggered() {
             return Ok(());
         }
         exit.block();
-        self.vec.flush()?;
+        self.inner.flush()?;
         exit.release();
         Ok(())
     }
 
     fn version(&self) -> Version {
-        self.vec.version()
+        self.inner.version()
     }
 
     pub fn len(&self) -> usize {
-        self.vec.len()
+        self.inner.len()
+    }
+    pub fn is_empty(&self) -> bool {
+        self.inner.is_empty()
     }
 
-    pub fn vec(&self) -> &brk_vec::StorableVec<I, T> {
+    fn file_name(&self) -> String {
-        &self.vec
+        self.inner.file_name()
     }
 
-    pub fn mut_vec(&mut self) -> &mut brk_vec::StorableVec<I, T> {
+    pub fn vec(&self) -> &StoredVec<I, T> {
-        &mut self.vec
+        &self.inner
     }
 
-    pub fn any_vec(&self) -> &dyn AnyStorableVec {
+    pub fn mut_vec(&mut self) -> &mut StoredVec<I, T> {
-        &self.vec
+        &mut self.inner
     }
 
-    pub fn mut_any_vec(&mut self) -> &mut dyn AnyStorableVec {
+    pub fn any_vec(&self) -> &dyn brk_vec::AnyStoredVec {
-        &mut self.vec
+        &self.inner
     }
 
-    pub fn get(&mut self, index: I) -> Result<Option<&T>> {
+    pub fn mut_any_vec(&mut self) -> &mut dyn brk_vec::AnyStoredVec {
-        self.vec.get(index)
+        &mut self.inner
     }
 
-    pub fn collect_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<T>> {
+    pub fn cached_get(&mut self, index: I) -> Result<Option<Value<T>>> {
-        self.vec.collect_range(from, to)
+        self.inner.cached_get(index)
+    }
+
+    pub fn collect_inclusive_range(&self, from: I, to: I) -> Result<Vec<T>> {
+        self.inner.collect_inclusive_range(from, to)
+    }
+
+    pub fn path(&self) -> &Path {
+        self.inner.path()
     }
 
     #[inline]
     fn path_computed_version(&self) -> PathBuf {
-        self.vec.path().join("computed_version")
+        self.inner.path().join("computed_version")
     }
 
     fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
         let path = self.path_computed_version();
         if version.validate(path.as_ref()).is_err() {
-            self.vec.reset()?;
+            self.inner.reset()?;
         }
         version.write(path.as_ref())?;
+
+        if self.is_empty() {
+            info!("Computing {}...", self.file_name())
+        }
+
         Ok(())
     }
 
     pub fn compute_transform<A, B, F>(
         &mut self,
         max_from: A,
-        other: &mut brk_vec::StorableVec<A, B>,
+        other: &mut StoredVec<A, B>,
         mut t: F,
         exit: &Exit,
     ) -> Result<()>
     where
         A: StoredIndex,
         B: StoredType,
-        F: FnMut((A, B, &mut Self, &mut brk_vec::StorableVec<A, B>)) -> (I, T),
+        F: FnMut((A, B, &mut Self, &mut dyn DynamicVec<I = A, T = B>)) -> (I, T),
     {
         self.validate_computed_version_or_reset_file(
             Version::ZERO + self.version() + other.version(),
         )?;
 
         let index = max_from.min(A::from(self.len()));
-        other.iter_from_cloned(index, |(a, b, other)| {
+        other.iter_from(index, |(a, b, other)| {
             let (i, v) = t((a, b, self, other));
             self.forced_push_at(i, v, exit)
         })?;
 
-        Ok(self.safe_flush(exit)?)
+        self.safe_flush(exit)
     }
 
     pub fn compute_inverse_more_to_less(
         &mut self,
         max_from: T,
-        other: &mut brk_vec::StorableVec<T, I>,
+        other: &mut StoredVec<T, I>,
         exit: &Exit,
     ) -> Result<()>
     where
@@ -163,24 +190,27 @@ where
             Version::ZERO + self.version() + other.version(),
         )?;
 
-        let index = max_from.min(self.vec.get_last()?.cloned().unwrap_or_default());
+        let index = max_from.min(
+            self.inner
+                .cached_get_last()?
+                .map_or_else(T::default, |v| v.into_inner()),
+        );
         other.iter_from(index, |(v, i, ..)| {
-            let i = *i;
+            if self.cached_get(i).unwrap().is_none_or(|old_v| *old_v > v) {
-            if self.get(i).unwrap().is_none_or(|old_v| *old_v > v) {
                 self.forced_push_at(i, v, exit)
             } else {
                 Ok(())
             }
         })?;
 
-        Ok(self.safe_flush(exit)?)
+        self.safe_flush(exit)
     }
 
     pub fn compute_inverse_less_to_more(
         &mut self,
         max_from: T,
-        first_indexes: &mut brk_vec::StorableVec<T, I>,
+        first_indexes: &mut StoredVec<T, I>,
-        last_indexes: &mut brk_vec::StorableVec<T, I>,
+        last_indexes: &mut StoredVec<T, I>,
         exit: &Exit,
     ) -> Result<()>
     where
@@ -194,18 +224,18 @@ where
         let index = max_from.min(T::from(self.len()));
         first_indexes.iter_from(index, |(value, first_index, ..)| {
             let first_index = (first_index).to_usize()?;
-            let last_index = (last_indexes.get(value)?.unwrap()).to_usize()?;
+            let last_index = (last_indexes.cached_get(value)?.unwrap()).to_usize()?;
             (first_index..last_index)
                 .try_for_each(|index| self.forced_push_at(I::from(index), value, exit))
         })?;
 
-        Ok(self.safe_flush(exit)?)
+        self.safe_flush(exit)
     }
 
     pub fn compute_last_index_from_first(
         &mut self,
         max_from: I,
-        first_indexes: &mut brk_vec::StorableVec<I, T>,
+        first_indexes: &mut StoredVec<I, T>,
         final_len: usize,
         exit: &Exit,
     ) -> Result<()>
@@ -219,11 +249,12 @@ where
         let index = max_from.min(I::from(self.len()));
         let one = T::from(1);
         let mut prev_index: Option<I> = None;
-        first_indexes.iter_from(index, |(i, v, ..)| {
+        first_indexes.iter_from(index, |(index, v, ..)| {
             if let Some(prev_index) = prev_index.take() {
-                self.forced_push_at(prev_index, v.checked_sub(one).unwrap(), exit)?;
+                let value = v.checked_sub(one).unwrap();
+                self.forced_push_at(prev_index, value, exit)?;
             }
-            prev_index.replace(i);
+            prev_index.replace(index);
             Ok(())
         })?;
         if let Some(prev_index) = prev_index {
@@ -234,19 +265,77 @@ where
             )?;
         }
 
-        Ok(self.safe_flush(exit)?)
+        self.safe_flush(exit)
     }
 
     pub fn compute_count_from_indexes<T2>(
         &mut self,
         max_from: I,
-        first_indexes: &mut brk_vec::StorableVec<I, T2>,
+        first_indexes: &mut StoredVec<I, T2>,
-        last_indexes: &mut brk_vec::StorableVec<I, T2>,
+        last_indexes: &mut StoredVec<I, T2>,
         exit: &Exit,
     ) -> Result<()>
     where
         T: From<T2>,
-        T2: StoredType + Copy + Add<usize, Output = T2> + CheckedSub<T2> + TryInto<T> + Default,
+        T2: StoredType
+            + StoredIndex
+            + Copy
+            + Add<usize, Output = T2>
+            + CheckedSub<T2>
+            + TryInto<T>
+            + Default,
+        <T2 as TryInto<T>>::Error: error::Error + 'static,
+    {
+        let opt: Option<Box<dyn FnMut(T2) -> bool>> = None;
+        self.compute_filtered_count_from_indexes_(max_from, first_indexes, last_indexes, opt, exit)
+    }
+
+    pub fn compute_filtered_count_from_indexes<T2, F>(
+        &mut self,
+        max_from: I,
+        first_indexes: &mut StoredVec<I, T2>,
+        last_indexes: &mut StoredVec<I, T2>,
+        filter: F,
+        exit: &Exit,
+    ) -> Result<()>
+    where
+        T: From<T2>,
+        T2: StoredType
+            + StoredIndex
+            + Copy
+            + Add<usize, Output = T2>
+            + CheckedSub<T2>
+            + TryInto<T>
+            + Default,
+        <T2 as TryInto<T>>::Error: error::Error + 'static,
+        F: FnMut(T2) -> bool,
+    {
+        self.compute_filtered_count_from_indexes_(
+            max_from,
+            first_indexes,
+            last_indexes,
+            Some(Box::new(filter)),
+            exit,
+        )
+    }
+
+    fn compute_filtered_count_from_indexes_<T2>(
+        &mut self,
+        max_from: I,
+        first_indexes: &mut StoredVec<I, T2>,
+        last_indexes: &mut StoredVec<I, T2>,
+        mut filter: Option<Box<dyn FnMut(T2) -> bool + '_>>,
+        exit: &Exit,
+    ) -> Result<()>
+    where
+        T: From<T2>,
+        T2: StoredType
+            + StoredIndex
+            + Copy
+            + Add<usize, Output = T2>
+            + CheckedSub<T2>
+            + TryInto<T>
+            + Default,
         <T2 as TryInto<T>>::Error: error::Error + 'static,
     {
         self.validate_computed_version_or_reset_file(
@@ -255,21 +344,24 @@ where
 
         let index = max_from.min(I::from(self.len()));
         first_indexes.iter_from(index, |(i, first_index, ..)| {
-            let last_index = last_indexes.get(i)?.unwrap();
+            let last_index = last_indexes.cached_get(i)?.unwrap().into_inner();
-            let count = (*last_index + 1_usize)
+            let range = first_index.to_usize().unwrap()..=last_index.to_usize().unwrap();
-                .checked_sub(*first_index)
+            let count = if let Some(filter) = filter.as_mut() {
-                .unwrap_or_default();
+                range.into_iter().filter(|i| filter(T2::from(*i))).count()
-            self.forced_push_at(i, count.into(), exit)
+            } else {
+                range.count()
+            };
+            self.forced_push_at(i, T::from(T2::from(count)), exit)
         })?;
 
-        Ok(self.safe_flush(exit)?)
+        self.safe_flush(exit)
     }
 
     pub fn compute_is_first_ordered<A>(
         &mut self,
         max_from: I,
-        self_to_other: &mut brk_vec::StorableVec<I, A>,
+        self_to_other: &mut StoredVec<I, A>,
-        other_to_self: &mut brk_vec::StorableVec<A, I>,
+        other_to_self: &mut StoredVec<A, I>,
         exit: &Exit,
     ) -> Result<()>
     where
@@ -283,40 +375,48 @@ where
 
         let index = max_from.min(I::from(self.len()));
         self_to_other.iter_from(index, |(i, other, ..)| {
-            self.forced_push_at(i, T::from(other_to_self.get(*other)?.unwrap() == &i), exit)
+            self.forced_push_at(
+                i,
+                T::from(other_to_self.cached_get(other)?.unwrap().into_inner() == i),
+                exit,
+            )
         })?;
 
-        Ok(self.safe_flush(exit)?)
+        self.safe_flush(exit)
     }
 
     pub fn compute_sum_from_indexes<T2>(
         &mut self,
         max_from: I,
-        first_indexes: &mut brk_vec::StorableVec<I, T2>,
+        first_indexes: &mut StoredVec<I, T2>,
-        last_indexes: &mut brk_vec::StorableVec<I, T2>,
+        last_indexes: &mut StoredVec<I, T2>,
+        source: &mut StoredVec<T2, T>,
         exit: &Exit,
     ) -> Result<()>
     where
-        T: From<T2>,
+        T: From<usize> + Add<T, Output = T>,
-        T2: StoredType + Copy + Add<usize, Output = T2> + Sub<T2, Output = T2> + TryInto<T>,
+        T2: StoredIndex + StoredType,
-        <T2 as TryInto<T>>::Error: error::Error + 'static,
     {
         self.validate_computed_version_or_reset_file(
             Version::ZERO + self.version() + first_indexes.version() + last_indexes.version(),
         )?;
 
         let index = max_from.min(I::from(self.len()));
-        first_indexes.iter_from(index, |(index, first_index, ..)| {
+        first_indexes.iter_from(index, |(i, first_index, ..)| {
-            let last_index = last_indexes.get(index)?.unwrap();
+            let last_index = last_indexes.cached_get(i)?.unwrap().into_inner();
-            let count = *last_index + 1_usize - *first_index;
+            let range = first_index.to_usize().unwrap()..=last_index.to_usize().unwrap();
-            self.forced_push_at(index, count.into(), exit)
+            let mut sum = T::from(0_usize);
+            range.into_iter().for_each(|i| {
+                sum = sum.clone() + source.cached_get_(i).unwrap().unwrap().into_inner();
+            });
+            self.forced_push_at(i, sum, exit)
         })?;
 
-        Ok(self.safe_flush(exit)?)
+        self.safe_flush(exit)
     }
 }
 
-impl<I, T> Clone for StorableVec<I, T>
+impl<I, T> Clone for ComputedVec<I, T>
 where
     I: StoredIndex,
     T: StoredType,
@@ -324,7 +424,7 @@ where
     fn clone(&self) -> Self {
         Self {
             computed_version: self.computed_version,
-            vec: self.vec.clone(),
+            inner: self.inner.clone(),
         }
     }
 }

crates/brk_computer/src/storage/vecs/blocks.rs

@@ -1,21 +1,27 @@
 use std::{fs, path::Path};
 
-use brk_core::{CheckedSub, Dateindex, Height, Timestamp};
+use brk_core::{CheckedSub, Height, StoredU32, StoredU64, StoredUsize, Timestamp, Weight};
 use brk_exit::Exit;
 use brk_indexer::Indexer;
-use brk_vec::{AnyStorableVec, Compressed, Version};
+use brk_parser::bitcoin;
+use brk_vec::{Compressed, Version};
 
 use super::{
-    Indexes, StorableVec, indexes,
+    Indexes,
-    stats::{StorableVecGeneatorOptions, StorableVecsStatsFromHeight},
+    base::ComputedVec,
+    grouped::{ComputedVecsFromHeight, StorableVecGeneatorOptions},
+    indexes,
 };
 
 #[derive(Clone)]
 pub struct Vecs {
-    pub height_to_block_interval: StorableVec<Height, Timestamp>,
+    pub height_to_interval: ComputedVec<Height, Timestamp>,
-    pub indexes_to_block_interval_stats: StorableVecsStatsFromHeight<Timestamp>,
+    pub indexes_to_block_interval: ComputedVecsFromHeight<Timestamp>,
-    pub dateindex_to_block_count: StorableVec<Dateindex, u16>,
+    pub indexes_to_block_count: ComputedVecsFromHeight<StoredU32>,
-    pub dateindex_to_total_block_count: StorableVec<Dateindex, u32>,
+    pub indexes_to_block_weight: ComputedVecsFromHeight<Weight>,
+    pub height_to_vbytes: ComputedVec<Height, StoredU64>,
+    pub indexes_to_block_vbytes: ComputedVecsFromHeight<StoredU64>,
+    pub indexes_to_block_size: ComputedVecsFromHeight<StoredUsize>,
 }
 
 impl Vecs {
@@ -23,28 +29,58 @@ impl Vecs {
         fs::create_dir_all(path)?;
 
         Ok(Self {
-            height_to_block_interval: StorableVec::forced_import(
+            height_to_interval: ComputedVec::forced_import(
-                &path.join("height_to_block_interval"),
+                &path.join("height_to_interval"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            indexes_to_block_interval_stats: StorableVecsStatsFromHeight::forced_import(
+            indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
-                &path.join("block_interval"),
+                path,
+                "block_interval",
+                false,
+                Version::ZERO,
                 compressed,
                 StorableVecGeneatorOptions::default()
                     .add_percentiles()
                     .add_minmax()
                     .add_average(),
             )?,
-            dateindex_to_block_count: StorableVec::forced_import(
+            indexes_to_block_count: ComputedVecsFromHeight::forced_import(
-                &path.join("dateindex_to_block_count"),
+                path,
-                Version::ONE,
+                "block_count",
+                true,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default().add_sum().add_total(),
+            )?,
+            indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
+                path,
+                "block_weight",
+                false,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default().add_sum().add_total(),
+            )?,
+            indexes_to_block_size: ComputedVecsFromHeight::forced_import(
+                path,
+                "block_size",
+                false,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default().add_sum().add_total(),
+            )?,
+            height_to_vbytes: ComputedVec::forced_import(
+                &path.join("height_to_vbytes"),
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_total_block_count: StorableVec::forced_import(
+            indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
-                &path.join("dateindex_to_total_block_count"),
+                path,
-                Version::ONE,
+                "block_vbytes",
+                false,
+                Version::ZERO,
                 compressed,
+                StorableVecGeneatorOptions::default().add_sum().add_total(),
             )?,
         })
     }
@@ -56,14 +92,12 @@ impl Vecs {
         starting_indexes: &Indexes,
         exit: &Exit,
     ) -> color_eyre::Result<()> {
-        let indexer_vecs = indexer.mut_vecs();
+        self.height_to_interval.compute_transform(
-
-        self.height_to_block_interval.compute_transform(
             starting_indexes.height,
-            indexer_vecs.height_to_timestamp.mut_vec(),
+            indexer.mut_vecs().height_to_timestamp.mut_vec(),
             |(height, timestamp, _, height_to_timestamp)| {
                 let interval = height.decremented().map_or(Timestamp::ZERO, |prev_h| {
-                    let prev_timestamp = *height_to_timestamp.get(prev_h).unwrap().unwrap();
+                    let prev_timestamp = *height_to_timestamp.cached_get(prev_h).unwrap().unwrap();
                     timestamp
                         .checked_sub(prev_timestamp)
                         .unwrap_or(Timestamp::ZERO)
@@ -73,24 +107,75 @@ impl Vecs {
             exit,
         )?;
 
-        self.indexes_to_block_interval_stats.compute(
+        self.indexes_to_block_interval.compute_rest(
-            &mut self.height_to_block_interval,
             indexes,
             starting_indexes,
             exit,
+            Some(self.height_to_interval.mut_vec()),
+        )?;
+
+        self.indexes_to_block_count.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |v, indexer, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.height,
+                    indexer.mut_vecs().height_to_weight.mut_vec(),
+                    |(h, ..)| (h, StoredU32::from(1_u32)),
+                    exit,
+                )
+            },
+        )?;
+
+        self.indexes_to_block_weight.compute_rest(
+            indexes,
+            starting_indexes,
+            exit,
+            Some(indexer.mut_vecs().height_to_weight.mut_vec()),
+        )?;
+
+        self.indexes_to_block_size.compute_rest(
+            indexes,
+            starting_indexes,
+            exit,
+            Some(indexer.mut_vecs().height_to_total_size.mut_vec()),
+        )?;
+
+        self.height_to_vbytes.compute_transform(
+            starting_indexes.height,
+            indexer.mut_vecs().height_to_weight.mut_vec(),
+            |(h, w, ..)| {
+                (
+                    h,
+                    StoredU64::from(bitcoin::Weight::from(w).to_vbytes_floor()),
+                )
+            },
+            exit,
+        )?;
+
+        self.indexes_to_block_vbytes.compute_rest(
+            indexes,
+            starting_indexes,
+            exit,
+            Some(self.height_to_vbytes.mut_vec()),
         )?;
 
         Ok(())
     }
 
-    pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
+    pub fn as_any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
         [
             vec![
-                self.height_to_block_interval.any_vec(),
+                self.height_to_interval.any_vec(),
-                self.dateindex_to_block_count.any_vec(),
+                self.height_to_vbytes.any_vec(),
-                self.dateindex_to_total_block_count.any_vec(),
             ],
-            self.indexes_to_block_interval_stats.as_any_vecs(),
+            self.indexes_to_block_interval.any_vecs(),
+            self.indexes_to_block_count.any_vecs(),
+            self.indexes_to_block_weight.any_vecs(),
+            self.indexes_to_block_size.any_vecs(),
+            self.indexes_to_block_vbytes.any_vecs(),
         ]
         .concat()
     }

crates/brk_computer/src/storage/vecs/grouped/builder.rs

@@ -1,112 +1,159 @@
 use std::path::Path;
 
 use brk_exit::Exit;
-use brk_vec::{AnyStorableVec, Compressed, Result, StoredIndex, StoredType, Version};
+use brk_vec::{
+    Compressed, DynamicVec, GenericVec, Result, StoredIndex, StoredType, StoredVec, Version,
+};
+use color_eyre::eyre::ContextCompat;
 
-use crate::storage::vecs::base::StorableVec;
+use crate::storage::vecs::base::ComputedVec;
 
 use super::ComputedType;
 
 #[derive(Clone, Debug)]
-pub struct StorableVecBuilder<I, T>
+pub struct ComputedVecBuilder<I, T>
 where
     I: StoredIndex,
     T: ComputedType,
 {
-    pub first: Option<StorableVec<I, T>>,
+    pub first: Option<ComputedVec<I, T>>,
-    pub average: Option<StorableVec<I, T>>,
+    pub average: Option<ComputedVec<I, T>>,
-    pub sum: Option<StorableVec<I, T>>,
+    pub sum: Option<ComputedVec<I, T>>,
-    pub max: Option<StorableVec<I, T>>,
+    pub max: Option<ComputedVec<I, T>>,
-    pub _90p: Option<StorableVec<I, T>>,
+    pub _90p: Option<ComputedVec<I, T>>,
-    pub _75p: Option<StorableVec<I, T>>,
+    pub _75p: Option<ComputedVec<I, T>>,
-    pub median: Option<StorableVec<I, T>>,
+    pub median: Option<ComputedVec<I, T>>,
-    pub _25p: Option<StorableVec<I, T>>,
+    pub _25p: Option<ComputedVec<I, T>>,
-    pub _10p: Option<StorableVec<I, T>>,
+    pub _10p: Option<ComputedVec<I, T>>,
-    pub min: Option<StorableVec<I, T>>,
+    pub min: Option<ComputedVec<I, T>>,
-    pub last: Option<StorableVec<I, T>>,
+    pub last: Option<ComputedVec<I, T>>,
+    pub total: Option<ComputedVec<I, T>>,
 }
 
-impl<I, T> StorableVecBuilder<I, T>
+impl<I, T> ComputedVecBuilder<I, T>
 where
     I: StoredIndex,
     T: ComputedType,
 {
     pub fn forced_import(
         path: &Path,
+        name: &str,
         compressed: Compressed,
         options: StorableVecGeneatorOptions,
     ) -> color_eyre::Result<Self> {
-        let name = path.file_name().unwrap().to_str().unwrap().to_string();
         let key = I::to_string().split("::").last().unwrap().to_lowercase();
 
         let only_one_active = options.is_only_one_active();
 
-        let prefix = |s: &str| {
+        let default = || path.join(format!("{key}_to_{name}"));
+
+        let prefix = |s: &str| path.join(format!("{key}_to_{s}_{name}"));
+
+        let maybe_prefix = |s: &str| {
             if only_one_active {
-                path.with_file_name(format!("{key}_to_{name}"))
+                default()
             } else {
-                path.with_file_name(format!("{key}_to_{s}_{name}"))
+                prefix(s)
             }
         };
 
-        let suffix = |s: &str| {
+        let suffix = |s: &str| path.join(format!("{key}_to_{name}_{s}"));
+
+        let maybe_suffix = |s: &str| {
             if only_one_active {
-                path.with_file_name(format!("{key}_to_{name}"))
+                default()
             } else {
-                path.with_file_name(format!("{key}_to_{name}_{s}"))
+                suffix(s)
             }
         };
 
         let s = Self {
             first: options.first.then(|| {
-                StorableVec::forced_import(&prefix("first"), Version::ONE, compressed).unwrap()
+                ComputedVec::forced_import(&maybe_prefix("first"), Version::ZERO, compressed)
+                    .unwrap()
             }),
             last: options.last.then(|| {
-                StorableVec::forced_import(
+                ComputedVec::forced_import(
-                    &path.with_file_name(format!("{key}_to_{name}")),
+                    &path.join(format!("{key}_to_{name}")),
-                    Version::ONE,
+                    Version::ZERO,
                     compressed,
                 )
                 .unwrap()
             }),
             min: options.min.then(|| {
-                StorableVec::forced_import(&suffix("min"), Version::ONE, compressed).unwrap()
+                ComputedVec::forced_import(&maybe_suffix("min"), Version::ZERO, compressed).unwrap()
             }),
             max: options.max.then(|| {
-                StorableVec::forced_import(&suffix("max"), Version::ONE, compressed).unwrap()
+                ComputedVec::forced_import(&maybe_suffix("max"), Version::ZERO, compressed).unwrap()
             }),
             median: options.median.then(|| {
-                StorableVec::forced_import(&suffix("median"), Version::ONE, compressed).unwrap()
+                ComputedVec::forced_import(&maybe_suffix("median"), Version::ZERO, compressed)
+                    .unwrap()
             }),
             average: options.average.then(|| {
-                StorableVec::forced_import(&suffix("average"), Version::ONE, compressed).unwrap()
+                ComputedVec::forced_import(&maybe_suffix("average"), Version::ZERO, compressed)
+                    .unwrap()
             }),
             sum: options.sum.then(|| {
-                StorableVec::forced_import(&suffix("sum"), Version::ONE, compressed).unwrap()
+                ComputedVec::forced_import(&maybe_suffix("sum"), Version::ZERO, compressed).unwrap()
+            }),
+            total: options.total.then(|| {
+                ComputedVec::forced_import(&prefix("total"), Version::ZERO, compressed).unwrap()
             }),
             _90p: options._90p.then(|| {
-                StorableVec::forced_import(&suffix("90p"), Version::ONE, compressed).unwrap()
+                ComputedVec::forced_import(&maybe_suffix("90p"), Version::ZERO, compressed).unwrap()
             }),
             _75p: options._75p.then(|| {
-                StorableVec::forced_import(&suffix("75p"), Version::ONE, compressed).unwrap()
+                ComputedVec::forced_import(&maybe_suffix("75p"), Version::ZERO, compressed).unwrap()
             }),
             _25p: options._25p.then(|| {
-                StorableVec::forced_import(&suffix("25p"), Version::ONE, compressed).unwrap()
+                ComputedVec::forced_import(&maybe_suffix("25p"), Version::ZERO, compressed).unwrap()
             }),
             _10p: options._10p.then(|| {
-                StorableVec::forced_import(&suffix("10p"), Version::ONE, compressed).unwrap()
+                ComputedVec::forced_import(&maybe_suffix("10p"), Version::ZERO, compressed).unwrap()
             }),
         };
 
         Ok(s)
     }
 
+    pub fn extend(&mut self, max_from: I, source: &mut StoredVec<I, T>, exit: &Exit) -> Result<()> {
+        if self.total.is_none() {
+            return Ok(());
+        };
+
+        let index = self.starting_index(max_from);
+
+        let total_vec = self.total.as_mut().unwrap();
+
+        source.iter_from(index, |(i, v, ..)| {
+            let prev = i
+                .to_usize()
+                .unwrap()
+                .checked_sub(1)
+                .map_or(T::from(0_usize), |prev_i| {
+                    total_vec
+                        .cached_get(I::from(prev_i))
+                        .unwrap()
+                        .map_or(T::from(0_usize), |v| v.into_inner())
+                });
+            let value = v.clone() + prev;
+            total_vec.forced_push_at(i, value, exit)?;
+
+            Ok(())
+        })?;
+
+        self.safe_flush(exit)?;
+
+        Ok(())
+    }
+
     pub fn compute<I2>(
         &mut self,
         max_from: I,
-        source: &mut StorableVec<I2, T>,
+        source: &mut StoredVec<I2, T>,
-        first_indexes: &mut brk_vec::StorableVec<I, I2>,
+        first_indexes: &mut StoredVec<I, I2>,
-        last_indexes: &mut brk_vec::StorableVec<I, I2>,
+        last_indexes: &mut StoredVec<I, I2>,
         exit: &Exit,
     ) -> Result<()>
     where
@@ -116,24 +163,27 @@ where
     {
         let index = self.starting_index(max_from);
 
-        first_indexes.iter_from(index, |(i, first_index)| {
+        first_indexes.iter_from(index, |(i, first_index, first_indexes)| {
-            let first_index = *first_index;
+            let last_index = last_indexes.cached_get(i)?.unwrap().into_inner();
-            let last_index = *last_indexes.get(i).unwrap().unwrap();
 
             if let Some(first) = self.first.as_mut() {
-                let v = source.get(first_index).unwrap().unwrap();
+                let v = source.cached_get(first_index)?.unwrap().into_inner();
-                first.forced_push_at(index, v.clone(), exit)?;
+                first.forced_push_at(index, v, exit)?;
             }
 
             if let Some(last) = self.last.as_mut() {
-                let v = source.get(last_index).unwrap().unwrap();
+                let v = source
-                last.forced_push_at(index, v.clone(), exit)?;
+                    .cached_get(last_index)
+                    .inspect_err(|_| {
+                        dbg!(last.path(), last_index);
+                    })?
+                    .unwrap()
+                    .into_inner();
+                last.forced_push_at(index, v, exit)?;
             }
 
-            let first_index = first_index.to_usize()?;
+            let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
-            let last_index = last_index.to_usize()?;
+            let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
-
-            let needs_sum_or_average = self.sum.is_some() || self.average.is_some();
             let needs_sorted = self.max.is_some()
                 || self._90p.is_some()
                 || self._75p.is_some()
@@ -141,17 +191,36 @@ where
                 || self._25p.is_some()
                 || self._10p.is_some()
                 || self.min.is_some();
-            let needs_values = needs_sorted || needs_sum_or_average;
+            let needs_values = needs_sorted || needs_average_sum_or_total;
 
             if needs_values {
-                let mut values =
+                let mut values = source.collect_inclusive_range(first_index, last_index)?;
-                    source.collect_range(Some(first_index as i64), Some(last_index as i64))?;
 
                 if needs_sorted {
                     values.sort_unstable();
 
                     if let Some(max) = self.max.as_mut() {
-                        max.forced_push_at(i, values.last().unwrap().clone(), exit)?;
+                        max.forced_push_at(
+                            i,
+                            values
+                                .last()
+                                .context("expect some")
+                                .inspect_err(|_| {
+                                    dbg!(
+                                        &values,
+                                        max.path(),
+                                        first_indexes.path(),
+                                        first_index,
+                                        last_indexes.path(),
+                                        last_index,
+                                        source.len(),
+                                        source.path()
+                                    );
+                                })
+                                .unwrap()
+                                .clone(),
+                            exit,
+                        )?;
                     }
 
                     if let Some(_90p) = self._90p.as_mut() {
@@ -179,7 +248,7 @@ where
                     }
                 }
 
-                if needs_sum_or_average {
+                if needs_average_sum_or_total {
                     let len = values.len();
 
                     if let Some(average) = self.average.as_mut() {
@@ -192,9 +261,27 @@ where
                         average.forced_push_at(i, avg, exit)?;
                     }
 
-                    if let Some(sum_vec) = self.sum.as_mut() {
+                    if needs_sum_or_total {
                         let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
-                        sum_vec.forced_push_at(i, sum, exit)?;
+
+                        if let Some(sum_vec) = self.sum.as_mut() {
+                            sum_vec.forced_push_at(i, sum.clone(), exit)?;
+                        }
+
+                        if let Some(total_vec) = self.total.as_mut() {
+                            let prev = i.to_usize().unwrap().checked_sub(1).map_or(
+                                T::from(0_usize),
+                                |prev_i| {
+                                    total_vec
+                                        .cached_get(I::from(prev_i))
+                                        .unwrap()
+                                        .unwrap()
+                                        .to_owned()
+                                        .into_inner()
+                                },
+                            );
+                            total_vec.forced_push_at(i, prev + sum, exit)?;
+                        }
                     }
                 }
             }
@@ -211,9 +298,9 @@ where
     pub fn from_aligned<I2>(
         &mut self,
         max_from: I,
-        source: &mut StorableVecBuilder<I2, T>,
+        source: &mut ComputedVecBuilder<I2, T>,
-        first_indexes: &mut brk_vec::StorableVec<I, I2>,
+        first_indexes: &mut StoredVec<I, I2>,
-        last_indexes: &mut brk_vec::StorableVec<I, I2>,
+        last_indexes: &mut StoredVec<I, I2>,
         exit: &Exit,
     ) -> Result<()>
     where
@@ -232,19 +319,18 @@ where
 
         let index = self.starting_index(max_from);
 
-        first_indexes.iter_from(index, |(i, first_index)| {
+        first_indexes.iter_from(index, |(i, first_index, ..)| {
-            let first_index = *first_index;
+            let last_index = *last_indexes.cached_get(i).unwrap().unwrap();
-            let last_index = *last_indexes.get(i).unwrap().unwrap();
 
             if let Some(first) = self.first.as_mut() {
                 let v = source
                     .first
                     .as_mut()
                     .unwrap()
-                    .get(first_index)
+                    .cached_get(first_index)
                     .unwrap()
-                    .cloned()
+                    .unwrap()
-                    .unwrap();
+                    .into_inner();
                 first.forced_push_at(index, v, exit)?;
             }
 
@@ -253,19 +339,17 @@ where
                     .last
                     .as_mut()
                     .unwrap()
-                    .get(last_index)
+                    .cached_get(last_index)
                     .unwrap()
-                    .cloned()
+                    .unwrap()
-                    .unwrap();
+                    .into_inner();
                 last.forced_push_at(index, v, exit)?;
             }
 
-            let first_index = Some(first_index.to_usize()? as i64);
+            let needs_sum_or_total = self.sum.is_some() || self.total.is_some();
-            let last_index = Some(last_index.to_usize()? as i64);
+            let needs_average_sum_or_total = needs_sum_or_total || self.average.is_some();
-
-            let needs_sum_or_average = self.sum.is_some() || self.average.is_some();
             let needs_sorted = self.max.is_some() || self.min.is_some();
-            let needs_values = needs_sorted || needs_sum_or_average;
+            let needs_values = needs_sorted || needs_average_sum_or_total;
 
             if needs_values {
                 if needs_sorted {
@@ -274,7 +358,7 @@ where
                             .max
                             .as_ref()
                             .unwrap()
-                            .collect_range(first_index, last_index)?;
+                            .collect_inclusive_range(first_index, last_index)?;
                         values.sort_unstable();
                         max.forced_push_at(i, values.last().unwrap().clone(), exit)?;
                     }
@@ -284,19 +368,19 @@ where
                             .min
                             .as_ref()
                             .unwrap()
-                            .collect_range(first_index, last_index)?;
+                            .collect_inclusive_range(first_index, last_index)?;
                         values.sort_unstable();
                         min.forced_push_at(i, values.first().unwrap().clone(), exit)?;
                     }
                 }
 
-                if needs_sum_or_average {
+                if needs_average_sum_or_total {
                     if let Some(average) = self.average.as_mut() {
                         let values = source
                             .average
                             .as_ref()
                             .unwrap()
-                            .collect_range(first_index, last_index)?;
+                            .collect_inclusive_range(first_index, last_index)?;
                         let len = values.len() as f64;
                         let total = values
                             .into_iter()
@@ -308,14 +392,31 @@ where
                         average.forced_push_at(i, avg, exit)?;
                     }
 
-                    if let Some(sum_vec) = self.sum.as_mut() {
+                    if needs_sum_or_total {
                         let values = source
                             .sum
                             .as_ref()
                             .unwrap()
-                            .collect_range(first_index, last_index)?;
+                            .collect_inclusive_range(first_index, last_index)?;
                         let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
-                        sum_vec.forced_push_at(i, sum, exit)?;
+
+                        if let Some(sum_vec) = self.sum.as_mut() {
+                            sum_vec.forced_push_at(i, sum.clone(), exit)?;
+                        }
+
+                        if let Some(total_vec) = self.total.as_mut() {
+                            let prev = i.to_usize().unwrap().checked_sub(1).map_or(
+                                T::from(0_usize),
+                                |prev_i| {
+                                    total_vec
+                                        .cached_get(I::from(prev_i))
+                                        .unwrap()
+                                        .unwrap()
+                                        .into_inner()
+                                },
+                            );
+                            total_vec.forced_push_at(i, prev + sum, exit)?;
+                        }
                     }
                 }
             }
@@ -352,16 +453,12 @@ where
 
     fn starting_index(&self, max_from: I) -> I {
         max_from.min(I::from(
-            self.as_any_vecs()
+            self.any_vecs().into_iter().map(|v| v.len()).min().unwrap(),
-                .into_iter()
-                .map(|v| v.len())
-                .min()
-                .unwrap(),
         ))
     }
 
-    pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
+    pub fn any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
-        let mut v: Vec<&dyn AnyStorableVec> = vec![];
+        let mut v: Vec<&dyn brk_vec::AnyStoredVec> = vec![];
 
         if let Some(first) = self.first.as_ref() {
             v.push(first.any_vec());
@@ -384,6 +481,9 @@ where
         if let Some(sum) = self.sum.as_ref() {
             v.push(sum.any_vec());
         }
+        if let Some(total) = self.total.as_ref() {
+            v.push(total.any_vec());
+        }
         if let Some(_90p) = self._90p.as_ref() {
             v.push(_90p.any_vec());
         }
@@ -422,6 +522,9 @@ where
         if let Some(sum) = self.sum.as_mut() {
             sum.safe_flush(exit)?;
         }
+        if let Some(total) = self.total.as_mut() {
+            total.safe_flush(exit)?;
+        }
         if let Some(_90p) = self._90p.as_mut() {
             _90p.safe_flush(exit)?;
         }
@@ -452,6 +555,7 @@ pub struct StorableVecGeneatorOptions {
     min: bool,
     first: bool,
     last: bool,
+    total: bool,
 }
 
 impl StorableVecGeneatorOptions {
@@ -510,6 +614,11 @@ impl StorableVecGeneatorOptions {
         self
     }
 
+    pub fn add_total(mut self) -> Self {
+        self.total = true;
+        self
+    }
+
     pub fn rm_min(mut self) -> Self {
         self.min = false;
         self
@@ -555,6 +664,11 @@ impl StorableVecGeneatorOptions {
         self
     }
 
+    pub fn rm_total(mut self) -> Self {
+        self.total = false;
+        self
+    }
+
     pub fn add_minmax(mut self) -> Self {
         self.min = true;
         self.max = true;
@@ -592,10 +706,18 @@ impl StorableVecGeneatorOptions {
             self.min,
             self.first,
             self.last,
+            self.total,
         ]
         .iter()
         .filter(|b| **b)
         .count()
             == 1
     }
+
+    pub fn copy_self_extra(&self) -> Self {
+        Self {
+            total: self.total,
+            ..Self::default()
+        }
+    }
 }

crates/brk_computer/src/storage/vecs/grouped/from_dateindex.rs

@@ -0,0 +1,141 @@
+use std::path::Path;
+
+use brk_core::{Dateindex, Decadeindex, Monthindex, Quarterindex, Weekindex, Yearindex};
+use brk_exit::Exit;
+use brk_indexer::Indexer;
+use brk_vec::{AnyStoredVec, Compressed, Result, Version};
+
+use crate::storage::vecs::{Indexes, base::ComputedVec, indexes};
+
+use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
+
+#[derive(Clone)]
+pub struct ComputedVecsFromDateindex<T>
+where
+    T: ComputedType + PartialOrd,
+{
+    pub dateindex: ComputedVec<Dateindex, T>,
+    pub dateindex_extra: ComputedVecBuilder<Dateindex, T>,
+    pub weekindex: ComputedVecBuilder<Weekindex, T>,
+    pub monthindex: ComputedVecBuilder<Monthindex, T>,
+    pub quarterindex: ComputedVecBuilder<Quarterindex, T>,
+    pub yearindex: ComputedVecBuilder<Yearindex, T>,
+    pub decadeindex: ComputedVecBuilder<Decadeindex, T>,
+}
+
+impl<T> ComputedVecsFromDateindex<T>
+where
+    T: ComputedType + Ord + From<f64>,
+    f64: From<T>,
+{
+    pub fn forced_import(
+        path: &Path,
+        name: &str,
+        version: Version,
+        compressed: Compressed,
+        options: StorableVecGeneatorOptions,
+    ) -> color_eyre::Result<Self> {
+        let dateindex_extra =
+            ComputedVecBuilder::forced_import(path, name, compressed, options.copy_self_extra())?;
+
+        let options = options.remove_percentiles();
+
+        Ok(Self {
+            dateindex: ComputedVec::forced_import(
+                &path.join(format!("dateindex_to_{name}")),
+                version,
+                compressed,
+            )?,
+            dateindex_extra,
+            weekindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            monthindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            quarterindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            yearindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            decadeindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+        })
+    }
+
+    pub fn compute<F>(
+        &mut self,
+        indexer: &mut Indexer,
+        indexes: &mut indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        mut compute: F,
+    ) -> color_eyre::Result<()>
+    where
+        F: FnMut(
+            &mut ComputedVec<Dateindex, T>,
+            &mut Indexer,
+            &mut indexes::Vecs,
+            &Indexes,
+            &Exit,
+        ) -> Result<()>,
+    {
+        compute(
+            &mut self.dateindex,
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+        )?;
+
+        self.dateindex_extra
+            .extend(starting_indexes.dateindex, self.dateindex.mut_vec(), exit)?;
+
+        self.weekindex.compute(
+            starting_indexes.weekindex,
+            self.dateindex.mut_vec(),
+            indexes.weekindex_to_first_dateindex.mut_vec(),
+            indexes.weekindex_to_last_dateindex.mut_vec(),
+            exit,
+        )?;
+
+        self.monthindex.compute(
+            starting_indexes.monthindex,
+            self.dateindex.mut_vec(),
+            indexes.monthindex_to_first_dateindex.mut_vec(),
+            indexes.monthindex_to_last_dateindex.mut_vec(),
+            exit,
+        )?;
+
+        self.quarterindex.from_aligned(
+            starting_indexes.quarterindex,
+            &mut self.monthindex,
+            indexes.quarterindex_to_first_monthindex.mut_vec(),
+            indexes.quarterindex_to_last_monthindex.mut_vec(),
+            exit,
+        )?;
+
+        self.yearindex.from_aligned(
+            starting_indexes.yearindex,
+            &mut self.monthindex,
+            indexes.yearindex_to_first_monthindex.mut_vec(),
+            indexes.yearindex_to_last_monthindex.mut_vec(),
+            exit,
+        )?;
+
+        self.decadeindex.from_aligned(
+            starting_indexes.decadeindex,
+            &mut self.yearindex,
+            indexes.decadeindex_to_first_yearindex.mut_vec(),
+            indexes.decadeindex_to_last_yearindex.mut_vec(),
+            exit,
+        )?;
+
+        Ok(())
+    }
+
+    pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
+        [
+            vec![self.dateindex.any_vec()],
+            self.dateindex_extra.any_vecs(),
+            self.weekindex.any_vecs(),
+            self.monthindex.any_vecs(),
+            self.quarterindex.any_vecs(),
+            self.yearindex.any_vecs(),
+            self.decadeindex.any_vecs(),
+        ]
+        .concat()
+    }
+}

crates/brk_computer/src/storage/vecs/grouped/from_height.rs

@@ -0,0 +1,186 @@
+use std::path::Path;
+
+use brk_core::{
+    Dateindex, Decadeindex, Difficultyepoch, Height, Monthindex, Quarterindex, Weekindex, Yearindex,
+};
+use brk_exit::Exit;
+use brk_indexer::Indexer;
+use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version};
+
+use crate::storage::vecs::{Indexes, base::ComputedVec, indexes};
+
+use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
+
+#[derive(Clone)]
+pub struct ComputedVecsFromHeight<T>
+where
+    T: ComputedType + PartialOrd,
+{
+    pub height: Option<ComputedVec<Height, T>>,
+    pub height_extra: ComputedVecBuilder<Height, T>,
+    pub dateindex: ComputedVecBuilder<Dateindex, T>,
+    pub weekindex: ComputedVecBuilder<Weekindex, T>,
+    pub difficultyepoch: ComputedVecBuilder<Difficultyepoch, T>,
+    pub monthindex: ComputedVecBuilder<Monthindex, T>,
+    pub quarterindex: ComputedVecBuilder<Quarterindex, T>,
+    pub yearindex: ComputedVecBuilder<Yearindex, T>,
+    // TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
+    pub decadeindex: ComputedVecBuilder<Decadeindex, T>,
+}
+
+impl<T> ComputedVecsFromHeight<T>
+where
+    T: ComputedType + Ord + From<f64>,
+    f64: From<T>,
+{
+    pub fn forced_import(
+        path: &Path,
+        name: &str,
+        compute_source: bool,
+        version: Version,
+        compressed: Compressed,
+        options: StorableVecGeneatorOptions,
+    ) -> color_eyre::Result<Self> {
+        let height = compute_source.then(|| {
+            ComputedVec::forced_import(&path.join(format!("height_to_{name}")), version, compressed)
+                .unwrap()
+        });
+
+        let height_extra =
+            ComputedVecBuilder::forced_import(path, name, compressed, options.copy_self_extra())?;
+
+        let dateindex = ComputedVecBuilder::forced_import(path, name, compressed, options)?;
+
+        let options = options.remove_percentiles();
+
+        Ok(Self {
+            height,
+            height_extra,
+            dateindex,
+            weekindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            difficultyepoch: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            monthindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            quarterindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            yearindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            // halvingepoch: StorableVecGeneator::forced_import(path, name, compressed, options)?,
+            decadeindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+        })
+    }
+
+    pub fn compute_all<F>(
+        &mut self,
+        indexer: &mut Indexer,
+        indexes: &mut indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        mut compute: F,
+    ) -> color_eyre::Result<()>
+    where
+        F: FnMut(
+            &mut ComputedVec<Height, T>,
+            &mut Indexer,
+            &mut indexes::Vecs,
+            &Indexes,
+            &Exit,
+        ) -> Result<()>,
+    {
+        compute(
+            self.height.as_mut().unwrap(),
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+        )?;
+
+        self.compute_rest(indexes, starting_indexes, exit, None)?;
+
+        Ok(())
+    }
+
+    pub fn compute_rest(
+        &mut self,
+        indexes: &mut indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        height: Option<&mut StoredVec<Height, T>>,
+    ) -> color_eyre::Result<()> {
+        let height = height.unwrap_or_else(|| self.height.as_mut().unwrap().mut_vec());
+
+        self.height_extra
+            .extend(starting_indexes.height, height, exit)?;
+
+        self.dateindex.compute(
+            starting_indexes.dateindex,
+            height,
+            indexes.dateindex_to_first_height.mut_vec(),
+            indexes.dateindex_to_last_height.mut_vec(),
+            exit,
+        )?;
+
+        self.weekindex.from_aligned(
+            starting_indexes.weekindex,
+            &mut self.dateindex,
+            indexes.weekindex_to_first_dateindex.mut_vec(),
+            indexes.weekindex_to_last_dateindex.mut_vec(),
+            exit,
+        )?;
+
+        self.monthindex.from_aligned(
+            starting_indexes.monthindex,
+            &mut self.dateindex,
+            indexes.monthindex_to_first_dateindex.mut_vec(),
+            indexes.monthindex_to_last_dateindex.mut_vec(),
+            exit,
+        )?;
+
+        self.quarterindex.from_aligned(
+            starting_indexes.quarterindex,
+            &mut self.monthindex,
+            indexes.quarterindex_to_first_monthindex.mut_vec(),
+            indexes.quarterindex_to_last_monthindex.mut_vec(),
+            exit,
+        )?;
+
+        self.yearindex.from_aligned(
+            starting_indexes.yearindex,
+            &mut self.monthindex,
+            indexes.yearindex_to_first_monthindex.mut_vec(),
+            indexes.yearindex_to_last_monthindex.mut_vec(),
+            exit,
+        )?;
+
+        self.decadeindex.from_aligned(
+            starting_indexes.decadeindex,
+            &mut self.yearindex,
+            indexes.decadeindex_to_first_yearindex.mut_vec(),
+            indexes.decadeindex_to_last_yearindex.mut_vec(),
+            exit,
+        )?;
+
+        self.difficultyepoch.compute(
+            starting_indexes.difficultyepoch,
+            height,
+            indexes.difficultyepoch_to_first_height.mut_vec(),
+            indexes.difficultyepoch_to_last_height.mut_vec(),
+            exit,
+        )?;
+
+        Ok(())
+    }
+
+    pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
+        [
+            self.height.as_ref().map_or(vec![], |v| vec![v.any_vec()]),
+            self.height_extra.any_vecs(),
+            self.dateindex.any_vecs(),
+            self.weekindex.any_vecs(),
+            self.difficultyepoch.any_vecs(),
+            self.monthindex.any_vecs(),
+            self.quarterindex.any_vecs(),
+            self.yearindex.any_vecs(),
+            // self.halvingepoch.as_any_vecs(),
+            self.decadeindex.any_vecs(),
+        ]
+        .concat()
+    }
+}

crates/brk_computer/src/storage/vecs/grouped/from_height_strict.rs

@@ -0,0 +1,96 @@
+use std::path::Path;
+
+use brk_core::{Difficultyepoch, Height};
+use brk_exit::Exit;
+use brk_indexer::Indexer;
+use brk_vec::{AnyStoredVec, Compressed, Result, Version};
+
+use crate::storage::vecs::{Indexes, base::ComputedVec, indexes};
+
+use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
+
+#[derive(Clone)]
+pub struct ComputedVecsFromHeightStrict<T>
+where
+    T: ComputedType + PartialOrd,
+{
+    pub height: ComputedVec<Height, T>,
+    pub height_extra: ComputedVecBuilder<Height, T>,
+    pub difficultyepoch: ComputedVecBuilder<Difficultyepoch, T>,
+    // TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
+}
+
+impl<T> ComputedVecsFromHeightStrict<T>
+where
+    T: ComputedType + Ord + From<f64>,
+    f64: From<T>,
+{
+    pub fn forced_import(
+        path: &Path,
+        name: &str,
+        version: Version,
+        compressed: Compressed,
+        options: StorableVecGeneatorOptions,
+    ) -> color_eyre::Result<Self> {
+        let height = ComputedVec::forced_import(
+            &path.join(format!("height_to_{name}")),
+            version,
+            compressed,
+        )?;
+
+        let height_extra =
+            ComputedVecBuilder::forced_import(path, name, compressed, options.copy_self_extra())?;
+
+        let options = options.remove_percentiles();
+
+        Ok(Self {
+            height,
+            height_extra,
+            difficultyepoch: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            // halvingepoch: StorableVecGeneator::forced_import(path, name, compressed, options)?,
+        })
+    }
+
+    pub fn compute<F>(
+        &mut self,
+        indexer: &mut Indexer,
+        indexes: &mut indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        mut compute: F,
+    ) -> color_eyre::Result<()>
+    where
+        F: FnMut(
+            &mut ComputedVec<Height, T>,
+            &mut Indexer,
+            &mut indexes::Vecs,
+            &Indexes,
+            &Exit,
+        ) -> Result<()>,
+    {
+        compute(&mut self.height, indexer, indexes, starting_indexes, exit)?;
+
+        self.height_extra
+            .extend(starting_indexes.height, self.height.mut_vec(), exit)?;
+
+        self.difficultyepoch.compute(
+            starting_indexes.difficultyepoch,
+            self.height.mut_vec(),
+            indexes.difficultyepoch_to_first_height.mut_vec(),
+            indexes.difficultyepoch_to_last_height.mut_vec(),
+            exit,
+        )?;
+
+        Ok(())
+    }
+
+    pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
+        [
+            vec![self.height.any_vec()],
+            self.height_extra.any_vecs(),
+            self.difficultyepoch.any_vecs(),
+            // self.halvingepoch.as_any_vecs(),
+        ]
+        .concat()
+    }
+}

crates/brk_computer/src/storage/vecs/grouped/from_txindex.rs

@@ -0,0 +1,194 @@
+use std::path::Path;
+
+use brk_core::{
+    Dateindex, Decadeindex, Difficultyepoch, Height, Monthindex, Quarterindex, Txindex, Weekindex,
+    Yearindex,
+};
+use brk_exit::Exit;
+use brk_indexer::Indexer;
+use brk_vec::{AnyStoredVec, Compressed, Result, StoredVec, Version};
+
+use crate::storage::vecs::{Indexes, base::ComputedVec, indexes};
+
+use super::{ComputedType, ComputedVecBuilder, StorableVecGeneatorOptions};
+
+#[derive(Clone)]
+pub struct ComputedVecsFromTxindex<T>
+where
+    T: ComputedType + PartialOrd,
+{
+    pub txindex: Option<ComputedVec<Txindex, T>>,
+    pub height: ComputedVecBuilder<Height, T>,
+    pub dateindex: ComputedVecBuilder<Dateindex, T>,
+    pub weekindex: ComputedVecBuilder<Weekindex, T>,
+    pub difficultyepoch: ComputedVecBuilder<Difficultyepoch, T>,
+    pub monthindex: ComputedVecBuilder<Monthindex, T>,
+    pub quarterindex: ComputedVecBuilder<Quarterindex, T>,
+    pub yearindex: ComputedVecBuilder<Yearindex, T>,
+    // TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
+    pub decadeindex: ComputedVecBuilder<Decadeindex, T>,
+}
+
+impl<T> ComputedVecsFromTxindex<T>
+where
+    T: ComputedType + Ord + From<f64>,
+    f64: From<T>,
+{
+    pub fn forced_import(
+        path: &Path,
+        name: &str,
+        compute_source: bool,
+        version: Version,
+        compressed: Compressed,
+        options: StorableVecGeneatorOptions,
+    ) -> color_eyre::Result<Self> {
+        let txindex = compute_source.then(|| {
+            ComputedVec::forced_import(
+                &path.join(format!("txindex_to_{name}")),
+                version,
+                compressed,
+            )
+            .unwrap()
+        });
+
+        let height = ComputedVecBuilder::forced_import(path, name, compressed, options)?;
+
+        let options = options.remove_percentiles();
+
+        Ok(Self {
+            txindex,
+            height,
+            dateindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            weekindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            difficultyepoch: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            monthindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            quarterindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            yearindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+            // halvingepoch: StorableVecGeneator::forced_import(path, name, compressed, options)?,
+            decadeindex: ComputedVecBuilder::forced_import(path, name, compressed, options)?,
+        })
+    }
+
+    pub fn compute_all<F>(
+        &mut self,
+        indexer: &mut Indexer,
+        indexes: &mut indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        mut compute: F,
+    ) -> color_eyre::Result<()>
+    where
+        F: FnMut(
+            &mut ComputedVec<Txindex, T>,
+            &mut Indexer,
+            &mut indexes::Vecs,
+            &Indexes,
+            &Exit,
+        ) -> Result<()>,
+    {
+        compute(
+            self.txindex.as_mut().unwrap(),
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+        )?;
+
+        self.compute_rest(indexer, indexes, starting_indexes, exit, None)?;
+
+        Ok(())
+    }
+
+    pub fn compute_rest(
+        &mut self,
+        indexer: &mut Indexer,
+        indexes: &mut indexes::Vecs,
+        starting_indexes: &Indexes,
+        exit: &Exit,
+        txindex: Option<&mut StoredVec<Txindex, T>>,
+    ) -> color_eyre::Result<()> {
+        let txindex = txindex.unwrap_or_else(|| self.txindex.as_mut().unwrap().mut_vec());
+
+        self.height.compute(
+            starting_indexes.height,
+            txindex,
+            indexer.mut_vecs().height_to_first_txindex.mut_vec(),
+            indexes.height_to_last_txindex.mut_vec(),
+            exit,
+        )?;
+
+        self.dateindex.from_aligned(
+            starting_indexes.dateindex,
+            &mut self.height,
+            indexes.dateindex_to_first_height.mut_vec(),
+            indexes.dateindex_to_last_height.mut_vec(),
+            exit,
+        )?;
+
+        self.weekindex.from_aligned(
+            starting_indexes.weekindex,
+            &mut self.dateindex,
+            indexes.weekindex_to_first_dateindex.mut_vec(),
+            indexes.weekindex_to_last_dateindex.mut_vec(),
+            exit,
+        )?;
+
+        self.monthindex.from_aligned(
+            starting_indexes.monthindex,
+            &mut self.dateindex,
+            indexes.monthindex_to_first_dateindex.mut_vec(),
+            indexes.monthindex_to_last_dateindex.mut_vec(),
+            exit,
+        )?;
+
+        self.quarterindex.from_aligned(
+            starting_indexes.quarterindex,
+            &mut self.monthindex,
+            indexes.quarterindex_to_first_monthindex.mut_vec(),
+            indexes.quarterindex_to_last_monthindex.mut_vec(),
+            exit,
+        )?;
+
+        self.yearindex.from_aligned(
+            starting_indexes.yearindex,
+            &mut self.monthindex,
+            indexes.yearindex_to_first_monthindex.mut_vec(),
+            indexes.yearindex_to_last_monthindex.mut_vec(),
+            exit,
+        )?;
+
+        self.decadeindex.from_aligned(
+            starting_indexes.decadeindex,
+            &mut self.yearindex,
+            indexes.decadeindex_to_first_yearindex.mut_vec(),
+            indexes.decadeindex_to_last_yearindex.mut_vec(),
+            exit,
+        )?;
+
+        self.difficultyepoch.from_aligned(
+            starting_indexes.difficultyepoch,
+            &mut self.height,
+            indexes.difficultyepoch_to_first_height.mut_vec(),
+            indexes.difficultyepoch_to_last_height.mut_vec(),
+            exit,
+        )?;
+
+        Ok(())
+    }
+
+    pub fn any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
+        [
+            self.txindex.as_ref().map_or(vec![], |v| vec![v.any_vec()]),
+            self.height.any_vecs(),
+            self.dateindex.any_vecs(),
+            self.weekindex.any_vecs(),
+            self.difficultyepoch.any_vecs(),
+            self.monthindex.any_vecs(),
+            self.quarterindex.any_vecs(),
+            self.yearindex.any_vecs(),
+            // self.halvingepoch.as_any_vecs(),
+            self.decadeindex.any_vecs(),
+        ]
+        .concat()
+    }
+}

crates/brk_computer/src/storage/vecs/grouped/mod.rs

@@ -1,11 +1,13 @@
-mod from_date;
+mod builder;
+mod from_dateindex;
 mod from_height;
 mod from_height_strict;
-mod generic;
+mod from_txindex;
 mod stored_type;
 
-pub use from_date::*;
+pub use builder::*;
+pub use from_dateindex::*;
 pub use from_height::*;
 pub use from_height_strict::*;
-pub use generic::*;
+pub use from_txindex::*;
 pub use stored_type::*;

crates/brk_computer/src/storage/vecs/indexes.rs

@@ -6,62 +6,60 @@ use brk_core::{
 };
 use brk_exit::Exit;
 use brk_indexer::Indexer;
-use brk_vec::{AnyStorableVec, Compressed, Version};
+use brk_vec::{Compressed, Version};
 
-use super::StorableVec;
+use super::ComputedVec;
 
 #[derive(Clone)]
 pub struct Vecs {
-    // pub height_to_last_addressindex: StorableVec<Height, Addressindex>,
+    pub dateindex_to_date: ComputedVec<Dateindex, Date>,
-    // pub height_to_last_txoutindex: StorableVec<Height, Txoutindex>,
+    pub dateindex_to_dateindex: ComputedVec<Dateindex, Dateindex>,
-    pub dateindex_to_date: StorableVec<Dateindex, Date>,
+    pub dateindex_to_first_height: ComputedVec<Dateindex, Height>,
-    pub dateindex_to_dateindex: StorableVec<Dateindex, Dateindex>,
+    pub dateindex_to_last_height: ComputedVec<Dateindex, Height>,
-    pub dateindex_to_first_height: StorableVec<Dateindex, Height>,
+    pub dateindex_to_monthindex: ComputedVec<Dateindex, Monthindex>,
-    pub dateindex_to_last_height: StorableVec<Dateindex, Height>,
+    pub dateindex_to_timestamp: ComputedVec<Dateindex, Timestamp>,
-    pub dateindex_to_monthindex: StorableVec<Dateindex, Monthindex>,
+    pub dateindex_to_weekindex: ComputedVec<Dateindex, Weekindex>,
-    pub dateindex_to_timestamp: StorableVec<Dateindex, Timestamp>,
+    pub decadeindex_to_decadeindex: ComputedVec<Decadeindex, Decadeindex>,
-    pub dateindex_to_weekindex: StorableVec<Dateindex, Weekindex>,
+    pub decadeindex_to_first_yearindex: ComputedVec<Decadeindex, Yearindex>,
-    pub decadeindex_to_decadeindex: StorableVec<Decadeindex, Decadeindex>,
+    pub decadeindex_to_last_yearindex: ComputedVec<Decadeindex, Yearindex>,
-    pub decadeindex_to_first_yearindex: StorableVec<Decadeindex, Yearindex>,
+    pub decadeindex_to_timestamp: ComputedVec<Decadeindex, Timestamp>,
-    pub decadeindex_to_last_yearindex: StorableVec<Decadeindex, Yearindex>,
+    pub difficultyepoch_to_difficultyepoch: ComputedVec<Difficultyepoch, Difficultyepoch>,
-    pub decadeindex_to_timestamp: StorableVec<Decadeindex, Timestamp>,
+    pub difficultyepoch_to_first_height: ComputedVec<Difficultyepoch, Height>,
-    pub difficultyepoch_to_difficultyepoch: StorableVec<Difficultyepoch, Difficultyepoch>,
+    pub difficultyepoch_to_last_height: ComputedVec<Difficultyepoch, Height>,
-    pub difficultyepoch_to_first_height: StorableVec<Difficultyepoch, Height>,
+    pub difficultyepoch_to_timestamp: ComputedVec<Difficultyepoch, Timestamp>,
-    pub difficultyepoch_to_last_height: StorableVec<Difficultyepoch, Height>,
+    pub halvingepoch_to_first_height: ComputedVec<Halvingepoch, Height>,
-    pub difficultyepoch_to_timestamp: StorableVec<Difficultyepoch, Timestamp>,
+    pub halvingepoch_to_halvingepoch: ComputedVec<Halvingepoch, Halvingepoch>,
-    pub halvingepoch_to_first_height: StorableVec<Halvingepoch, Height>,
+    pub halvingepoch_to_last_height: ComputedVec<Halvingepoch, Height>,
-    pub halvingepoch_to_halvingepoch: StorableVec<Halvingepoch, Halvingepoch>,
+    pub halvingepoch_to_timestamp: ComputedVec<Halvingepoch, Timestamp>,
-    pub halvingepoch_to_last_height: StorableVec<Halvingepoch, Height>,
+    pub height_to_dateindex: ComputedVec<Height, Dateindex>,
-    pub halvingepoch_to_timestamp: StorableVec<Halvingepoch, Timestamp>,
+    pub height_to_difficultyepoch: ComputedVec<Height, Difficultyepoch>,
-    pub height_to_dateindex: StorableVec<Height, Dateindex>,
+    pub height_to_fixed_date: ComputedVec<Height, Date>,
-    pub height_to_difficultyepoch: StorableVec<Height, Difficultyepoch>,
+    pub height_to_fixed_timestamp: ComputedVec<Height, Timestamp>,
-    pub height_to_fixed_date: StorableVec<Height, Date>,
+    pub height_to_halvingepoch: ComputedVec<Height, Halvingepoch>,
-    pub height_to_fixed_timestamp: StorableVec<Height, Timestamp>,
+    pub height_to_height: ComputedVec<Height, Height>,
-    pub height_to_halvingepoch: StorableVec<Height, Halvingepoch>,
+    pub height_to_last_txindex: ComputedVec<Height, Txindex>,
-    pub height_to_height: StorableVec<Height, Height>,
+    pub height_to_real_date: ComputedVec<Height, Date>,
-    pub height_to_last_txindex: StorableVec<Height, Txindex>,
+    pub monthindex_to_first_dateindex: ComputedVec<Monthindex, Dateindex>,
-    pub height_to_real_date: StorableVec<Height, Date>,
+    pub monthindex_to_last_dateindex: ComputedVec<Monthindex, Dateindex>,
-    pub monthindex_to_first_dateindex: StorableVec<Monthindex, Dateindex>,
+    pub monthindex_to_monthindex: ComputedVec<Monthindex, Monthindex>,
-    pub monthindex_to_last_dateindex: StorableVec<Monthindex, Dateindex>,
+    pub monthindex_to_quarterindex: ComputedVec<Monthindex, Quarterindex>,
-    pub monthindex_to_monthindex: StorableVec<Monthindex, Monthindex>,
+    pub monthindex_to_timestamp: ComputedVec<Monthindex, Timestamp>,
-    pub monthindex_to_quarterindex: StorableVec<Monthindex, Quarterindex>,
+    pub monthindex_to_yearindex: ComputedVec<Monthindex, Yearindex>,
-    pub monthindex_to_timestamp: StorableVec<Monthindex, Timestamp>,
+    pub quarterindex_to_first_monthindex: ComputedVec<Quarterindex, Monthindex>,
-    pub monthindex_to_yearindex: StorableVec<Monthindex, Yearindex>,
+    pub quarterindex_to_last_monthindex: ComputedVec<Quarterindex, Monthindex>,
-    pub quarterindex_to_first_monthindex: StorableVec<Quarterindex, Monthindex>,
+    pub quarterindex_to_quarterindex: ComputedVec<Quarterindex, Quarterindex>,
-    pub quarterindex_to_last_monthindex: StorableVec<Quarterindex, Monthindex>,
+    pub quarterindex_to_timestamp: ComputedVec<Quarterindex, Timestamp>,
-    pub quarterindex_to_quarterindex: StorableVec<Quarterindex, Quarterindex>,
+    pub txindex_to_last_txinindex: ComputedVec<Txindex, Txinindex>,
-    pub quarterindex_to_timestamp: StorableVec<Quarterindex, Timestamp>,
+    pub txindex_to_last_txoutindex: ComputedVec<Txindex, Txoutindex>,
-    pub txindex_to_last_txinindex: StorableVec<Txindex, Txinindex>,
+    pub weekindex_to_first_dateindex: ComputedVec<Weekindex, Dateindex>,
-    pub txindex_to_last_txoutindex: StorableVec<Txindex, Txoutindex>,
+    pub weekindex_to_last_dateindex: ComputedVec<Weekindex, Dateindex>,
-    pub weekindex_to_first_dateindex: StorableVec<Weekindex, Dateindex>,
+    pub weekindex_to_timestamp: ComputedVec<Weekindex, Timestamp>,
-    pub weekindex_to_last_dateindex: StorableVec<Weekindex, Dateindex>,
+    pub weekindex_to_weekindex: ComputedVec<Weekindex, Weekindex>,
-    pub weekindex_to_timestamp: StorableVec<Weekindex, Timestamp>,
+    pub yearindex_to_decadeindex: ComputedVec<Yearindex, Decadeindex>,
-    pub weekindex_to_weekindex: StorableVec<Weekindex, Weekindex>,
+    pub yearindex_to_first_monthindex: ComputedVec<Yearindex, Monthindex>,
-    pub yearindex_to_decadeindex: StorableVec<Yearindex, Decadeindex>,
+    pub yearindex_to_last_monthindex: ComputedVec<Yearindex, Monthindex>,
-    pub yearindex_to_first_monthindex: StorableVec<Yearindex, Monthindex>,
+    pub yearindex_to_timestamp: ComputedVec<Yearindex, Timestamp>,
-    pub yearindex_to_last_monthindex: StorableVec<Yearindex, Monthindex>,
+    pub yearindex_to_yearindex: ComputedVec<Yearindex, Yearindex>,
-    pub yearindex_to_timestamp: StorableVec<Yearindex, Timestamp>,
-    pub yearindex_to_yearindex: StorableVec<Yearindex, Yearindex>,
 }
 
 impl Vecs {
@@ -69,244 +67,244 @@ impl Vecs {
         fs::create_dir_all(path)?;
 
         Ok(Self {
-            dateindex_to_date: StorableVec::forced_import(
+            dateindex_to_date: ComputedVec::forced_import(
                 &path.join("dateindex_to_date"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_dateindex: StorableVec::forced_import(
+            dateindex_to_dateindex: ComputedVec::forced_import(
                 &path.join("dateindex_to_dateindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_first_height: StorableVec::forced_import(
+            dateindex_to_first_height: ComputedVec::forced_import(
                 &path.join("dateindex_to_first_height"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_last_height: StorableVec::forced_import(
+            dateindex_to_last_height: ComputedVec::forced_import(
                 &path.join("dateindex_to_last_height"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_real_date: StorableVec::forced_import(
+            height_to_real_date: ComputedVec::forced_import(
                 &path.join("height_to_real_date"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_fixed_date: StorableVec::forced_import(
+            height_to_fixed_date: ComputedVec::forced_import(
                 &path.join("height_to_fixed_date"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_dateindex: StorableVec::forced_import(
+            height_to_dateindex: ComputedVec::forced_import(
                 &path.join("height_to_dateindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_height: StorableVec::forced_import(
+            height_to_height: ComputedVec::forced_import(
                 &path.join("height_to_height"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_last_txindex: StorableVec::forced_import(
+            height_to_last_txindex: ComputedVec::forced_import(
                 &path.join("height_to_last_txindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            txindex_to_last_txinindex: StorableVec::forced_import(
+            txindex_to_last_txinindex: ComputedVec::forced_import(
                 &path.join("txindex_to_last_txinindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            txindex_to_last_txoutindex: StorableVec::forced_import(
+            txindex_to_last_txoutindex: ComputedVec::forced_import(
                 &path.join("txindex_to_last_txoutindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            difficultyepoch_to_first_height: StorableVec::forced_import(
+            difficultyepoch_to_first_height: ComputedVec::forced_import(
                 &path.join("difficultyepoch_to_first_height"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            difficultyepoch_to_last_height: StorableVec::forced_import(
+            difficultyepoch_to_last_height: ComputedVec::forced_import(
                 &path.join("difficultyepoch_to_last_height"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            halvingepoch_to_first_height: StorableVec::forced_import(
+            halvingepoch_to_first_height: ComputedVec::forced_import(
                 &path.join("halvingepoch_to_first_height"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            halvingepoch_to_last_height: StorableVec::forced_import(
+            halvingepoch_to_last_height: ComputedVec::forced_import(
                 &path.join("halvingepoch_to_last_height"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            weekindex_to_first_dateindex: StorableVec::forced_import(
+            weekindex_to_first_dateindex: ComputedVec::forced_import(
                 &path.join("weekindex_to_first_dateindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            weekindex_to_last_dateindex: StorableVec::forced_import(
+            weekindex_to_last_dateindex: ComputedVec::forced_import(
                 &path.join("weekindex_to_last_dateindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            monthindex_to_first_dateindex: StorableVec::forced_import(
+            monthindex_to_first_dateindex: ComputedVec::forced_import(
                 &path.join("monthindex_to_first_dateindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            monthindex_to_last_dateindex: StorableVec::forced_import(
+            monthindex_to_last_dateindex: ComputedVec::forced_import(
                 &path.join("monthindex_to_last_dateindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            yearindex_to_first_monthindex: StorableVec::forced_import(
+            yearindex_to_first_monthindex: ComputedVec::forced_import(
                 &path.join("yearindex_to_first_monthindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            yearindex_to_last_monthindex: StorableVec::forced_import(
+            yearindex_to_last_monthindex: ComputedVec::forced_import(
                 &path.join("yearindex_to_last_monthindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            decadeindex_to_first_yearindex: StorableVec::forced_import(
+            decadeindex_to_first_yearindex: ComputedVec::forced_import(
                 &path.join("decadeindex_to_first_yearindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            decadeindex_to_last_yearindex: StorableVec::forced_import(
+            decadeindex_to_last_yearindex: ComputedVec::forced_import(
                 &path.join("decadeindex_to_last_yearindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_weekindex: StorableVec::forced_import(
+            dateindex_to_weekindex: ComputedVec::forced_import(
                 &path.join("dateindex_to_weekindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_monthindex: StorableVec::forced_import(
+            dateindex_to_monthindex: ComputedVec::forced_import(
                 &path.join("dateindex_to_monthindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            monthindex_to_yearindex: StorableVec::forced_import(
+            monthindex_to_yearindex: ComputedVec::forced_import(
                 &path.join("monthindex_to_yearindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            yearindex_to_decadeindex: StorableVec::forced_import(
+            yearindex_to_decadeindex: ComputedVec::forced_import(
                 &path.join("yearindex_to_decadeindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_difficultyepoch: StorableVec::forced_import(
+            height_to_difficultyepoch: ComputedVec::forced_import(
                 &path.join("height_to_difficultyepoch"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_halvingepoch: StorableVec::forced_import(
+            height_to_halvingepoch: ComputedVec::forced_import(
                 &path.join("height_to_halvingepoch"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            weekindex_to_weekindex: StorableVec::forced_import(
+            weekindex_to_weekindex: ComputedVec::forced_import(
                 &path.join("weekindex_to_weekindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            monthindex_to_monthindex: StorableVec::forced_import(
+            monthindex_to_monthindex: ComputedVec::forced_import(
                 &path.join("monthindex_to_monthindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            yearindex_to_yearindex: StorableVec::forced_import(
+            yearindex_to_yearindex: ComputedVec::forced_import(
                 &path.join("yearindex_to_yearindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            decadeindex_to_decadeindex: StorableVec::forced_import(
+            decadeindex_to_decadeindex: ComputedVec::forced_import(
                 &path.join("decadeindex_to_decadeindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            difficultyepoch_to_difficultyepoch: StorableVec::forced_import(
+            difficultyepoch_to_difficultyepoch: ComputedVec::forced_import(
                 &path.join("difficultyepoch_to_difficultyepoch"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            halvingepoch_to_halvingepoch: StorableVec::forced_import(
+            halvingepoch_to_halvingepoch: ComputedVec::forced_import(
                 &path.join("halvingepoch_to_halvingepoch"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_timestamp: StorableVec::forced_import(
+            dateindex_to_timestamp: ComputedVec::forced_import(
                 &path.join("dateindex_to_timestamp"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            decadeindex_to_timestamp: StorableVec::forced_import(
+            decadeindex_to_timestamp: ComputedVec::forced_import(
                 &path.join("decadeindex_to_timestamp"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            difficultyepoch_to_timestamp: StorableVec::forced_import(
+            difficultyepoch_to_timestamp: ComputedVec::forced_import(
                 &path.join("difficultyepoch_to_timestamp"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            halvingepoch_to_timestamp: StorableVec::forced_import(
+            halvingepoch_to_timestamp: ComputedVec::forced_import(
                 &path.join("halvingepoch_to_timestamp"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            monthindex_to_timestamp: StorableVec::forced_import(
+            monthindex_to_timestamp: ComputedVec::forced_import(
                 &path.join("monthindex_to_timestamp"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            weekindex_to_timestamp: StorableVec::forced_import(
+            weekindex_to_timestamp: ComputedVec::forced_import(
                 &path.join("weekindex_to_timestamp"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            yearindex_to_timestamp: StorableVec::forced_import(
+            yearindex_to_timestamp: ComputedVec::forced_import(
                 &path.join("yearindex_to_timestamp"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_fixed_timestamp: StorableVec::forced_import(
+            height_to_fixed_timestamp: ComputedVec::forced_import(
                 &path.join("height_to_fixed_timestamp"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            monthindex_to_quarterindex: StorableVec::forced_import(
+            monthindex_to_quarterindex: ComputedVec::forced_import(
                 &path.join("monthindex_to_quarterindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            quarterindex_to_first_monthindex: StorableVec::forced_import(
+            quarterindex_to_first_monthindex: ComputedVec::forced_import(
                 &path.join("quarterindex_to_first_monthindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            quarterindex_to_last_monthindex: StorableVec::forced_import(
+            quarterindex_to_last_monthindex: ComputedVec::forced_import(
                 &path.join("quarterindex_to_last_monthindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            quarterindex_to_quarterindex: StorableVec::forced_import(
+            quarterindex_to_quarterindex: ComputedVec::forced_import(
                 &path.join("quarterindex_to_quarterindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            quarterindex_to_timestamp: StorableVec::forced_import(
+            quarterindex_to_timestamp: ComputedVec::forced_import(
                 &path.join("quarterindex_to_timestamp"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
         })
@@ -320,7 +318,7 @@ impl Vecs {
     ) -> color_eyre::Result<Indexes> {
         let indexer_vecs = indexer.mut_vecs();
 
-        let height_count = indexer_vecs.height_to_size.len();
+        let height_count = indexer_vecs.height_to_total_size.len();
         let txindexes_count = indexer_vecs.txindex_to_txid.len();
         let txinindexes_count = indexer_vecs.txinindex_to_txoutindex.len();
         let txoutindexes_count = indexer_vecs.txoutindex_to_addressindex.len();
@@ -345,7 +343,7 @@ impl Vecs {
             |(h, d, s, ..)| {
                 let d = h
                     .decremented()
-                    .and_then(|h| s.get(h).ok())
+                    .and_then(|h| s.cached_get(h).ok())
                     .flatten()
                     .map_or(d, |prev_d| {
                         let prev_d = *prev_d;
@@ -363,11 +361,12 @@ impl Vecs {
             exit,
         )?;
 
+        let decremented_starting_height = starting_indexes.height.decremented().unwrap_or_default();
+
         let starting_dateindex = self
             .height_to_dateindex
-            .get(starting_indexes.height.decremented().unwrap_or_default())?
+            .cached_get(decremented_starting_height)?
-            .copied()
+            .map_or_else(Default::default, |v| v.into_inner());
-            .unwrap_or_default();
 
         self.height_to_dateindex.compute_transform(
             starting_indexes.height,
@@ -378,8 +377,8 @@ impl Vecs {
 
         let starting_dateindex = if let Some(dateindex) = self
             .height_to_dateindex
-            .get(starting_indexes.height.decremented().unwrap_or_default())?
+            .cached_get(decremented_starting_height)?
-            .copied()
+            .map(|v| v.into_inner())
         {
             starting_dateindex.min(dateindex)
         } else {
@@ -451,9 +450,8 @@ impl Vecs {
 
         let starting_weekindex = self
             .dateindex_to_weekindex
-            .get(starting_dateindex)?
+            .cached_get(starting_dateindex)?
-            .copied()
+            .map_or_else(Default::default, |v| v.into_inner());
-            .unwrap_or_default();
 
         self.dateindex_to_weekindex.compute_transform(
             starting_dateindex,
@@ -487,7 +485,12 @@ impl Vecs {
         self.weekindex_to_timestamp.compute_transform(
             starting_weekindex,
             self.weekindex_to_first_dateindex.mut_vec(),
-            |(i, d, ..)| (i, *self.dateindex_to_timestamp.get(d).unwrap().unwrap()),
+            |(i, d, ..)| {
+                (
+                    i,
+                    *self.dateindex_to_timestamp.cached_get(d).unwrap().unwrap(),
+                )
+            },
             exit,
         )?;
 
@@ -495,9 +498,8 @@ impl Vecs {
 
         let starting_monthindex = self
             .dateindex_to_monthindex
-            .get(starting_dateindex)?
+            .cached_get(starting_dateindex)?
-            .copied()
+            .map_or_else(Default::default, |v| v.into_inner());
-            .unwrap_or_default();
 
         self.dateindex_to_monthindex.compute_transform(
             starting_dateindex,
@@ -533,7 +535,12 @@ impl Vecs {
         self.monthindex_to_timestamp.compute_transform(
             starting_monthindex,
             self.monthindex_to_first_dateindex.mut_vec(),
-            |(i, d, ..)| (i, *self.dateindex_to_timestamp.get(d).unwrap().unwrap()),
+            |(i, d, ..)| {
+                (
+                    i,
+                    *self.dateindex_to_timestamp.cached_get(d).unwrap().unwrap(),
+                )
+            },
             exit,
         )?;
 
@@ -541,9 +548,8 @@ impl Vecs {
 
         let starting_quarterindex = self
             .monthindex_to_quarterindex
-            .get(starting_monthindex)?
+            .cached_get(starting_monthindex)?
-            .copied()
+            .map_or_else(Default::default, |v| v.into_inner());
-            .unwrap_or_default();
 
         self.monthindex_to_quarterindex.compute_transform(
             starting_monthindex,
@@ -579,7 +585,12 @@ impl Vecs {
         self.quarterindex_to_timestamp.compute_transform(
             starting_quarterindex,
             self.quarterindex_to_first_monthindex.mut_vec(),
-            |(i, m, ..)| (i, *self.monthindex_to_timestamp.get(m).unwrap().unwrap()),
+            |(i, m, ..)| {
+                (
+                    i,
+                    *self.monthindex_to_timestamp.cached_get(m).unwrap().unwrap(),
+                )
+            },
             exit,
         )?;
 
@@ -587,9 +598,8 @@ impl Vecs {
 
         let starting_yearindex = self
             .monthindex_to_yearindex
-            .get(starting_monthindex)?
+            .cached_get(starting_monthindex)?
-            .copied()
+            .map_or_else(Default::default, |v| v.into_inner());
-            .unwrap_or_default();
 
         self.monthindex_to_yearindex.compute_transform(
             starting_monthindex,
@@ -625,7 +635,12 @@ impl Vecs {
         self.yearindex_to_timestamp.compute_transform(
             starting_yearindex,
             self.yearindex_to_first_monthindex.mut_vec(),
-            |(i, m, ..)| (i, *self.monthindex_to_timestamp.get(m).unwrap().unwrap()),
+            |(i, m, ..)| {
+                (
+                    i,
+                    *self.monthindex_to_timestamp.cached_get(m).unwrap().unwrap(),
+                )
+            },
             exit,
         )?;
 
@@ -633,9 +648,8 @@ impl Vecs {
 
         let starting_decadeindex = self
             .yearindex_to_decadeindex
-            .get(starting_yearindex)?
+            .cached_get(starting_yearindex)?
-            .copied()
+            .map_or_else(Default::default, |v| v.into_inner());
-            .unwrap_or_default();
 
         self.yearindex_to_decadeindex.compute_transform(
             starting_yearindex,
@@ -669,7 +683,12 @@ impl Vecs {
         self.decadeindex_to_timestamp.compute_transform(
             starting_decadeindex,
             self.decadeindex_to_first_yearindex.mut_vec(),
-            |(i, y, ..)| (i, *self.yearindex_to_timestamp.get(y).unwrap().unwrap()),
+            |(i, y, ..)| {
+                (
+                    i,
+                    *self.yearindex_to_timestamp.cached_get(y).unwrap().unwrap(),
+                )
+            },
             exit,
         )?;
 
@@ -677,9 +696,8 @@ impl Vecs {
 
         let starting_difficultyepoch = self
             .height_to_difficultyepoch
-            .get(starting_indexes.height)?
+            .cached_get(decremented_starting_height)?
-            .copied()
+            .map_or_else(Default::default, |v| v.into_inner());
-            .unwrap_or_default();
 
         self.height_to_difficultyepoch.compute_transform(
             starting_indexes.height,
@@ -716,7 +734,11 @@ impl Vecs {
             |(i, h, ..)| {
                 (
                     i,
-                    *indexer_vecs.height_to_timestamp.get(h).unwrap().unwrap(),
+                    *indexer_vecs
+                        .height_to_timestamp
+                        .cached_get(h)
+                        .unwrap()
+                        .unwrap(),
                 )
             },
             exit,
@@ -726,9 +748,8 @@ impl Vecs {
 
         let starting_halvingepoch = self
             .height_to_halvingepoch
-            .get(starting_indexes.height)?
+            .cached_get(decremented_starting_height)?
-            .copied()
+            .map_or_else(Default::default, |v| v.into_inner());
-            .unwrap_or_default();
 
         self.height_to_halvingepoch.compute_transform(
             starting_indexes.height,
@@ -765,7 +786,7 @@ impl Vecs {
         //     |(i, h, ..)| {
         //         (
         //             i,
-        //             *indexer_vecs.height_to_timestamp.get(h).unwrap().unwrap(),
+        //             *indexer_vecs.height_to_timestamp.cached_get(h).unwrap().unwrap(),
         //         )
         //     },
         //     exit,
@@ -784,7 +805,7 @@ impl Vecs {
         })
     }
 
-    pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
+    pub fn as_any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
         vec![
             self.dateindex_to_date.any_vec(),
             self.dateindex_to_dateindex.any_vec(),

crates/brk_computer/src/storage/vecs/marketprice.rs

@@ -7,252 +7,218 @@ use brk_core::{
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
-use brk_vec::{AnyStorableVec, Compressed, Version};
+use brk_vec::{Compressed, Version};
 
 use super::{
-    Indexes, StorableVec, indexes,
+    ComputedVec, Indexes,
-    stats::{
+    grouped::{
-        StorableVecGeneatorOptions, StorableVecsStatsFromDate, StorableVecsStatsFromHeightStrict,
+        ComputedVecsFromDateindex, ComputedVecsFromHeightStrict, StorableVecGeneatorOptions,
     },
+    indexes,
 };
 
 #[derive(Clone)]
 pub struct Vecs {
-    pub dateindex_to_close: StorableVec<Dateindex, Close<Dollars>>,
+    pub dateindex_to_close_in_cents: ComputedVec<Dateindex, Close<Cents>>,
-    pub dateindex_to_close_in_cents: StorableVec<Dateindex, Close<Cents>>,
+    pub dateindex_to_high_in_cents: ComputedVec<Dateindex, High<Cents>>,
-    pub dateindex_to_high: StorableVec<Dateindex, High<Dollars>>,
+    pub dateindex_to_low_in_cents: ComputedVec<Dateindex, Low<Cents>>,
-    pub dateindex_to_high_in_cents: StorableVec<Dateindex, High<Cents>>,
+    pub dateindex_to_ohlc: ComputedVec<Dateindex, OHLCDollars>,
-    pub dateindex_to_low: StorableVec<Dateindex, Low<Dollars>>,
+    pub dateindex_to_ohlc_in_cents: ComputedVec<Dateindex, OHLCCents>,
-    pub dateindex_to_low_in_cents: StorableVec<Dateindex, Low<Cents>>,
+    pub dateindex_to_open_in_cents: ComputedVec<Dateindex, Open<Cents>>,
-    pub dateindex_to_ohlc: StorableVec<Dateindex, OHLCDollars>,
+    pub height_to_close_in_cents: ComputedVec<Height, Close<Cents>>,
-    pub dateindex_to_ohlc_in_cents: StorableVec<Dateindex, OHLCCents>,
+    pub height_to_high_in_cents: ComputedVec<Height, High<Cents>>,
-    pub dateindex_to_open: StorableVec<Dateindex, Open<Dollars>>,
+    pub height_to_low_in_cents: ComputedVec<Height, Low<Cents>>,
-    pub dateindex_to_open_in_cents: StorableVec<Dateindex, Open<Cents>>,
+    pub height_to_ohlc: ComputedVec<Height, OHLCDollars>,
-    pub dateindex_to_sats_per_dollar: StorableVec<Dateindex, Close<Sats>>,
+    pub height_to_ohlc_in_cents: ComputedVec<Height, OHLCCents>,
-    pub height_to_close: StorableVec<Height, Close<Dollars>>,
+    pub height_to_open_in_cents: ComputedVec<Height, Open<Cents>>,
-    pub height_to_close_in_cents: StorableVec<Height, Close<Cents>>,
+    pub timeindexes_to_close: ComputedVecsFromDateindex<Close<Dollars>>,
-    pub height_to_high: StorableVec<Height, High<Dollars>>,
+    pub timeindexes_to_high: ComputedVecsFromDateindex<High<Dollars>>,
-    pub height_to_high_in_cents: StorableVec<Height, High<Cents>>,
+    pub timeindexes_to_low: ComputedVecsFromDateindex<Low<Dollars>>,
-    pub height_to_low: StorableVec<Height, Low<Dollars>>,
+    pub timeindexes_to_open: ComputedVecsFromDateindex<Open<Dollars>>,
-    pub height_to_low_in_cents: StorableVec<Height, Low<Cents>>,
+    pub timeindexes_to_sats_per_dollar: ComputedVecsFromDateindex<Close<Sats>>,
-    pub height_to_ohlc: StorableVec<Height, OHLCDollars>,
+    pub chainindexes_to_close: ComputedVecsFromHeightStrict<Close<Dollars>>,
-    pub height_to_ohlc_in_cents: StorableVec<Height, OHLCCents>,
+    pub chainindexes_to_high: ComputedVecsFromHeightStrict<High<Dollars>>,
-    pub height_to_open: StorableVec<Height, Open<Dollars>>,
+    pub chainindexes_to_low: ComputedVecsFromHeightStrict<Low<Dollars>>,
-    pub height_to_open_in_cents: StorableVec<Height, Open<Cents>>,
+    pub chainindexes_to_open: ComputedVecsFromHeightStrict<Open<Dollars>>,
-    pub height_to_sats_per_dollar: StorableVec<Height, Close<Sats>>,
+    pub chainindexes_to_sats_per_dollar: ComputedVecsFromHeightStrict<Close<Sats>>,
-    pub timeindexes_to_close: StorableVecsStatsFromDate<Close<Dollars>>,
+    pub weekindex_to_ohlc: ComputedVec<Weekindex, OHLCDollars>,
-    pub timeindexes_to_high: StorableVecsStatsFromDate<High<Dollars>>,
+    pub difficultyepoch_to_ohlc: ComputedVec<Difficultyepoch, OHLCDollars>,
-    pub timeindexes_to_low: StorableVecsStatsFromDate<Low<Dollars>>,
+    pub monthindex_to_ohlc: ComputedVec<Monthindex, OHLCDollars>,
-    pub timeindexes_to_open: StorableVecsStatsFromDate<Open<Dollars>>,
+    pub quarterindex_to_ohlc: ComputedVec<Quarterindex, OHLCDollars>,
-    pub timeindexes_to_sats_per_dollar: StorableVecsStatsFromDate<Close<Sats>>,
+    pub yearindex_to_ohlc: ComputedVec<Yearindex, OHLCDollars>,
-    pub chainindexes_to_close: StorableVecsStatsFromHeightStrict<Close<Dollars>>,
-    pub chainindexes_to_high: StorableVecsStatsFromHeightStrict<High<Dollars>>,
-    pub chainindexes_to_low: StorableVecsStatsFromHeightStrict<Low<Dollars>>,
-    pub chainindexes_to_open: StorableVecsStatsFromHeightStrict<Open<Dollars>>,
-    pub chainindexes_to_sats_per_dollar: StorableVecsStatsFromHeightStrict<Close<Sats>>,
-    pub weekindex_to_ohlc: StorableVec<Weekindex, OHLCDollars>,
-    pub difficultyepoch_to_ohlc: StorableVec<Difficultyepoch, OHLCDollars>,
-    pub monthindex_to_ohlc: StorableVec<Monthindex, OHLCDollars>,
-    pub quarterindex_to_ohlc: StorableVec<Quarterindex, OHLCDollars>,
-    pub yearindex_to_ohlc: StorableVec<Yearindex, OHLCDollars>,
     // pub halvingepoch_to_ohlc: StorableVec<Halvingepoch, OHLCDollars>,
-    pub decadeindex_to_ohlc: StorableVec<Decadeindex, OHLCDollars>,
+    pub decadeindex_to_ohlc: ComputedVec<Decadeindex, OHLCDollars>,
 }
 
 impl Vecs {
     pub fn forced_import(path: &Path, compressed: Compressed) -> color_eyre::Result<Self> {
         fs::create_dir_all(path)?;
 
+        let mut fetched_path = path.to_owned();
+        fetched_path.pop();
+        fetched_path.pop();
+        fetched_path = fetched_path.join("fetched/vecs");
+
         Ok(Self {
-            dateindex_to_ohlc_in_cents: StorableVec::forced_import(
+            dateindex_to_ohlc_in_cents: ComputedVec::forced_import(
-                &path.join("dateindex_to_ohlc_in_cents"),
+                &fetched_path.join("dateindex_to_ohlc_in_cents"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_ohlc: StorableVec::forced_import(
+            dateindex_to_ohlc: ComputedVec::forced_import(
                 &path.join("dateindex_to_ohlc"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_close_in_cents: StorableVec::forced_import(
+            dateindex_to_close_in_cents: ComputedVec::forced_import(
                 &path.join("dateindex_to_close_in_cents"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_close: StorableVec::forced_import(
+            dateindex_to_high_in_cents: ComputedVec::forced_import(
-                &path.join("dateindex_to_close"),
-                Version::ONE,
-                compressed,
-            )?,
-            dateindex_to_high_in_cents: StorableVec::forced_import(
                 &path.join("dateindex_to_high_in_cents"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_high: StorableVec::forced_import(
+            dateindex_to_low_in_cents: ComputedVec::forced_import(
-                &path.join("dateindex_to_high"),
-                Version::ONE,
-                compressed,
-            )?,
-            dateindex_to_low_in_cents: StorableVec::forced_import(
                 &path.join("dateindex_to_low_in_cents"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_low: StorableVec::forced_import(
+            dateindex_to_open_in_cents: ComputedVec::forced_import(
-                &path.join("dateindex_to_low"),
-                Version::ONE,
-                compressed,
-            )?,
-            dateindex_to_open_in_cents: StorableVec::forced_import(
                 &path.join("dateindex_to_open_in_cents"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_open: StorableVec::forced_import(
+            height_to_ohlc_in_cents: ComputedVec::forced_import(
-                &path.join("dateindex_to_open"),
+                &fetched_path.join("height_to_ohlc_in_cents"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            dateindex_to_sats_per_dollar: StorableVec::forced_import(
+            height_to_ohlc: ComputedVec::forced_import(
-                &path.join("dateindex_to_sats_per_dollar"),
-                Version::ONE,
-                compressed,
-            )?,
-            height_to_ohlc_in_cents: StorableVec::forced_import(
-                &path.join("height_to_ohlc_in_cents"),
-                Version::ONE,
-                compressed,
-            )?,
-            height_to_ohlc: StorableVec::forced_import(
                 &path.join("height_to_ohlc"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_close_in_cents: StorableVec::forced_import(
+            height_to_close_in_cents: ComputedVec::forced_import(
                 &path.join("height_to_close_in_cents"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_close: StorableVec::forced_import(
+            height_to_high_in_cents: ComputedVec::forced_import(
-                &path.join("height_to_close"),
-                Version::ONE,
-                compressed,
-            )?,
-            height_to_high_in_cents: StorableVec::forced_import(
                 &path.join("height_to_high_in_cents"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_high: StorableVec::forced_import(
+            height_to_low_in_cents: ComputedVec::forced_import(
-                &path.join("height_to_high"),
-                Version::ONE,
-                compressed,
-            )?,
-            height_to_low_in_cents: StorableVec::forced_import(
                 &path.join("height_to_low_in_cents"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_low: StorableVec::forced_import(
+            height_to_open_in_cents: ComputedVec::forced_import(
-                &path.join("height_to_low"),
-                Version::ONE,
-                compressed,
-            )?,
-            height_to_open_in_cents: StorableVec::forced_import(
                 &path.join("height_to_open_in_cents"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_open: StorableVec::forced_import(
+            timeindexes_to_open: ComputedVecsFromDateindex::forced_import(
-                &path.join("height_to_open"),
+                path,
-                Version::ONE,
+                "open",
-                compressed,
+                Version::ZERO,
-            )?,
-            height_to_sats_per_dollar: StorableVec::forced_import(
-                &path.join("height_to_sats_per_dollar"),
-                Version::ONE,
-                compressed,
-            )?,
-            timeindexes_to_open: StorableVecsStatsFromDate::forced_import(
-                &path.join("open"),
                 compressed,
                 StorableVecGeneatorOptions::default().add_first(),
             )?,
-            timeindexes_to_high: StorableVecsStatsFromDate::forced_import(
+            timeindexes_to_high: ComputedVecsFromDateindex::forced_import(
-                &path.join("high"),
+                path,
+                "high",
+                Version::ZERO,
                 compressed,
                 StorableVecGeneatorOptions::default().add_max(),
             )?,
-            timeindexes_to_low: StorableVecsStatsFromDate::forced_import(
+            timeindexes_to_low: ComputedVecsFromDateindex::forced_import(
-                &path.join("low"),
+                path,
+                "low",
+                Version::ZERO,
                 compressed,
                 StorableVecGeneatorOptions::default().add_min(),
             )?,
-            timeindexes_to_close: StorableVecsStatsFromDate::forced_import(
+            timeindexes_to_close: ComputedVecsFromDateindex::forced_import(
-                &path.join("close"),
+                path,
+                "close",
+                Version::ZERO,
                 compressed,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
-            timeindexes_to_sats_per_dollar: StorableVecsStatsFromDate::forced_import(
+            timeindexes_to_sats_per_dollar: ComputedVecsFromDateindex::forced_import(
-                &path.join("sats_per_dollar"),
+                path,
+                "sats_per_dollar",
+                Version::ZERO,
                 compressed,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
-            chainindexes_to_open: StorableVecsStatsFromHeightStrict::forced_import(
+            chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import(
-                &path.join("open"),
+                path,
+                "open",
+                Version::ZERO,
                 compressed,
                 StorableVecGeneatorOptions::default().add_first(),
             )?,
-            chainindexes_to_high: StorableVecsStatsFromHeightStrict::forced_import(
+            chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import(
-                &path.join("high"),
+                path,
+                "high",
+                Version::ZERO,
                 compressed,
                 StorableVecGeneatorOptions::default().add_max(),
             )?,
-            chainindexes_to_low: StorableVecsStatsFromHeightStrict::forced_import(
+            chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import(
-                &path.join("low"),
+                path,
+                "low",
+                Version::ZERO,
                 compressed,
                 StorableVecGeneatorOptions::default().add_min(),
             )?,
-            chainindexes_to_close: StorableVecsStatsFromHeightStrict::forced_import(
+            chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import(
-                &path.join("close"),
+                path,
+                "close",
+                Version::ZERO,
                 compressed,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
-            chainindexes_to_sats_per_dollar: StorableVecsStatsFromHeightStrict::forced_import(
+            chainindexes_to_sats_per_dollar: ComputedVecsFromHeightStrict::forced_import(
-                &path.join("sats_per_dollar"),
+                path,
+                "sats_per_dollar",
+                Version::ZERO,
                 compressed,
                 StorableVecGeneatorOptions::default().add_last(),
             )?,
-            weekindex_to_ohlc: StorableVec::forced_import(
+            weekindex_to_ohlc: ComputedVec::forced_import(
                 &path.join("weekindex_to_ohlc"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            difficultyepoch_to_ohlc: StorableVec::forced_import(
+            difficultyepoch_to_ohlc: ComputedVec::forced_import(
                 &path.join("difficultyepoch_to_ohlc"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            monthindex_to_ohlc: StorableVec::forced_import(
+            monthindex_to_ohlc: ComputedVec::forced_import(
                 &path.join("monthindex_to_ohlc"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            quarterindex_to_ohlc: StorableVec::forced_import(
+            quarterindex_to_ohlc: ComputedVec::forced_import(
                 &path.join("quarterindex_to_ohlc"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            yearindex_to_ohlc: StorableVec::forced_import(
+            yearindex_to_ohlc: ComputedVec::forced_import(
                 &path.join("yearindex_to_ohlc"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            // halvingepoch_to_ohlc: StorableVec::forced_import(&path.join("halvingepoch_to_ohlc"), Version::ONE, compressed)?,
+            // halvingepoch_to_ohlc: StorableVec::forced_import(&path.join("halvingepoch_to_ohlc"), Version::ZERO, compressed)?,
-            decadeindex_to_ohlc: StorableVec::forced_import(
+            decadeindex_to_ohlc: ComputedVec::forced_import(
                 &path.join("decadeindex_to_ohlc"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
         })
@@ -276,8 +242,9 @@ impl Vecs {
                     .get_height(
                         h,
                         t,
-                        h.decremented()
+                        h.decremented().map(|prev_h| {
-                            .map(|prev_h| *height_to_timestamp.get(prev_h).unwrap().unwrap()),
+                            *height_to_timestamp.cached_get(prev_h).unwrap().unwrap()
+                        }),
                     )
                     .unwrap();
                 (h, ohlc)
@@ -320,41 +287,6 @@ impl Vecs {
             exit,
         )?;
 
-        self.height_to_open.compute_transform(
-            starting_indexes.height,
-            self.height_to_ohlc.mut_vec(),
-            |(di, ohlc, ..)| (di, ohlc.open),
-            exit,
-        )?;
-
-        self.height_to_high.compute_transform(
-            starting_indexes.height,
-            self.height_to_ohlc.mut_vec(),
-            |(di, ohlc, ..)| (di, ohlc.high),
-            exit,
-        )?;
-
-        self.height_to_low.compute_transform(
-            starting_indexes.height,
-            self.height_to_ohlc.mut_vec(),
-            |(di, ohlc, ..)| (di, ohlc.low),
-            exit,
-        )?;
-
-        self.height_to_close.compute_transform(
-            starting_indexes.height,
-            self.height_to_ohlc.mut_vec(),
-            |(di, ohlc, ..)| (di, ohlc.close),
-            exit,
-        )?;
-
-        self.height_to_sats_per_dollar.compute_transform(
-            starting_indexes.height,
-            self.height_to_close.mut_vec(),
-            |(di, close, ..)| (di, Close::from(Sats::ONE_BTC / *close)),
-            exit,
-        )?;
-
         self.dateindex_to_ohlc_in_cents.compute_transform(
             starting_indexes.dateindex,
             indexes.dateindex_to_date.mut_vec(),
@@ -400,95 +332,124 @@ impl Vecs {
             exit,
         )?;
 
-        self.dateindex_to_open.compute_transform(
-            starting_indexes.dateindex,
-            self.dateindex_to_ohlc.mut_vec(),
-            |(di, ohlc, ..)| (di, ohlc.open),
-            exit,
-        )?;
-
-        self.dateindex_to_high.compute_transform(
-            starting_indexes.dateindex,
-            self.dateindex_to_ohlc.mut_vec(),
-            |(di, ohlc, ..)| (di, ohlc.high),
-            exit,
-        )?;
-
-        self.dateindex_to_low.compute_transform(
-            starting_indexes.dateindex,
-            self.dateindex_to_ohlc.mut_vec(),
-            |(di, ohlc, ..)| (di, ohlc.low),
-            exit,
-        )?;
-
-        self.dateindex_to_close.compute_transform(
-            starting_indexes.dateindex,
-            self.dateindex_to_ohlc.mut_vec(),
-            |(di, ohlc, ..)| (di, ohlc.close),
-            exit,
-        )?;
-
-        self.dateindex_to_sats_per_dollar.compute_transform(
-            starting_indexes.dateindex,
-            self.dateindex_to_close.mut_vec(),
-            |(di, close, ..)| (di, Close::from(Sats::ONE_BTC / *close)),
-            exit,
-        )?;
-
         self.timeindexes_to_close.compute(
-            &mut self.dateindex_to_close,
+            indexer,
             indexes,
             starting_indexes,
             exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.dateindex,
+                    self.dateindex_to_ohlc.mut_vec(),
+                    |(di, ohlc, ..)| (di, ohlc.close),
+                    exit,
+                )
+            },
         )?;
 
         self.timeindexes_to_high.compute(
-            &mut self.dateindex_to_high,
+            indexer,
             indexes,
             starting_indexes,
             exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.dateindex,
+                    self.dateindex_to_ohlc.mut_vec(),
+                    |(di, ohlc, ..)| (di, ohlc.high),
+                    exit,
+                )
+            },
         )?;
 
         self.timeindexes_to_low.compute(
-            &mut self.dateindex_to_low,
+            indexer,
             indexes,
             starting_indexes,
             exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.dateindex,
+                    self.dateindex_to_ohlc.mut_vec(),
+                    |(di, ohlc, ..)| (di, ohlc.low),
+                    exit,
+                )
+            },
         )?;
 
         self.timeindexes_to_open.compute(
-            &mut self.dateindex_to_open,
+            indexer,
             indexes,
             starting_indexes,
             exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.dateindex,
+                    self.dateindex_to_ohlc.mut_vec(),
+                    |(di, ohlc, ..)| (di, ohlc.open),
+                    exit,
+                )
+            },
         )?;
 
         self.chainindexes_to_close.compute(
-            &mut self.height_to_close,
+            indexer,
             indexes,
             starting_indexes,
             exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.height,
+                    self.height_to_ohlc.mut_vec(),
+                    |(di, ohlc, ..)| (di, ohlc.close),
+                    exit,
+                )
+            },
         )?;
 
         self.chainindexes_to_high.compute(
-            &mut self.height_to_high,
+            indexer,
             indexes,
             starting_indexes,
             exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.height,
+                    self.height_to_ohlc.mut_vec(),
+                    |(di, ohlc, ..)| (di, ohlc.high),
+                    exit,
+                )
+            },
         )?;
 
         self.chainindexes_to_low.compute(
-            &mut self.height_to_low,
+            indexer,
             indexes,
             starting_indexes,
             exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.height,
+                    self.height_to_ohlc.mut_vec(),
+                    |(di, ohlc, ..)| (di, ohlc.low),
+                    exit,
+                )
+            },
         )?;
 
         self.chainindexes_to_open.compute(
-            &mut self.height_to_open,
+            indexer,
             indexes,
             starting_indexes,
             exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.height,
+                    self.height_to_ohlc.mut_vec(),
+                    |(di, ohlc, ..)| (di, ohlc.open),
+                    exit,
+                )
+            },
         )?;
 
         self.weekindex_to_ohlc.compute_transform(
@@ -509,7 +470,7 @@ impl Vecs {
                             .first
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         high: *self
@@ -518,7 +479,7 @@ impl Vecs {
                             .max
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         low: *self
@@ -527,7 +488,7 @@ impl Vecs {
                             .min
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         close,
@@ -555,7 +516,7 @@ impl Vecs {
                             .first
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         high: *self
@@ -564,7 +525,7 @@ impl Vecs {
                             .max
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         low: *self
@@ -573,7 +534,7 @@ impl Vecs {
                             .min
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         close,
@@ -601,7 +562,7 @@ impl Vecs {
                             .first
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         high: *self
@@ -610,7 +571,7 @@ impl Vecs {
                             .max
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         low: *self
@@ -619,7 +580,7 @@ impl Vecs {
                             .min
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         close,
@@ -647,7 +608,7 @@ impl Vecs {
                             .first
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         high: *self
@@ -656,7 +617,7 @@ impl Vecs {
                             .max
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         low: *self
@@ -665,7 +626,7 @@ impl Vecs {
                             .min
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         close,
@@ -693,7 +654,7 @@ impl Vecs {
                             .first
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         high: *self
@@ -702,7 +663,7 @@ impl Vecs {
                             .max
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         low: *self
@@ -711,7 +672,7 @@ impl Vecs {
                             .min
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         close,
@@ -743,7 +704,7 @@ impl Vecs {
                             .first
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         high: *self
@@ -752,7 +713,7 @@ impl Vecs {
                             .max
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         low: *self
@@ -761,7 +722,7 @@ impl Vecs {
                             .min
                             .as_mut()
                             .unwrap()
-                            .get(i)
+                            .cached_get(i)
                             .unwrap()
                             .unwrap(),
                         close,
@@ -771,34 +732,54 @@ impl Vecs {
             exit,
         )?;
 
+        self.chainindexes_to_sats_per_dollar.compute(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.height,
+                    self.chainindexes_to_close.height.mut_vec(),
+                    |(i, close, ..)| (i, Close::from(Sats::ONE_BTC / *close)),
+                    exit,
+                )
+            },
+        )?;
+
+        self.timeindexes_to_sats_per_dollar.compute(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |v, _, _, starting_indexes, exit| {
+                v.compute_transform(
+                    starting_indexes.dateindex,
+                    self.timeindexes_to_close.dateindex.mut_vec(),
+                    |(i, close, ..)| (i, Close::from(Sats::ONE_BTC / *close)),
+                    exit,
+                )
+            },
+        )?;
+
         Ok(())
     }
 
-    pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
+    pub fn as_any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
         vec![
             vec![
-                self.dateindex_to_close.any_vec(),
                 self.dateindex_to_close_in_cents.any_vec(),
-                self.dateindex_to_high.any_vec(),
                 self.dateindex_to_high_in_cents.any_vec(),
-                self.dateindex_to_low.any_vec(),
                 self.dateindex_to_low_in_cents.any_vec(),
                 self.dateindex_to_ohlc.any_vec(),
                 self.dateindex_to_ohlc_in_cents.any_vec(),
-                self.dateindex_to_open.any_vec(),
                 self.dateindex_to_open_in_cents.any_vec(),
-                self.dateindex_to_sats_per_dollar.any_vec(),
-                self.height_to_close.any_vec(),
                 self.height_to_close_in_cents.any_vec(),
-                self.height_to_high.any_vec(),
                 self.height_to_high_in_cents.any_vec(),
-                self.height_to_low.any_vec(),
                 self.height_to_low_in_cents.any_vec(),
                 self.height_to_ohlc.any_vec(),
                 self.height_to_ohlc_in_cents.any_vec(),
-                self.height_to_open.any_vec(),
                 self.height_to_open_in_cents.any_vec(),
-                self.height_to_sats_per_dollar.any_vec(),
                 self.weekindex_to_ohlc.any_vec(),
                 self.difficultyepoch_to_ohlc.any_vec(),
                 self.monthindex_to_ohlc.any_vec(),
@@ -807,14 +788,16 @@ impl Vecs {
                 // self.halvingepoch_to_ohlc.any_vec(),
                 self.decadeindex_to_ohlc.any_vec(),
             ],
-            self.timeindexes_to_close.as_any_vecs(),
+            self.chainindexes_to_sats_per_dollar.any_vecs(),
-            self.timeindexes_to_high.as_any_vecs(),
+            self.timeindexes_to_sats_per_dollar.any_vecs(),
-            self.timeindexes_to_low.as_any_vecs(),
+            self.timeindexes_to_close.any_vecs(),
-            self.timeindexes_to_open.as_any_vecs(),
+            self.timeindexes_to_high.any_vecs(),
-            self.chainindexes_to_close.as_any_vecs(),
+            self.timeindexes_to_low.any_vecs(),
-            self.chainindexes_to_high.as_any_vecs(),
+            self.timeindexes_to_open.any_vecs(),
-            self.chainindexes_to_low.as_any_vecs(),
+            self.chainindexes_to_close.any_vecs(),
-            self.chainindexes_to_open.as_any_vecs(),
+            self.chainindexes_to_high.any_vecs(),
+            self.chainindexes_to_low.any_vecs(),
+            self.chainindexes_to_open.any_vecs(),
         ]
         .concat()
     }

crates/brk_computer/src/storage/vecs/mod.rs

@@ -3,13 +3,13 @@ use std::{fs, path::Path};
 use brk_exit::Exit;
 use brk_fetcher::Fetcher;
 use brk_indexer::Indexer;
-use brk_vec::{AnyStorableVec, Compressed};
+use brk_vec::{AnyStoredVec, Compressed};
 
 mod base;
 mod blocks;
+mod grouped;
 mod indexes;
 mod marketprice;
-mod stats;
 mod transactions;
 
 use base::*;
@@ -63,7 +63,7 @@ impl Vecs {
         Ok(())
     }
 
-    pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
+    pub fn as_any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
         [
             self.indexes.as_any_vecs(),
             self.blocks.as_any_vecs(),

crates/brk_computer/src/storage/vecs/stats/from_date.rs

@@ -1,104 +0,0 @@
-use std::path::Path;
-
-use brk_core::{Dateindex, Decadeindex, Monthindex, Quarterindex, Weekindex, Yearindex};
-use brk_exit::Exit;
-use brk_vec::{AnyStorableVec, Compressed};
-
-use crate::storage::vecs::{Indexes, base::StorableVec, indexes};
-
-use super::{ComputedType, StorableVecBuilder, StorableVecGeneatorOptions};
-
-#[derive(Clone)]
-pub struct StorableVecsStatsFromDate<T>
-where
-    T: ComputedType + PartialOrd,
-{
-    pub weekindex: StorableVecBuilder<Weekindex, T>,
-    pub monthindex: StorableVecBuilder<Monthindex, T>,
-    pub quarterindex: StorableVecBuilder<Quarterindex, T>,
-    pub yearindex: StorableVecBuilder<Yearindex, T>,
-    pub decadeindex: StorableVecBuilder<Decadeindex, T>,
-}
-
-impl<T> StorableVecsStatsFromDate<T>
-where
-    T: ComputedType + Ord + From<f64>,
-    f64: From<T>,
-{
-    pub fn forced_import(
-        path: &Path,
-        compressed: Compressed,
-        options: StorableVecGeneatorOptions,
-    ) -> color_eyre::Result<Self> {
-        let options = options.remove_percentiles();
-
-        Ok(Self {
-            weekindex: StorableVecBuilder::forced_import(path, compressed, options)?,
-            monthindex: StorableVecBuilder::forced_import(path, compressed, options)?,
-            quarterindex: StorableVecBuilder::forced_import(path, compressed, options)?,
-            yearindex: StorableVecBuilder::forced_import(path, compressed, options)?,
-            decadeindex: StorableVecBuilder::forced_import(path, compressed, options)?,
-        })
-    }
-
-    pub fn compute(
-        &mut self,
-        source: &mut StorableVec<Dateindex, T>,
-        indexes: &mut indexes::Vecs,
-        starting_indexes: &Indexes,
-        exit: &Exit,
-    ) -> color_eyre::Result<()> {
-        self.weekindex.compute(
-            starting_indexes.weekindex,
-            source,
-            indexes.weekindex_to_first_dateindex.mut_vec(),
-            indexes.weekindex_to_last_dateindex.mut_vec(),
-            exit,
-        )?;
-
-        self.monthindex.compute(
-            starting_indexes.monthindex,
-            source,
-            indexes.monthindex_to_first_dateindex.mut_vec(),
-            indexes.monthindex_to_last_dateindex.mut_vec(),
-            exit,
-        )?;
-
-        self.quarterindex.from_aligned(
-            starting_indexes.quarterindex,
-            &mut self.monthindex,
-            indexes.quarterindex_to_first_monthindex.mut_vec(),
-            indexes.quarterindex_to_last_monthindex.mut_vec(),
-            exit,
-        )?;
-
-        self.yearindex.from_aligned(
-            starting_indexes.yearindex,
-            &mut self.monthindex,
-            indexes.yearindex_to_first_monthindex.mut_vec(),
-            indexes.yearindex_to_last_monthindex.mut_vec(),
-            exit,
-        )?;
-
-        self.decadeindex.from_aligned(
-            starting_indexes.decadeindex,
-            &mut self.yearindex,
-            indexes.decadeindex_to_first_yearindex.mut_vec(),
-            indexes.decadeindex_to_last_yearindex.mut_vec(),
-            exit,
-        )?;
-
-        Ok(())
-    }
-
-    pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
-        [
-            self.weekindex.as_any_vecs(),
-            self.monthindex.as_any_vecs(),
-            self.quarterindex.as_any_vecs(),
-            self.yearindex.as_any_vecs(),
-            self.decadeindex.as_any_vecs(),
-        ]
-        .concat()
-    }
-}

crates/brk_computer/src/storage/vecs/stats/from_height.rs

@@ -1,133 +0,0 @@
-use std::path::Path;
-
-use brk_core::{
-    Dateindex, Decadeindex, Difficultyepoch, Height, Monthindex, Quarterindex, Weekindex, Yearindex,
-};
-use brk_exit::Exit;
-use brk_vec::{AnyStorableVec, Compressed};
-
-use crate::storage::vecs::{Indexes, base::StorableVec, indexes};
-
-use super::{ComputedType, StorableVecBuilder, StorableVecGeneatorOptions};
-
-#[derive(Clone)]
-pub struct StorableVecsStatsFromHeight<T>
-where
-    T: ComputedType + PartialOrd,
-{
-    pub dateindex: StorableVecBuilder<Dateindex, T>,
-    pub weekindex: StorableVecBuilder<Weekindex, T>,
-    pub difficultyepoch: StorableVecBuilder<Difficultyepoch, T>,
-    pub monthindex: StorableVecBuilder<Monthindex, T>,
-    pub quarterindex: StorableVecBuilder<Quarterindex, T>,
-    pub yearindex: StorableVecBuilder<Yearindex, T>,
-    // TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
-    pub decadeindex: StorableVecBuilder<Decadeindex, T>,
-}
-
-impl<T> StorableVecsStatsFromHeight<T>
-where
-    T: ComputedType + Ord + From<f64>,
-    f64: From<T>,
-{
-    pub fn forced_import(
-        path: &Path,
-        compressed: Compressed,
-        options: StorableVecGeneatorOptions,
-    ) -> color_eyre::Result<Self> {
-        let dateindex = StorableVecBuilder::forced_import(path, compressed, options)?;
-
-        let options = options.remove_percentiles();
-
-        Ok(Self {
-            dateindex,
-            weekindex: StorableVecBuilder::forced_import(path, compressed, options)?,
-            difficultyepoch: StorableVecBuilder::forced_import(path, compressed, options)?,
-            monthindex: StorableVecBuilder::forced_import(path, compressed, options)?,
-            quarterindex: StorableVecBuilder::forced_import(path, compressed, options)?,
-            yearindex: StorableVecBuilder::forced_import(path, compressed, options)?,
-            // halvingepoch: StorableVecGeneator::forced_import(path, compressed, options)?,
-            decadeindex: StorableVecBuilder::forced_import(path, compressed, options)?,
-        })
-    }
-
-    pub fn compute(
-        &mut self,
-        source: &mut StorableVec<Height, T>,
-        indexes: &mut indexes::Vecs,
-        starting_indexes: &Indexes,
-        exit: &Exit,
-    ) -> color_eyre::Result<()> {
-        self.dateindex.compute(
-            starting_indexes.dateindex,
-            source,
-            indexes.dateindex_to_first_height.mut_vec(),
-            indexes.dateindex_to_last_height.mut_vec(),
-            exit,
-        )?;
-
-        self.weekindex.from_aligned(
-            starting_indexes.weekindex,
-            &mut self.dateindex,
-            indexes.weekindex_to_first_dateindex.mut_vec(),
-            indexes.weekindex_to_last_dateindex.mut_vec(),
-            exit,
-        )?;
-
-        self.monthindex.from_aligned(
-            starting_indexes.monthindex,
-            &mut self.dateindex,
-            indexes.monthindex_to_first_dateindex.mut_vec(),
-            indexes.monthindex_to_last_dateindex.mut_vec(),
-            exit,
-        )?;
-
-        self.quarterindex.from_aligned(
-            starting_indexes.quarterindex,
-            &mut self.monthindex,
-            indexes.quarterindex_to_first_monthindex.mut_vec(),
-            indexes.quarterindex_to_last_monthindex.mut_vec(),
-            exit,
-        )?;
-
-        self.yearindex.from_aligned(
-            starting_indexes.yearindex,
-            &mut self.monthindex,
-            indexes.yearindex_to_first_monthindex.mut_vec(),
-            indexes.yearindex_to_last_monthindex.mut_vec(),
-            exit,
-        )?;
-
-        self.decadeindex.from_aligned(
-            starting_indexes.decadeindex,
-            &mut self.yearindex,
-            indexes.decadeindex_to_first_yearindex.mut_vec(),
-            indexes.decadeindex_to_last_yearindex.mut_vec(),
-            exit,
-        )?;
-
-        self.difficultyepoch.compute(
-            starting_indexes.difficultyepoch,
-            source,
-            indexes.difficultyepoch_to_first_height.mut_vec(),
-            indexes.difficultyepoch_to_last_height.mut_vec(),
-            exit,
-        )?;
-
-        Ok(())
-    }
-
-    pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
-        [
-            self.dateindex.as_any_vecs(),
-            self.weekindex.as_any_vecs(),
-            self.difficultyepoch.as_any_vecs(),
-            self.monthindex.as_any_vecs(),
-            self.quarterindex.as_any_vecs(),
-            self.yearindex.as_any_vecs(),
-            // self.halvingepoch.as_any_vecs(),
-            self.decadeindex.as_any_vecs(),
-        ]
-        .concat()
-    }
-}

crates/brk_computer/src/storage/vecs/stats/from_height_strict.rs

@@ -1,63 +0,0 @@
-use std::path::Path;
-
-use brk_core::{Difficultyepoch, Height};
-use brk_exit::Exit;
-use brk_vec::{AnyStorableVec, Compressed};
-
-use crate::storage::vecs::{Indexes, base::StorableVec, indexes};
-
-use super::{ComputedType, StorableVecBuilder, StorableVecGeneatorOptions};
-
-#[derive(Clone)]
-pub struct StorableVecsStatsFromHeightStrict<T>
-where
-    T: ComputedType + PartialOrd,
-{
-    pub difficultyepoch: StorableVecBuilder<Difficultyepoch, T>,
-    // TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
-}
-
-impl<T> StorableVecsStatsFromHeightStrict<T>
-where
-    T: ComputedType + Ord + From<f64>,
-    f64: From<T>,
-{
-    pub fn forced_import(
-        path: &Path,
-        compressed: Compressed,
-        options: StorableVecGeneatorOptions,
-    ) -> color_eyre::Result<Self> {
-        let options = options.remove_percentiles();
-
-        Ok(Self {
-            difficultyepoch: StorableVecBuilder::forced_import(path, compressed, options)?,
-            // halvingepoch: StorableVecGeneator::forced_import(path, compressed, options)?,
-        })
-    }
-
-    pub fn compute(
-        &mut self,
-        source: &mut StorableVec<Height, T>,
-        indexes: &mut indexes::Vecs,
-        starting_indexes: &Indexes,
-        exit: &Exit,
-    ) -> color_eyre::Result<()> {
-        self.difficultyepoch.compute(
-            starting_indexes.difficultyepoch,
-            source,
-            indexes.difficultyepoch_to_first_height.mut_vec(),
-            indexes.difficultyepoch_to_last_height.mut_vec(),
-            exit,
-        )?;
-
-        Ok(())
-    }
-
-    pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
-        [
-            self.difficultyepoch.as_any_vecs(),
-            // self.halvingepoch.as_any_vecs(),
-        ]
-        .concat()
-    }
-}

crates/brk_computer/src/storage/vecs/transactions.rs

@@ -1,31 +1,44 @@
 use std::{fs, path::Path};
 
-use brk_core::Txindex;
+use brk_core::{
+    CheckedSub, Feerate, Sats, StoredU32, StoredU64, StoredUsize, TxVersion, Txindex, Txinindex,
+    Txoutindex, Weight,
+};
 use brk_exit::Exit;
 use brk_indexer::Indexer;
-use brk_vec::{AnyStorableVec, Compressed, Version};
+use brk_parser::bitcoin;
+use brk_vec::{Compressed, DynamicVec, StoredIndex, Version};
 
-use super::{Indexes, StorableVec, indexes};
+use super::{
+    ComputedVec, Indexes,
+    grouped::{ComputedVecsFromHeight, ComputedVecsFromTxindex, StorableVecGeneatorOptions},
+    indexes,
+};
 
 #[derive(Clone)]
 pub struct Vecs {
-    // pub height_to_fee: StorableVec<Txindex, Sats>,
+    pub indexes_to_tx_count: ComputedVecsFromHeight<StoredU64>,
-    // pub height_to_inputcount: StorableVec<Height, u32>,
+    pub indexes_to_fee: ComputedVecsFromTxindex<Sats>,
-    // pub height_to_maxfeerate: StorableVec<Height, Feerate>,
+    pub indexes_to_feerate: ComputedVecsFromTxindex<Feerate>,
-    // pub height_to_medianfeerate: StorableVec<Height, Feerate>,
+    pub indexes_to_input_value: ComputedVecsFromTxindex<Sats>,
-    // pub height_to_minfeerate: StorableVec<Height, Feerate>,
+    pub indexes_to_output_value: ComputedVecsFromTxindex<Sats>,
-    // pub height_to_outputcount: StorableVec<Height, u32>,
+    // pub txindex_to_is_v1: ComputedVec<Txindex, bool>,
-    // pub height_to_subsidy: StorableVec<Height, u32>,
+    pub indexes_to_tx_v1: ComputedVecsFromHeight<StoredU32>,
-    // pub height_to_totalfees: StorableVec<Height, Sats>,
+    // pub txindex_to_is_v2: ComputedVec<Txindex, bool>,
-    // pub height_to_txcount: StorableVec<Height, u32>,
+    pub indexes_to_tx_v2: ComputedVecsFromHeight<StoredU32>,
-    // pub txindex_to_fee: StorableVec<Txindex, Sats>,
+    // pub txindex_to_is_v3: ComputedVec<Txindex, bool>,
-    pub txindex_to_is_coinbase: StorableVec<Txindex, bool>,
+    pub indexes_to_tx_v3: ComputedVecsFromHeight<StoredU32>,
-    // pub txindex_to_feerate: StorableVec<Txindex, Feerate>,
+    pub indexes_to_tx_vsize: ComputedVecsFromTxindex<StoredUsize>,
-    pub txindex_to_inputs_count: StorableVec<Txindex, u32>,
+    pub indexes_to_tx_weight: ComputedVecsFromTxindex<Weight>,
-    // pub txindex_to_inputs_sum: StorableVec<Txindex, Sats>,
+    pub txindex_to_input_count: ComputedVecsFromTxindex<StoredU64>,
-    pub txindex_to_outputs_count: StorableVec<Txindex, u32>,
+    pub txindex_to_is_coinbase: ComputedVec<Txindex, bool>,
-    // pub txindex_to_outputs_sum: StorableVec<Txindex, Sats>,
+    pub txindex_to_output_count: ComputedVecsFromTxindex<StoredU64>,
-    // pub txinindex_to_value: StorableVec<Txinindex, Sats>,
+    pub txindex_to_vsize: ComputedVec<Txindex, StoredUsize>,
+    pub txindex_to_weight: ComputedVec<Txindex, Weight>,
+    /// Value == 0 when Coinbase
+    pub txinindex_to_value: ComputedVec<Txinindex, Sats>,
+    pub indexes_to_subsidy: ComputedVecsFromHeight<Sats>,
+    pub indexes_to_coinbase: ComputedVecsFromHeight<Sats>,
 }
 
 impl Vecs {
@@ -33,54 +46,184 @@ impl Vecs {
         fs::create_dir_all(path)?;
 
         Ok(Self {
-            // height_to_fee: StorableVec::forced_import(&path.join("height_to_fee"), Version::ONE)?,
+            indexes_to_tx_count: ComputedVecsFromHeight::forced_import(
-            // height_to_input_count: StorableVec::forced_import(
+                path,
-            //     &path.join("height_to_input_count"),
+                "tx_count",
-            //     Version::ONE,
+                true,
-            // )?,
+                Version::ZERO,
-            // height_to_maxfeerate: StorableVec::forced_import(&path.join("height_to_maxfeerate"), Version::ONE)?,
+                compressed,
-            // height_to_medianfeerate: StorableVec::forced_import(&path.join("height_to_medianfeerate"), Version::ONE)?,
+                StorableVecGeneatorOptions::default()
-            // height_to_minfeerate: StorableVec::forced_import(&path.join("height_to_minfeerate"), Version::ONE)?,
+                    .add_average()
-            // height_to_output_count: StorableVec::forced_import(
+                    .add_minmax()
-            //     &path.join("height_to_output_count"),
+                    .add_percentiles()
-            //     Version::ONE,
+                    .add_sum()
-            // )?,
+                    .add_total(),
-            // height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::ONE)?,
+            )?,
-            // height_to_totalfees: StorableVec::forced_import(&path.join("height_to_totalfees"), Version::ONE)?,
+            // height_to_subsidy: StorableVec::forced_import(&path.join("height_to_subsidy"), Version::ZERO)?,
-            // height_to_txcount: StorableVec::forced_import(&path.join("height_to_txcount"), Version::ONE)?,
+            txindex_to_is_coinbase: ComputedVec::forced_import(
-            // txindex_to_fee: StorableVec::forced_import(
-            //     &path.join("txindex_to_fee"),
-            //     Version::ONE,
-            // )?,
-            txindex_to_is_coinbase: StorableVec::forced_import(
                 &path.join("txindex_to_is_coinbase"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            // txindex_to_feerate: StorableVec::forced_import(&path.join("txindex_to_feerate"), Version::ONE)?,
+            txindex_to_input_count: ComputedVecsFromTxindex::forced_import(
-            txindex_to_inputs_count: StorableVec::forced_import(
+                path,
-                &path.join("txindex_to_inputs_count"),
+                "input_count",
-                Version::ONE,
+                true,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default()
+                    .add_average()
+                    .add_minmax()
+                    .add_percentiles()
+                    .add_sum()
+                    .add_total(),
+            )?,
+            txindex_to_output_count: ComputedVecsFromTxindex::forced_import(
+                path,
+                "output_count",
+                true,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default()
+                    .add_average()
+                    .add_minmax()
+                    .add_percentiles()
+                    .add_sum()
+                    .add_total(),
+            )?,
+            txinindex_to_value: ComputedVec::forced_import(
+                &path.join("txinindex_to_value"),
+                Version::ZERO,
                 compressed,
             )?,
-            // txindex_to_inputs_sum: StorableVec::forced_import(
+            indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
-            //     &path.join("txindex_to_inputs_sum"),
+                path,
-            //     Version::ONE,
+                "tx_v1",
-            // )?,
+                true,
-            txindex_to_outputs_count: StorableVec::forced_import(
+                Version::ZERO,
-                &path.join("txindex_to_outputs_count"),
+                compressed,
-                Version::ONE,
+                StorableVecGeneatorOptions::default().add_sum().add_total(),
+            )?,
+            indexes_to_tx_v2: ComputedVecsFromHeight::forced_import(
+                path,
+                "tx_v2",
+                true,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default().add_sum().add_total(),
+            )?,
+            indexes_to_tx_v3: ComputedVecsFromHeight::forced_import(
+                path,
+                "tx_v3",
+                true,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default().add_sum().add_total(),
+            )?,
+            indexes_to_input_value: ComputedVecsFromTxindex::forced_import(
+                path,
+                "input_value",
+                true,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default()
+                    .add_average()
+                    .add_sum()
+                    .add_total(),
+            )?,
+            indexes_to_output_value: ComputedVecsFromTxindex::forced_import(
+                path,
+                "output_value",
+                true,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default()
+                    .add_average()
+                    .add_sum()
+                    .add_total(),
+            )?,
+            indexes_to_fee: ComputedVecsFromTxindex::forced_import(
+                path,
+                "fee",
+                true,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default()
+                    .add_sum()
+                    .add_total()
+                    .add_percentiles()
+                    .add_minmax()
+                    .add_average(),
+            )?,
+            indexes_to_feerate: ComputedVecsFromTxindex::forced_import(
+                path,
+                "feerate",
+                true,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default()
+                    .add_percentiles()
+                    .add_minmax()
+                    .add_average(),
+            )?,
+            txindex_to_weight: ComputedVec::forced_import(
+                &path.join("txindex_to_weight"),
+                Version::ZERO,
                 compressed,
             )?,
-            // txindex_to_outputs_sum: StorableVec::forced_import(
+            txindex_to_vsize: ComputedVec::forced_import(
-            //     &path.join("txindex_to_outputs_sum"),
+                &path.join("txindex_to_vsize"),
-            //     Version::ONE,
+                Version::ZERO,
-            // )?,
+                compressed,
-            // txinindex_to_value: StorableVec::forced_import(
+            )?,
-            //     &path.join("txinindex_to_value"),
+            indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import(
-            //     Version::ONE,
+                path,
-            //     compressed,
+                "tx_vsize",
-            // )?,
+                false,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default()
+                    .add_percentiles()
+                    .add_minmax()
+                    .add_average(),
+            )?,
+            indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import(
+                path,
+                "tx_weight",
+                false,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default()
+                    .add_percentiles()
+                    .add_minmax()
+                    .add_average(),
+            )?,
+            indexes_to_subsidy: ComputedVecsFromHeight::forced_import(
+                path,
+                "subsidy",
+                true,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default()
+                    .add_percentiles()
+                    .add_sum()
+                    .add_total()
+                    .add_minmax()
+                    .add_average(),
+            )?,
+            indexes_to_coinbase: ComputedVecsFromHeight::forced_import(
+                path,
+                "coinbase",
+                true,
+                Version::ZERO,
+                compressed,
+                StorableVecGeneatorOptions::default()
+                    .add_sum()
+                    .add_total()
+                    .add_percentiles()
+                    .add_minmax()
+                    .add_average(),
+            )?,
         })
     }
 
@@ -91,22 +234,84 @@ impl Vecs {
         starting_indexes: &Indexes,
         exit: &Exit,
     ) -> color_eyre::Result<()> {
+        self.indexes_to_tx_count.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |v, indexer, indexes, starting_indexes, exit| {
+                v.compute_count_from_indexes(
+                    starting_indexes.height,
+                    indexer.mut_vecs().height_to_first_txindex.mut_vec(),
+                    indexes.height_to_last_txindex.mut_vec(),
+                    exit,
+                )
+            },
+        )?;
+
+        self.txindex_to_input_count.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |v, indexer, indexes, starting_indexes, exit| {
+                v.compute_count_from_indexes(
+                    starting_indexes.txindex,
+                    indexer.mut_vecs().txindex_to_first_txinindex.mut_vec(),
+                    indexes.txindex_to_last_txinindex.mut_vec(),
+                    exit,
+                )
+            },
+        )?;
+
+        self.txindex_to_output_count.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |v, indexer, indexes, starting_indexes, exit| {
+                v.compute_count_from_indexes(
+                    starting_indexes.txindex,
+                    indexer.mut_vecs().txindex_to_first_txoutindex.mut_vec(),
+                    indexes.txindex_to_last_txoutindex.mut_vec(),
+                    exit,
+                )
+            },
+        )?;
+
+        let mut compute_indexes_to_tx_vany =
+            |indexes_to_tx_vany: &mut ComputedVecsFromHeight<StoredU32>, txversion| {
+                indexes_to_tx_vany.compute_all(
+                    indexer,
+                    indexes,
+                    starting_indexes,
+                    exit,
+                    |vec, indexer, indexes, starting_indexes, exit| {
+                        let indexer_vecs = indexer.mut_vecs();
+                        vec.compute_filtered_count_from_indexes(
+                            starting_indexes.height,
+                            indexer_vecs.height_to_first_txindex.mut_vec(),
+                            indexes.height_to_last_txindex.mut_vec(),
+                            |txindex| {
+                                let v = indexer_vecs
+                                    .txindex_to_txversion
+                                    .cached_get(txindex)
+                                    .unwrap()
+                                    .unwrap()
+                                    .into_inner();
+                                v == txversion
+                            },
+                            exit,
+                        )
+                    },
+                )
+            };
+        compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v1, TxVersion::ONE)?;
+        compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v2, TxVersion::TWO)?;
+        compute_indexes_to_tx_vany(&mut self.indexes_to_tx_v3, TxVersion::THREE)?;
+
         let indexer_vecs = indexer.mut_vecs();
 
-        self.txindex_to_inputs_count.compute_count_from_indexes(
-            starting_indexes.txindex,
-            indexer_vecs.txindex_to_first_txinindex.mut_vec(),
-            indexes.txindex_to_last_txinindex.mut_vec(),
-            exit,
-        )?;
-
-        self.txindex_to_outputs_count.compute_count_from_indexes(
-            starting_indexes.txindex,
-            indexer_vecs.txindex_to_first_txoutindex.mut_vec(),
-            indexes.txindex_to_last_txoutindex.mut_vec(),
-            exit,
-        )?;
-
         self.txindex_to_is_coinbase.compute_is_first_ordered(
             starting_indexes.txindex,
             indexer_vecs.txindex_to_height.mut_vec(),
@@ -114,52 +319,268 @@ impl Vecs {
             exit,
         )?;
 
-        // self.txinindex_to_value.compute_transform(
+        self.txindex_to_weight.compute_transform(
-        //     starting_indexes.txinindex,
+            starting_indexes.txindex,
-        //     indexer_vecs.txinindex_to_txoutindex.mut_vec(),
+            indexer_vecs.txindex_to_base_size.mut_vec(),
-        //     |(txinindex, txoutindex, slf, other)| {
+            |(txindex, base_size, ..)| {
-        //         let value =
+                let total_size = indexer_vecs
-        //             if let Ok(Some(value)) = indexer_vecs.txoutindex_to_value.read(txoutindex) {
+                    .txindex_to_total_size
-        //                 *value
+                    .mut_vec()
-        //             } else {
+                    .cached_get(txindex)
-        //                 dbg!(txinindex, txoutindex, slf.len(), other.len());
+                    .unwrap()
-        //                 panic!()
+                    .unwrap()
-        //             };
+                    .into_inner();
-        //         (txinindex, value)
-        //     },
-        //     exit,
-        // )?;
 
-        // self.vecs.txindex_to_fee.compute_transform(
+                // This is the exact definition of a weight unit, as defined by BIP-141 (quote above).
-        //     &mut self.vecs.txindex_to_height,
+                let wu = base_size * 3 + total_size;
-        //     &mut indexer.vecs().height_to_first_txindex,
+                let weight = Weight::from(bitcoin::Weight::from_wu_usize(wu));
-        // )?;
 
-        // self.vecs.height_to_dateindex.compute(...)
+                (txindex, weight)
+            },
+            exit,
+        )?;
 
-        // ---
+        self.txindex_to_vsize.compute_transform(
-        // Date to X
+            starting_indexes.txindex,
-        // ---
+            self.txindex_to_weight.mut_vec(),
-        // ...
+            |(txindex, weight, ..)| {
+                let vbytes =
+                    StoredUsize::from(bitcoin::Weight::from(weight).to_vbytes_ceil() as usize);
+                (txindex, vbytes)
+            },
+            exit,
+        )?;
 
-        // ---
+        self.txinindex_to_value.compute_transform(
-        // Month to X
+            starting_indexes.txinindex,
-        // ---
+            indexer_vecs.txinindex_to_txoutindex.mut_vec(),
-        // ...
+            |(txinindex, txoutindex, slf, other)| {
+                let value = if txoutindex == Txoutindex::COINBASE {
+                    Sats::ZERO
+                } else if let Ok(Some(value)) = indexer_vecs
+                    .txoutindex_to_value
+                    .mut_vec()
+                    .cached_get(txoutindex)
+                {
+                    *value
+                } else {
+                    dbg!(txinindex, txoutindex, slf.len(), other.len());
+                    panic!()
+                };
+                (txinindex, value)
+            },
+            exit,
+        )?;
 
-        // ---
+        self.indexes_to_output_value.compute_all(
-        // Year to X
+            indexer,
-        // ---
+            indexes,
-        // ...
+            starting_indexes,
+            exit,
+            |vec, indexer, indexes, starting_indexes, exit| {
+                let indexer_vecs = indexer.mut_vecs();
+                vec.compute_sum_from_indexes(
+                    starting_indexes.txindex,
+                    indexer_vecs.txindex_to_first_txoutindex.mut_vec(),
+                    indexes.txindex_to_last_txoutindex.mut_vec(),
+                    indexer_vecs.txoutindex_to_value.mut_vec(),
+                    exit,
+                )
+            },
+        )?;
+
+        self.indexes_to_input_value.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |vec, indexer, indexes, starting_indexes, exit| {
+                let indexer_vecs = indexer.mut_vecs();
+                vec.compute_sum_from_indexes(
+                    starting_indexes.txindex,
+                    indexer_vecs.txindex_to_first_txinindex.mut_vec(),
+                    indexes.txindex_to_last_txinindex.mut_vec(),
+                    self.txinindex_to_value.mut_vec(),
+                    exit,
+                )
+            },
+        )?;
+
+        self.indexes_to_fee.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |vec, _, _, starting_indexes, exit| {
+                let txindex_to_output_value = self
+                    .indexes_to_output_value
+                    .txindex
+                    .as_mut()
+                    .unwrap()
+                    .mut_vec();
+                vec.compute_transform(
+                    starting_indexes.txindex,
+                    self.indexes_to_input_value
+                        .txindex
+                        .as_mut()
+                        .unwrap()
+                        .mut_vec(),
+                    |(txindex, input_value, ..)| {
+                        if input_value.is_zero() {
+                            (txindex, input_value)
+                        } else {
+                            let output_value = txindex_to_output_value
+                                .cached_get(txindex)
+                                .unwrap()
+                                .unwrap()
+                                .into_inner();
+                            (txindex, input_value.checked_sub(output_value).unwrap())
+                        }
+                    },
+                    exit,
+                )
+            },
+        )?;
+
+        self.indexes_to_feerate.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |vec, _, _, starting_indexes, exit| {
+                vec.compute_transform(
+                    starting_indexes.txindex,
+                    self.indexes_to_fee.txindex.as_mut().unwrap().mut_vec(),
+                    |(txindex, fee, ..)| {
+                        let vsize = self
+                            .txindex_to_vsize
+                            .mut_vec()
+                            .cached_get(txindex)
+                            .unwrap()
+                            .unwrap()
+                            .into_inner();
+
+                        (txindex, Feerate::from((fee, vsize)))
+                    },
+                    exit,
+                )
+            },
+        )?;
+
+        self.indexes_to_tx_weight.compute_rest(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            Some(self.txindex_to_weight.mut_vec()),
+        )?;
+
+        self.indexes_to_tx_vsize.compute_rest(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            Some(self.txindex_to_vsize.mut_vec()),
+        )?;
+
+        self.indexes_to_subsidy.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |vec, indexer, indexes, starting_indexes, exit| {
+                let indexer_vecs = indexer.mut_vecs();
+                vec.compute_transform(
+                    starting_indexes.height,
+                    indexer_vecs.height_to_first_txindex.mut_vec(),
+                    |(height, txindex, ..)| {
+                        let first_txoutindex = indexer_vecs
+                            .txindex_to_first_txoutindex
+                            .cached_get(txindex)
+                            .unwrap()
+                            .unwrap()
+                            .into_inner()
+                            .to_usize()
+                            .unwrap();
+                        let last_txoutindex = indexes
+                            .txindex_to_last_txoutindex
+                            .mut_vec()
+                            .cached_get(txindex)
+                            .unwrap()
+                            .unwrap()
+                            .into_inner()
+                            .to_usize()
+                            .unwrap();
+                        let mut sats = Sats::ZERO;
+                        (first_txoutindex..=last_txoutindex).for_each(|txoutindex| {
+                            sats += indexer_vecs
+                                .txoutindex_to_value
+                                .cached_get_(txoutindex)
+                                .unwrap()
+                                .unwrap()
+                                .into_inner();
+                        });
+                        (height, sats)
+                    },
+                    exit,
+                )
+            },
+        )?;
+
+        self.indexes_to_coinbase.compute_all(
+            indexer,
+            indexes,
+            starting_indexes,
+            exit,
+            |vec, _, _, starting_indexes, exit| {
+                vec.compute_transform(
+                    starting_indexes.height,
+                    self.indexes_to_subsidy.height.as_mut().unwrap().mut_vec(),
+                    |(height, subsidy, ..)| {
+                        let fees = self
+                            .indexes_to_fee
+                            .height
+                            .sum
+                            .as_mut()
+                            .unwrap()
+                            .mut_vec()
+                            .cached_get(height)
+                            .unwrap()
+                            .unwrap()
+                            .into_inner();
+                        (height, subsidy.checked_sub(fees).unwrap())
+                    },
+                    exit,
+                )
+            },
+        )?;
 
         Ok(())
     }
 
-    pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
+    pub fn as_any_vecs(&self) -> Vec<&dyn brk_vec::AnyStoredVec> {
-        vec![
+        [
-            self.txindex_to_is_coinbase.any_vec(),
+            vec![
-            self.txindex_to_inputs_count.any_vec(),
+                self.txindex_to_is_coinbase.any_vec(),
-            self.txindex_to_outputs_count.any_vec(),
+                self.txinindex_to_value.any_vec(),
+                self.txindex_to_weight.any_vec(),
+                self.txindex_to_vsize.any_vec(),
+            ],
+            self.indexes_to_tx_count.any_vecs(),
+            self.indexes_to_coinbase.any_vecs(),
+            self.indexes_to_fee.any_vecs(),
+            self.indexes_to_feerate.any_vecs(),
+            self.indexes_to_input_value.any_vecs(),
+            self.indexes_to_output_value.any_vecs(),
+            self.indexes_to_subsidy.any_vecs(),
+            self.indexes_to_tx_v1.any_vecs(),
+            self.indexes_to_tx_v2.any_vecs(),
+            self.indexes_to_tx_v3.any_vecs(),
+            self.indexes_to_tx_vsize.any_vecs(),
+            self.indexes_to_tx_weight.any_vecs(),
+            self.txindex_to_input_count.any_vecs(),
+            self.txindex_to_output_count.any_vecs(),
         ]
+        .concat()
     }
 }

crates/brk_core/README.md

@@ -20,7 +20,7 @@
   <a href="https://deps.rs/crate/brk_core">
     <img src="https://deps.rs/crate/brk_core/latest/status.svg" alt="Dependency status">
   </a>
-  <a href="https://discord.gg/Cvrwpv3zEG">
+  <a href="https://discord.gg/HaR3wpH3nr">
     <img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
   </a>
   <a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">

crates/brk_core/src/structs/feerate.rs

@@ -1,5 +1,66 @@
+use std::ops::{Add, Div};
+
 use serde::Serialize;
 use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-#[derive(Debug, Clone, Copy, Serialize, FromBytes, Immutable, IntoBytes, KnownLayout)]
+use super::{Sats, StoredUsize};
+
+#[derive(
+    Debug,
+    Clone,
+    Copy,
+    Serialize,
+    PartialEq,
+    PartialOrd,
+    FromBytes,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+)]
 pub struct Feerate(f32);
+
+impl From<(Sats, StoredUsize)> for Feerate {
+    fn from((sats, vsize): (Sats, StoredUsize)) -> Self {
+        Self((f64::from(sats) / f64::from(vsize)) as f32)
+    }
+}
+
+impl From<f64> for Feerate {
+    fn from(value: f64) -> Self {
+        Self(value as f32)
+    }
+}
+impl From<Feerate> for f64 {
+    fn from(value: Feerate) -> Self {
+        value.0 as f64
+    }
+}
+
+impl Add for Feerate {
+    type Output = Self;
+    fn add(self, rhs: Self) -> Self::Output {
+        Self(self.0 + rhs.0)
+    }
+}
+
+impl Div<usize> for Feerate {
+    type Output = Self;
+    fn div(self, rhs: usize) -> Self::Output {
+        Self((self.0 as f64 / rhs as f64) as f32)
+    }
+}
+
+impl From<usize> for Feerate {
+    fn from(value: usize) -> Self {
+        Self(value as f32)
+    }
+}
+
+impl Eq for Feerate {}
+
+#[allow(clippy::derive_ord_xor_partial_ord)]
+impl Ord for Feerate {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.0.partial_cmp(&other.0).unwrap()
+    }
+}

crates/brk_core/src/structs/mod.rs

@@ -20,6 +20,10 @@ mod monthindex;
 mod ohlc;
 mod quarterindex;
 mod sats;
+mod stored_u32;
+mod stored_u64;
+mod stored_u8;
+mod stored_usize;
 mod timestamp;
 mod txid;
 mod txindex;
@@ -55,6 +59,10 @@ pub use monthindex::*;
 pub use ohlc::*;
 pub use quarterindex::*;
 pub use sats::*;
+pub use stored_u8::*;
+pub use stored_u32::*;
+pub use stored_u64::*;
+pub use stored_usize::*;
 pub use timestamp::*;
 pub use txid::*;
 pub use txindex::*;

crates/brk_core/src/structs/stored_u32.rs

@@ -0,0 +1,79 @@
+use std::ops::{Add, Div};
+
+use derive_deref::Deref;
+use serde::Serialize;
+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::CheckedSub;
+
+#[derive(
+    Debug,
+    Deref,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    FromBytes,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    Serialize,
+)]
+pub struct StoredU32(u32);
+
+impl StoredU32 {
+    pub const ZERO: Self = Self(0);
+
+    pub fn new(counter: u32) -> Self {
+        Self(counter)
+    }
+}
+
+impl From<u32> for StoredU32 {
+    fn from(value: u32) -> Self {
+        Self(value)
+    }
+}
+
+impl From<usize> for StoredU32 {
+    fn from(value: usize) -> Self {
+        Self(value as u32)
+    }
+}
+
+impl CheckedSub<StoredU32> for StoredU32 {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        self.0.checked_sub(rhs.0).map(Self)
+    }
+}
+
+impl Div<usize> for StoredU32 {
+    type Output = Self;
+    fn div(self, rhs: usize) -> Self::Output {
+        Self(self.0 / rhs as u32)
+    }
+}
+
+impl Add for StoredU32 {
+    type Output = Self;
+    fn add(self, rhs: Self) -> Self::Output {
+        Self(self.0 + rhs.0)
+    }
+}
+
+impl From<f64> for StoredU32 {
+    fn from(value: f64) -> Self {
+        if value < 0.0 || value > u32::MAX as f64 {
+            panic!()
+        }
+        Self(value as u32)
+    }
+}
+
+impl From<StoredU32> for f64 {
+    fn from(value: StoredU32) -> Self {
+        value.0 as f64
+    }
+}

crates/brk_core/src/structs/stored_u64.rs

@@ -0,0 +1,99 @@
+use std::ops::{Add, Div};
+
+use derive_deref::Deref;
+use serde::Serialize;
+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::CheckedSub;
+
+use super::{Txindex, Txinindex, Txoutindex};
+
+#[derive(
+    Debug,
+    Deref,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    FromBytes,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    Serialize,
+)]
+pub struct StoredU64(u64);
+
+impl StoredU64 {
+    pub const ZERO: Self = Self(0);
+
+    pub fn new(counter: u64) -> Self {
+        Self(counter)
+    }
+}
+
+impl From<u64> for StoredU64 {
+    fn from(value: u64) -> Self {
+        Self(value)
+    }
+}
+
+impl From<usize> for StoredU64 {
+    fn from(value: usize) -> Self {
+        Self(value as u64)
+    }
+}
+
+impl CheckedSub<StoredU64> for StoredU64 {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        self.0.checked_sub(rhs.0).map(Self)
+    }
+}
+
+impl Div<usize> for StoredU64 {
+    type Output = Self;
+    fn div(self, rhs: usize) -> Self::Output {
+        Self(self.0 / rhs as u64)
+    }
+}
+
+impl Add for StoredU64 {
+    type Output = Self;
+    fn add(self, rhs: Self) -> Self::Output {
+        Self(self.0 + rhs.0)
+    }
+}
+
+impl From<f64> for StoredU64 {
+    fn from(value: f64) -> Self {
+        if value < 0.0 || value > u32::MAX as f64 {
+            panic!()
+        }
+        Self(value as u64)
+    }
+}
+
+impl From<StoredU64> for f64 {
+    fn from(value: StoredU64) -> Self {
+        value.0 as f64
+    }
+}
+
+impl From<Txindex> for StoredU64 {
+    fn from(value: Txindex) -> Self {
+        Self(*value as u64)
+    }
+}
+
+impl From<Txinindex> for StoredU64 {
+    fn from(value: Txinindex) -> Self {
+        Self(*value)
+    }
+}
+
+impl From<Txoutindex> for StoredU64 {
+    fn from(value: Txoutindex) -> Self {
+        Self(*value)
+    }
+}

crates/brk_core/src/structs/stored_u8.rs

@@ -0,0 +1,79 @@
+use std::ops::{Add, Div};
+
+use derive_deref::Deref;
+use serde::Serialize;
+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::CheckedSub;
+
+#[derive(
+    Debug,
+    Deref,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    FromBytes,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    Serialize,
+)]
+pub struct StoredU8(u8);
+
+impl StoredU8 {
+    pub const ZERO: Self = Self(0);
+
+    pub fn new(counter: u8) -> Self {
+        Self(counter)
+    }
+}
+
+impl From<u8> for StoredU8 {
+    fn from(value: u8) -> Self {
+        Self(value)
+    }
+}
+
+impl From<usize> for StoredU8 {
+    fn from(value: usize) -> Self {
+        Self(value as u8)
+    }
+}
+
+impl CheckedSub<StoredU8> for StoredU8 {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        self.0.checked_sub(rhs.0).map(Self)
+    }
+}
+
+impl Div<usize> for StoredU8 {
+    type Output = Self;
+    fn div(self, rhs: usize) -> Self::Output {
+        Self(self.0 / rhs as u8)
+    }
+}
+
+impl Add for StoredU8 {
+    type Output = Self;
+    fn add(self, rhs: Self) -> Self::Output {
+        Self(self.0 + rhs.0)
+    }
+}
+
+impl From<f64> for StoredU8 {
+    fn from(value: f64) -> Self {
+        if value < 0.0 || value > u32::MAX as f64 {
+            panic!()
+        }
+        Self(value as u8)
+    }
+}
+
+impl From<StoredU8> for f64 {
+    fn from(value: StoredU8) -> Self {
+        value.0 as f64
+    }
+}

crates/brk_core/src/structs/stored_usize.rs

@@ -0,0 +1,73 @@
+use std::ops::{Add, Div};
+
+use derive_deref::Deref;
+use serde::Serialize;
+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::CheckedSub;
+
+#[derive(
+    Debug,
+    Deref,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    FromBytes,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    Serialize,
+)]
+pub struct StoredUsize(usize);
+
+impl StoredUsize {
+    pub const ZERO: Self = Self(0);
+
+    pub fn new(counter: usize) -> Self {
+        Self(counter)
+    }
+}
+
+impl From<usize> for StoredUsize {
+    fn from(value: usize) -> Self {
+        Self(value)
+    }
+}
+
+impl CheckedSub<StoredUsize> for StoredUsize {
+    fn checked_sub(self, rhs: Self) -> Option<Self> {
+        self.0.checked_sub(rhs.0).map(Self)
+    }
+}
+
+impl Div<usize> for StoredUsize {
+    type Output = Self;
+    fn div(self, rhs: usize) -> Self::Output {
+        Self(self.0 / rhs)
+    }
+}
+
+impl Add for StoredUsize {
+    type Output = Self;
+    fn add(self, rhs: Self) -> Self::Output {
+        Self(self.0 + rhs.0)
+    }
+}
+
+impl From<f64> for StoredUsize {
+    fn from(value: f64) -> Self {
+        if value < 0.0 || value > u32::MAX as f64 {
+            panic!()
+        }
+        Self(value as usize)
+    }
+}
+
+impl From<StoredUsize> for f64 {
+    fn from(value: StoredUsize) -> Self {
+        value.0 as f64
+    }
+}

crates/brk_core/src/structs/txindex.rs

@@ -7,6 +7,8 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
 use crate::{CheckedSub, Error};
 
+use super::StoredU32;
+
 #[derive(
     Debug,
     PartialEq,
@@ -97,3 +99,9 @@ impl From<Txindex> for ByteView {
         Self::new(value.as_bytes())
     }
 }
+
+impl From<Txindex> for StoredU32 {
+    fn from(value: Txindex) -> Self {
+        Self::from(value.0)
+    }
+}

crates/brk_core/src/structs/txversion.rs

@@ -2,9 +2,31 @@ use derive_deref::Deref;
 use serde::Serialize;
 use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
+use super::StoredU8;
+
+#[derive(
+    Debug,
+    Deref,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    FromBytes,
+    Serialize,
+)]
 pub struct TxVersion(i32);
 
+impl TxVersion {
+    pub const ONE: Self = Self(1);
+    pub const TWO: Self = Self(2);
+    pub const THREE: Self = Self(3);
+}
+
 impl From<bitcoin::transaction::Version> for TxVersion {
     fn from(value: bitcoin::transaction::Version) -> Self {
         Self(value.0)
@@ -16,3 +38,9 @@ impl From<TxVersion> for bitcoin::transaction::Version {
         Self(value.0)
     }
 }
+
+impl From<TxVersion> for StoredU8 {
+    fn from(value: TxVersion) -> Self {
+        Self::from(value.0 as u8)
+    }
+}

crates/brk_core/src/structs/weight.rs

@@ -1,8 +1,24 @@
+use std::ops::{Add, Div};
+
 use derive_deref::Deref;
 use serde::Serialize;
 use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
-#[derive(Debug, Deref, Clone, Copy, Immutable, IntoBytes, KnownLayout, FromBytes, Serialize)]
+#[derive(
+    Debug,
+    Deref,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Immutable,
+    IntoBytes,
+    KnownLayout,
+    FromBytes,
+    Serialize,
+)]
 pub struct Weight(u64);
 
 impl From<bitcoin::Weight> for Weight {
@@ -16,3 +32,35 @@ impl From<Weight> for bitcoin::Weight {
         Self::from_wu(*value)
     }
 }
+
+impl From<usize> for Weight {
+    fn from(value: usize) -> Self {
+        Self(value as u64)
+    }
+}
+
+impl From<f64> for Weight {
+    fn from(value: f64) -> Self {
+        Self(value as u64)
+    }
+}
+
+impl From<Weight> for f64 {
+    fn from(value: Weight) -> Self {
+        value.0 as f64
+    }
+}
+
+impl Add for Weight {
+    type Output = Self;
+    fn add(self, rhs: Self) -> Self::Output {
+        Self(self.0 + rhs.0)
+    }
+}
+
+impl Div<usize> for Weight {
+    type Output = Self;
+    fn div(self, rhs: usize) -> Self::Output {
+        Self::from(self.0 as usize / rhs)
+    }
+}

crates/brk_exit/Cargo.toml

@@ -8,5 +8,5 @@ repository.workspace = true
 
 [dependencies]
 brk_logger = { workspace = true }
-ctrlc = { version = "3.4.5", features = ["termination"] }
+ctrlc = { version = "3.4.6", features = ["termination"] }
 log = { workspace = true }

crates/brk_exit/README.md

@@ -20,7 +20,7 @@
   <a href="https://deps.rs/crate/brk_exit">
     <img src="https://deps.rs/crate/brk_exit/latest/status.svg" alt="Dependency status">
   </a>
-  <a href="https://discord.gg/Cvrwpv3zEG">
+  <a href="https://discord.gg/HaR3wpH3nr">
     <img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
   </a>
   <a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">

crates/brk_fetcher/README.md

@@ -20,7 +20,7 @@
   <a href="https://deps.rs/crate/brk_fetcher">
     <img src="https://deps.rs/crate/brk_fetcher/latest/status.svg" alt="Dependency status">
   </a>
-  <a href="https://discord.gg/Cvrwpv3zEG">
+  <a href="https://discord.gg/HaR3wpH3nr">
     <img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
   </a>
   <a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">

crates/brk_indexer/README.md

@@ -20,7 +20,7 @@
   <a href="https://deps.rs/crate/brk_indexer">
     <img src="https://deps.rs/crate/brk_indexer/latest/status.svg" alt="Dependency status">
   </a>
-  <a href="https://discord.gg/Cvrwpv3zEG">
+  <a href="https://discord.gg/HaR3wpH3nr">
     <img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
   </a>
   <a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
@@ -58,9 +58,14 @@ Stores: `src/storage/stores/mod.rs`
 
 ## Benchmark
 
-Indexing `0..885_835` took `11 hours 6 min 50 s` on a Macbook Pro M3 Pro with 36 GB of RAM
+### Result 1 - 2025-04-12
 
-`footprint` report:
+- version: `v0.0.21`
-- Peak memory: `5115 MB`
+- machine: `Macbook Pro M3 Pro (36GB RAM)`
-- Memory while waiting for a new block: `890 MB`
+- mode: `raw`
-- Reclaimable memory: `6478 MB`
+- from: `0`
+- to: `892_098`
+- time: `7 hours 10 min 22s`
+- peak memory: `6.1GB`
+- disk usage: `270 GB`
+- overhead: `36%` (`270 GB / 741 GB`)

crates/brk_indexer/examples/main.rs

@@ -1,4 +1,4 @@
-use std::path::Path;
+use std::{path::Path, time::Instant};
 
 use brk_core::default_bitcoin_path;
 use brk_exit::Exit;
@@ -8,6 +8,8 @@ use brk_parser::{Parser, rpc};
 fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
 
+    let i = Instant::now();
+
     brk_logger::init(Some(Path::new(".log")));
 
     let bitcoin_dir = default_bitcoin_path();
@@ -29,5 +31,7 @@ fn main() -> color_eyre::Result<()> {
 
     indexer.index(&parser, rpc, &exit)?;
 
+    dbg!(i.elapsed());
+
     Ok(())
 }

crates/brk_indexer/src/indexes.rs

@@ -5,9 +5,10 @@ use brk_core::{
     Pushonlyindex, Txindex, Txinindex, Txoutindex, Unknownindex,
 };
 use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
+use brk_vec::{Result, StoredIndex, StoredType, Value};
 use color_eyre::eyre::ContextCompat;
 
-use crate::{Stores, Vecs};
+use crate::{IndexedVec, Stores, Vecs};
 
 #[derive(Debug, Default, Clone)]
 pub struct Indexes {
@@ -65,13 +66,7 @@ impl Indexes {
             .push_if_needed(height, self.p2wpkhindex)?;
         vecs.height_to_first_p2wshindex
             .push_if_needed(height, self.p2wshindex)?;
-        Ok(())
-    }
 
-    pub fn push_future_if_needed(&mut self, vecs: &mut Vecs) -> brk_vec::Result<()> {
-        self.height.increment();
-        self.push_if_needed(vecs)?;
-        self.height.decrement();
         Ok(())
     }
 }
@@ -111,32 +106,121 @@ impl TryFrom<(&mut Vecs, &Stores, &Client)> for Indexes {
             .unwrap_or(starting_height);
 
         Ok(Self {
-            addressindex: *vecs.height_to_first_addressindex.get(height)?.context("")?,
+            addressindex: *starting_index(
-            emptyindex: *vecs.height_to_first_emptyindex.get(height)?.context("")?,
+                &vecs.height_to_first_addressindex,
+                &vecs.addressindex_to_height,
+                height,
+            )?
+            .context("")?,
+            emptyindex: *starting_index(
+                &vecs.height_to_first_emptyindex,
+                &vecs.emptyindex_to_height,
+                height,
+            )?
+            .context("")?,
             height,
-            multisigindex: *vecs
+            multisigindex: *starting_index(
-                .height_to_first_multisigindex
+                &vecs.height_to_first_multisigindex,
-                .get(height)?
+                &vecs.multisigindex_to_height,
-                .context("")?,
+                height,
-            opreturnindex: *vecs
+            )?
-                .height_to_first_opreturnindex
+            .context("")?,
-                .get(height)?
+            opreturnindex: *starting_index(
-                .context("")?,
+                &vecs.height_to_first_opreturnindex,
-            p2pk33index: *vecs.height_to_first_p2pk33index.get(height)?.context("")?,
+                &vecs.opreturnindex_to_height,
-            p2pk65index: *vecs.height_to_first_p2pk65index.get(height)?.context("")?,
+                height,
-            p2pkhindex: *vecs.height_to_first_p2pkhindex.get(height)?.context("")?,
+            )?
-            p2shindex: *vecs.height_to_first_p2shindex.get(height)?.context("")?,
+            .context("")?,
-            p2trindex: *vecs.height_to_first_p2trindex.get(height)?.context("")?,
+            p2pk33index: *starting_index(
-            p2wpkhindex: *vecs.height_to_first_p2wpkhindex.get(height)?.context("")?,
+                &vecs.height_to_first_p2pk33index,
-            p2wshindex: *vecs.height_to_first_p2wshindex.get(height)?.context("")?,
+                &vecs.p2pk33index_to_height,
-            pushonlyindex: *vecs
+                height,
-                .height_to_first_pushonlyindex
+            )?
-                .get(height)?
+            .context("")?,
-                .context("")?,
+            p2pk65index: *starting_index(
-            txindex: *vecs.height_to_first_txindex.get(height)?.context("")?,
+                &vecs.height_to_first_p2pk65index,
-            txinindex: *vecs.height_to_first_txinindex.get(height)?.context("")?,
+                &vecs.p2pk65index_to_height,
-            txoutindex: *vecs.height_to_first_txoutindex.get(height)?.context("")?,
+                height,
-            unknownindex: *vecs.height_to_first_unknownindex.get(height)?.context("")?,
+            )?
+            .context("")?,
+            p2pkhindex: *starting_index(
+                &vecs.height_to_first_p2pkhindex,
+                &vecs.p2pkhindex_to_height,
+                height,
+            )?
+            .context("")?,
+            p2shindex: *starting_index(
+                &vecs.height_to_first_p2shindex,
+                &vecs.p2shindex_to_height,
+                height,
+            )?
+            .context("")?,
+            p2trindex: *starting_index(
+                &vecs.height_to_first_p2trindex,
+                &vecs.p2trindex_to_height,
+                height,
+            )?
+            .context("")?,
+            p2wpkhindex: *starting_index(
+                &vecs.height_to_first_p2wpkhindex,
+                &vecs.p2wpkhindex_to_height,
+                height,
+            )?
+            .context("")?,
+            p2wshindex: *starting_index(
+                &vecs.height_to_first_p2wshindex,
+                &vecs.p2wshindex_to_height,
+                height,
+            )?
+            .context("")?,
+            pushonlyindex: *starting_index(
+                &vecs.height_to_first_pushonlyindex,
+                &vecs.pushonlyindex_to_height,
+                height,
+            )?
+            .context("")?,
+            txindex: *starting_index(
+                &vecs.height_to_first_txindex,
+                &vecs.txindex_to_height,
+                height,
+            )?
+            .context("")?,
+            txinindex: *starting_index(
+                &vecs.height_to_first_txinindex,
+                &vecs.txinindex_to_height,
+                height,
+            )?
+            .context("")?,
+            txoutindex: *starting_index(
+                &vecs.height_to_first_txoutindex,
+                &vecs.txoutindex_to_height,
+                height,
+            )?
+            .context("")?,
+            unknownindex: *starting_index(
+                &vecs.height_to_first_unknownindex,
+                &vecs.unknownindex_to_height,
+                height,
+            )?
+            .context("")?,
         })
     }
 }
+
+pub fn starting_index<'a, I>(
+    height_to_index: &'a IndexedVec<Height, I>,
+    index_to_height: &'a IndexedVec<I, Height>,
+    starting_height: Height,
+) -> Result<Option<Value<'a, I>>>
+where
+    I: StoredType + StoredIndex + From<usize>,
+{
+    if height_to_index
+        .height()
+        .is_ok_and(|h| h + 1_u32 == starting_height)
+    {
+        Ok(Some(Value::Owned(I::from(index_to_height.len()))))
+    } else {
+        height_to_index.get(starting_height)
+    }
+}

crates/brk_indexer/src/lib.rs

@@ -31,7 +31,7 @@ pub use stores::*;
 pub use vecs::*;
 
 const SNAPSHOT_BLOCK_RANGE: usize = 1000;
-const COLLISIONS_CHECKED_UP_TO: u32 = 888_000;
+const COLLISIONS_CHECKED_UP_TO: u32 = 890_000;
 
 #[derive(Clone)]
 pub struct Indexer {
@@ -81,7 +81,7 @@ impl Indexer {
             self.stores.as_ref().unwrap(),
             rpc,
         ))
-        .unwrap_or_else(|_| {
+        .unwrap_or_else(|_report| {
             let indexes = Indexes::default();
             indexes.push_if_needed(self.vecs.as_mut().unwrap()).unwrap();
             indexes
@@ -101,6 +101,7 @@ impl Indexer {
         let vecs = self.vecs.as_mut().unwrap();
         let stores = self.stores.as_mut().unwrap();
 
+        // Cloned because we want to return starting indexes for the computer
         let mut idxs = starting_indexes.clone();
 
         let start = Some(idxs.height);
@@ -153,6 +154,8 @@ impl Indexer {
                     return Err(eyre!("Collision, expect prefix to need be set yet"));
                 }
 
+                idxs.push_if_needed(vecs)?;
+
                 stores
                     .blockhash_prefix_to_height
                     .insert_if_needed(blockhash_prefix, height, height);
@@ -162,7 +165,7 @@ impl Indexer {
                     .push_if_needed(height, block.header.difficulty_float())?;
                 vecs.height_to_timestamp
                     .push_if_needed(height, Timestamp::from(block.header.time))?;
-                vecs.height_to_size.push_if_needed(height, block.total_size())?;
+                vecs.height_to_total_size.push_if_needed(height, block.total_size().into())?;
                 vecs.height_to_weight.push_if_needed(height, block.weight().into())?;
 
                 let inputs = block
@@ -462,6 +465,9 @@ impl Indexer {
 
                         vecs.txoutindex_to_value.push_if_needed(txoutindex, sats)?;
 
+                        vecs.txoutindex_to_height
+                            .push_if_needed(txoutindex, height)?;
+
                         let mut addressindex = idxs.addressindex;
 
                         let mut addresshash = None;
@@ -481,18 +487,55 @@ impl Indexer {
                             idxs.addressindex.increment();
 
                             let addresstypeindex = match addresstype {
-                                Addresstype::Empty => idxs.emptyindex.copy_then_increment(),
+                                Addresstype::Empty => {
-                                Addresstype::Multisig => idxs.multisigindex.copy_then_increment(),
+                                    vecs.emptyindex_to_height
-                                Addresstype::OpReturn => idxs.opreturnindex.copy_then_increment(),
+                                        .push_if_needed(idxs.emptyindex, height)?;
-                                Addresstype::PushOnly => idxs.pushonlyindex.copy_then_increment(),
+                                    idxs.emptyindex.copy_then_increment()
-                                Addresstype::Unknown => idxs.unknownindex.copy_then_increment(),
+                                },
-                                Addresstype::P2PK65 => idxs.p2pk65index.copy_then_increment(),
+                                Addresstype::Multisig => {
-                                Addresstype::P2PK33 => idxs.p2pk33index.copy_then_increment(),
+                                    vecs.multisigindex_to_height.push_if_needed(idxs.multisigindex, height)?;
-                                Addresstype::P2PKH => idxs.p2pkhindex.copy_then_increment(),
+                                    idxs.multisigindex.copy_then_increment()
-                                Addresstype::P2SH => idxs.p2shindex.copy_then_increment(),
+                                },
-                                Addresstype::P2WPKH => idxs.p2wpkhindex.copy_then_increment(),
+                                Addresstype::OpReturn => {
-                                Addresstype::P2WSH => idxs.p2wshindex.copy_then_increment(),
+                                    vecs.opreturnindex_to_height.push_if_needed(idxs.opreturnindex, height)?;
-                                Addresstype::P2TR => idxs.p2trindex.copy_then_increment(),
+                                    idxs.opreturnindex.copy_then_increment()
+                                },
+                                Addresstype::PushOnly => {
+                                    vecs.pushonlyindex_to_height.push_if_needed(idxs.pushonlyindex, height)?;
+                                    idxs.pushonlyindex.copy_then_increment()
+                                },
+                                Addresstype::Unknown => {
+                                    vecs.unknownindex_to_height.push_if_needed(idxs.unknownindex, height)?;
+                                    idxs.unknownindex.copy_then_increment()
+                                },
+                                Addresstype::P2PK65 => {
+                                    vecs.p2pk65index_to_height.push_if_needed(idxs.p2pk65index, height)?;
+                                    idxs.p2pk65index.copy_then_increment()
+                                },
+                                Addresstype::P2PK33 => {
+                                    vecs.p2pk33index_to_height.push_if_needed(idxs.p2pk33index, height)?;
+                                    idxs.p2pk33index.copy_then_increment()
+                                },
+                                Addresstype::P2PKH => {
+                                    vecs.p2pkhindex_to_height.push_if_needed(idxs.p2pkhindex, height)?;
+                                    idxs.p2pkhindex.copy_then_increment()
+                                },
+                                Addresstype::P2SH => {
+                                    vecs.p2shindex_to_height.push_if_needed(idxs.p2shindex, height)?;
+                                    idxs.p2shindex.copy_then_increment()
+                                },
+                                Addresstype::P2WPKH => {
+                                    vecs.p2wpkhindex_to_height.push_if_needed(idxs.p2wpkhindex, height)?;
+                                    idxs.p2wpkhindex.copy_then_increment()
+                                },
+                                Addresstype::P2WSH => {
+                                    vecs.p2wshindex_to_height.push_if_needed(idxs.p2wshindex, height)?;
+                                    idxs.p2wshindex.copy_then_increment()
+                                },
+                                Addresstype::P2TR => {
+                                    vecs.p2trindex_to_height.push_if_needed(idxs.p2trindex, height)?;
+                                    idxs.p2trindex.copy_then_increment()
+                                },
                             };
 
                             vecs.addressindex_to_addresstype
@@ -580,6 +623,10 @@ impl Indexer {
 
                         vecs.txinindex_to_txoutindex.push_if_needed(txinindex, txoutindex)?;
 
+                        vecs.txinindex_to_height
+                            .push_if_needed(txinindex, height)?;
+
+
                         Ok(())
                     })?;
 
@@ -668,8 +715,6 @@ impl Indexer {
                 idxs.txinindex += Txinindex::from(inputs_len);
                 idxs.txoutindex += Txoutindex::from(outputs_len);
 
-                idxs.push_future_if_needed(vecs)?;
-
                 export_if_needed(stores, vecs, height, false, exit)?;
 
                 Ok(())

crates/brk_indexer/src/stores/base.rs

@@ -98,10 +98,10 @@ where
 
         if !self.puts.is_empty() {
             unreachable!("Shouldn't reach this");
-            // self.puts.remove(&key);
         }
-        // dbg!(&key);
+
-        if !self.dels.insert(key) {
+        if !self.dels.insert(key.clone()) {
+            dbg!(key, &self.meta.path());
             unreachable!();
         }
     }

crates/brk_indexer/src/stores/meta.rs

@@ -46,6 +46,10 @@ impl StoreMeta {
         self.len() == 0
     }
 
+    pub fn version(&self) -> Version {
+        self.version
+    }
+
     pub fn export(&mut self, len: usize, height: Height) -> io::Result<()> {
         self.len = len;
         self.write_length()?;
@@ -61,6 +65,10 @@ impl StoreMeta {
         fs::create_dir(path)
     }
 
+    pub fn path(&self) -> &Path {
+        &self.pathbuf
+    }
+
     fn path_version(&self) -> PathBuf {
         Self::path_version_(&self.pathbuf)
     }

crates/brk_indexer/src/stores/mod.rs

@@ -27,11 +27,11 @@ impl Stores {
     pub fn import(path: &Path) -> color_eyre::Result<Self> {
         thread::scope(|scope| {
             let addresshash_to_addressindex = scope
-                .spawn(|| Store::import(&path.join("addresshash_to_addressindex"), Version::ONE));
+                .spawn(|| Store::import(&path.join("addresshash_to_addressindex"), Version::ZERO));
             let blockhash_prefix_to_height = scope
-                .spawn(|| Store::import(&path.join("blockhash_prefix_to_height"), Version::ONE));
+                .spawn(|| Store::import(&path.join("blockhash_prefix_to_height"), Version::ZERO));
             let txid_prefix_to_txindex =
-                scope.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::ONE));
+                scope.spawn(|| Store::import(&path.join("txid_prefix_to_txindex"), Version::ZERO));
 
             Ok(Self {
                 addresshash_to_addressindex: addresshash_to_addressindex.join().unwrap()?,

crates/brk_indexer/src/vecs/base.rs

@@ -1,113 +1,80 @@
 use std::{
     cmp::Ordering,
     fmt::Debug,
-    io,
     path::{Path, PathBuf},
 };
 
 use brk_vec::{
-    AnyStorableVec, Compressed, Error, MAX_CACHE_SIZE, MAX_PAGE_SIZE, Result, StoredIndex,
+    Compressed, DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, StoredVec, Value,
-    StoredType, Value, Version,
+    Version,
 };
 
 use super::Height;
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
-pub struct StorableVec<I, T> {
+pub struct IndexedVec<I, T>
-    height: Option<Height>,
-    vec: brk_vec::StorableVec<I, T>,
-}
-
-impl<I, T> StorableVec<I, T>
 where
     I: StoredIndex,
     T: StoredType,
 {
-    pub const SIZE_OF_T: usize = size_of::<T>();
+    height: Option<Height>,
-    pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T;
+    inner: StoredVec<I, T>,
-    pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE_OF_T;
+}
-    pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
 
+impl<I, T> IndexedVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
     pub fn forced_import(
         path: &Path,
         version: Version,
         compressed: Compressed,
     ) -> brk_vec::Result<Self> {
-        let mut vec = brk_vec::StorableVec::forced_import(path, version, compressed)?;
+        let mut inner = StoredVec::forced_import(path, version, compressed)?;
 
-        vec.enable_large_cache();
+        inner.enable_large_cache_if_needed();
 
         Ok(Self {
             height: Height::try_from(Self::path_height_(path).as_path()).ok(),
-            vec,
+            inner,
         })
     }
 
     #[inline]
     pub fn get(&self, index: I) -> Result<Option<Value<'_, T>>> {
-        self.get_(index.to_usize()?)
+        self.inner.get(index)
     }
-    fn get_(&self, index: usize) -> Result<Option<Value<'_, T>>> {
+    #[inline]
-        match self.vec.index_to_pushed_index(index) {
+    pub fn cached_get(&mut self, index: I) -> Result<Option<Value<'_, T>>> {
-            Ok(index) => {
+        self.inner.cached_get(index)
-                if let Some(index) = index {
+    }
-                    return Ok(self.vec.pushed().get(index).map(|v| Value::Ref(v)));
+    #[inline]
-                }
+    pub fn cached_get_(&mut self, index: usize) -> Result<Option<Value<'_, T>>> {
-            }
+        self.inner.cached_get_(index)
-            Err(Error::IndexTooHigh) => return Ok(None),
-            Err(Error::IndexTooLow) => {}
-            Err(error) => return Err(error),
-        }
-
-        let large_cache_len = self.vec.large_cache_len();
-        if large_cache_len != 0 {
-            let page_index = Self::index_to_page_index(index);
-            let last_index = self.vec.stored_len() - 1;
-            let max_page_index = Self::index_to_page_index(last_index);
-            let min_page_index = (max_page_index + 1) - large_cache_len;
-
-            if page_index >= min_page_index {
-                let values = self
-                    .vec
-                    .pages()
-                    .unwrap()
-                    .get(page_index - min_page_index)
-                    .ok_or(Error::MmapsVecIsTooSmall)?
-                    .get_or_init(|| self.vec.decode_page(page_index).unwrap());
-
-                return Ok(values.get(index)?.map(|v| Value::Ref(v)));
-            }
-        }
-
-        Ok(self.vec.read_(index)?.map(|v| Value::Owned(v)))
     }
 
     pub fn iter_from<F>(&mut self, index: I, f: F) -> Result<()>
     where
-        F: FnMut((I, &T)) -> Result<()>,
+        F: FnMut((I, T, &mut dyn DynamicVec<I = I, T = T>)) -> Result<()>,
     {
-        self.vec.iter_from(index, f)
+        self.inner.iter_from(index, f)
-    }
-
-    #[inline(always)]
-    fn index_to_page_index(index: usize) -> usize {
-        index / Self::PER_PAGE
     }
 
     #[inline]
     pub fn push_if_needed(&mut self, index: I, value: T) -> Result<()> {
-        match self.vec.len().cmp(&index.to_usize()?) {
+        match self.inner.len().cmp(&index.to_usize()?) {
             Ordering::Greater => {
                 // dbg!(len, index, &self.pathbuf);
                 // panic!();
                 Ok(())
             }
             Ordering::Equal => {
-                self.vec.push(value);
+                self.inner.push(value);
                 Ok(())
             }
             Ordering::Less => {
-                dbg!(index, value);
+                dbg!(index, value, self.inner.len(), self.path_height());
                 Err(Error::IndexTooHigh)
             }
         }
@@ -117,70 +84,57 @@ where
         if self.height.is_none_or(|self_height| self_height != height) {
             height.write(&self.path_height())?;
         }
-        self.vec.truncate_if_needed(index)?;
+        self.inner.truncate_if_needed(index)?;
         Ok(())
     }
 
-    pub fn flush(&mut self, height: Height) -> io::Result<()> {
+    pub fn flush(&mut self, height: Height) -> Result<()> {
         height.write(&self.path_height())?;
-        self.vec.flush()
+        self.inner.flush()
     }
 
-    pub fn vec(&self) -> &brk_vec::StorableVec<I, T> {
+    pub fn vec(&self) -> &StoredVec<I, T> {
-        &self.vec
+        &self.inner
     }
 
-    pub fn mut_vec(&mut self) -> &mut brk_vec::StorableVec<I, T> {
+    pub fn mut_vec(&mut self) -> &mut StoredVec<I, T> {
-        &mut self.vec
+        &mut self.inner
     }
 
-    pub fn any_vec(&self) -> &dyn AnyStorableVec {
+    pub fn any_vec(&self) -> &dyn brk_vec::AnyStoredVec {
-        &self.vec
+        &self.inner
     }
 
     pub fn len(&self) -> usize {
-        self.vec.len()
+        self.inner.len()
     }
 
     pub fn is_empty(&self) -> bool {
-        self.vec.is_empty()
+        self.inner.is_empty()
     }
 
     #[inline]
     pub fn hasnt(&self, index: I) -> Result<bool> {
-        self.vec.has(index).map(|b| !b)
+        self.inner.has(index).map(|b| !b)
     }
 
     pub fn height(&self) -> brk_core::Result<Height> {
         Height::try_from(self.path_height().as_path())
     }
     fn path_height(&self) -> PathBuf {
-        Self::path_height_(self.vec.path())
+        Self::path_height_(self.inner.path())
     }
     fn path_height_(path: &Path) -> PathBuf {
         path.join("height")
     }
 }
 
-impl<I, T> Clone for StorableVec<I, T>
-where
-    I: StoredIndex,
-    T: StoredType,
-{
-    fn clone(&self) -> Self {
-        Self {
-            height: self.height,
-            vec: self.vec.clone(),
-        }
-    }
-}
-
 pub trait AnyIndexedVec: Send + Sync {
     fn height(&self) -> brk_core::Result<Height>;
-    fn flush(&mut self, height: Height) -> io::Result<()>;
+    fn flush(&mut self, height: Height) -> Result<()>;
 }
 
-impl<I, T> AnyIndexedVec for StorableVec<I, T>
+impl<I, T> AnyIndexedVec for IndexedVec<I, T>
 where
     I: StoredIndex,
     T: StoredType,
@@ -189,7 +143,7 @@ where
         self.height()
     }
 
-    fn flush(&mut self, height: Height) -> io::Result<()> {
+    fn flush(&mut self, height: Height) -> Result<()> {
         self.flush(height)
     }
 }

crates/brk_indexer/src/vecs/mod.rs

@@ -1,13 +1,13 @@
-use std::{fs, io, path::Path};
+use std::{fs, path::Path};
 
 use brk_core::{
     Addressbytes, Addressindex, Addresstype, Addresstypeindex, BlockHash, Emptyindex, Height,
     LockTime, Multisigindex, Opreturnindex, P2PK33AddressBytes, P2PK33index, P2PK65AddressBytes,
     P2PK65index, P2PKHAddressBytes, P2PKHindex, P2SHAddressBytes, P2SHindex, P2TRAddressBytes,
     P2TRindex, P2WPKHAddressBytes, P2WPKHindex, P2WSHAddressBytes, P2WSHindex, Pushonlyindex, Sats,
-    Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight,
+    StoredUsize, Timestamp, TxVersion, Txid, Txindex, Txinindex, Txoutindex, Unknownindex, Weight,
 };
-use brk_vec::{AnyStorableVec, Compressed, Version};
+use brk_vec::{AnyStoredVec, Compressed, Result, Version};
 use rayon::prelude::*;
 
 use crate::Indexes;
@@ -18,50 +18,64 @@ pub use base::*;
 
 #[derive(Clone)]
 pub struct Vecs {
-    pub addressindex_to_addresstype: StorableVec<Addressindex, Addresstype>,
+    pub addressindex_to_addresstype: IndexedVec<Addressindex, Addresstype>,
-    pub addressindex_to_addresstypeindex: StorableVec<Addressindex, Addresstypeindex>,
+    pub addressindex_to_addresstypeindex: IndexedVec<Addressindex, Addresstypeindex>,
-    pub addressindex_to_height: StorableVec<Addressindex, Height>,
+    pub addressindex_to_height: IndexedVec<Addressindex, Height>,
-    pub height_to_blockhash: StorableVec<Height, BlockHash>,
+    pub emptyindex_to_height: IndexedVec<Emptyindex, Height>,
-    pub height_to_difficulty: StorableVec<Height, f64>,
+    pub height_to_blockhash: IndexedVec<Height, BlockHash>,
-    pub height_to_first_addressindex: StorableVec<Height, Addressindex>,
+    pub height_to_difficulty: IndexedVec<Height, f64>,
-    pub height_to_first_emptyindex: StorableVec<Height, Emptyindex>,
+    pub height_to_first_addressindex: IndexedVec<Height, Addressindex>,
-    pub height_to_first_multisigindex: StorableVec<Height, Multisigindex>,
+    pub height_to_first_emptyindex: IndexedVec<Height, Emptyindex>,
-    pub height_to_first_opreturnindex: StorableVec<Height, Opreturnindex>,
+    pub height_to_first_multisigindex: IndexedVec<Height, Multisigindex>,
-    pub height_to_first_pushonlyindex: StorableVec<Height, Pushonlyindex>,
+    pub height_to_first_opreturnindex: IndexedVec<Height, Opreturnindex>,
-    pub height_to_first_txindex: StorableVec<Height, Txindex>,
+    pub height_to_first_p2pk33index: IndexedVec<Height, P2PK33index>,
-    pub height_to_first_txinindex: StorableVec<Height, Txinindex>,
+    pub height_to_first_p2pk65index: IndexedVec<Height, P2PK65index>,
-    pub height_to_first_txoutindex: StorableVec<Height, Txoutindex>,
+    pub height_to_first_p2pkhindex: IndexedVec<Height, P2PKHindex>,
-    pub height_to_first_unknownindex: StorableVec<Height, Unknownindex>,
+    pub height_to_first_p2shindex: IndexedVec<Height, P2SHindex>,
-    pub height_to_first_p2pk33index: StorableVec<Height, P2PK33index>,
+    pub height_to_first_p2trindex: IndexedVec<Height, P2TRindex>,
-    pub height_to_first_p2pk65index: StorableVec<Height, P2PK65index>,
+    pub height_to_first_p2wpkhindex: IndexedVec<Height, P2WPKHindex>,
-    pub height_to_first_p2pkhindex: StorableVec<Height, P2PKHindex>,
+    pub height_to_first_p2wshindex: IndexedVec<Height, P2WSHindex>,
-    pub height_to_first_p2shindex: StorableVec<Height, P2SHindex>,
+    pub height_to_first_pushonlyindex: IndexedVec<Height, Pushonlyindex>,
-    pub height_to_first_p2trindex: StorableVec<Height, P2TRindex>,
+    pub height_to_first_txindex: IndexedVec<Height, Txindex>,
-    pub height_to_first_p2wpkhindex: StorableVec<Height, P2WPKHindex>,
+    pub height_to_first_txinindex: IndexedVec<Height, Txinindex>,
-    pub height_to_first_p2wshindex: StorableVec<Height, P2WSHindex>,
+    pub height_to_first_txoutindex: IndexedVec<Height, Txoutindex>,
-    pub height_to_size: StorableVec<Height, usize>,
+    pub height_to_first_unknownindex: IndexedVec<Height, Unknownindex>,
-    pub height_to_timestamp: StorableVec<Height, Timestamp>,
+    pub height_to_total_size: IndexedVec<Height, StoredUsize>,
-    pub height_to_weight: StorableVec<Height, Weight>,
+    pub height_to_timestamp: IndexedVec<Height, Timestamp>,
-    pub p2pk33index_to_p2pk33addressbytes: StorableVec<P2PK33index, P2PK33AddressBytes>,
+    pub height_to_weight: IndexedVec<Height, Weight>,
-    pub p2pk65index_to_p2pk65addressbytes: StorableVec<P2PK65index, P2PK65AddressBytes>,
+    pub multisigindex_to_height: IndexedVec<Multisigindex, Height>,
-    pub p2pkhindex_to_p2pkhaddressbytes: StorableVec<P2PKHindex, P2PKHAddressBytes>,
+    pub opreturnindex_to_height: IndexedVec<Opreturnindex, Height>,
-    pub p2shindex_to_p2shaddressbytes: StorableVec<P2SHindex, P2SHAddressBytes>,
+    pub p2pk33index_to_height: IndexedVec<P2PK33index, Height>,
-    pub p2trindex_to_p2traddressbytes: StorableVec<P2TRindex, P2TRAddressBytes>,
+    pub p2pk33index_to_p2pk33addressbytes: IndexedVec<P2PK33index, P2PK33AddressBytes>,
-    pub p2wpkhindex_to_p2wpkhaddressbytes: StorableVec<P2WPKHindex, P2WPKHAddressBytes>,
+    pub p2pk65index_to_height: IndexedVec<P2PK65index, Height>,
-    pub p2wshindex_to_p2wshaddressbytes: StorableVec<P2WSHindex, P2WSHAddressBytes>,
+    pub p2pk65index_to_p2pk65addressbytes: IndexedVec<P2PK65index, P2PK65AddressBytes>,
-    pub txindex_to_first_txinindex: StorableVec<Txindex, Txinindex>,
+    pub p2pkhindex_to_height: IndexedVec<P2PKHindex, Height>,
-    pub txindex_to_first_txoutindex: StorableVec<Txindex, Txoutindex>,
+    pub p2pkhindex_to_p2pkhaddressbytes: IndexedVec<P2PKHindex, P2PKHAddressBytes>,
-    pub txindex_to_height: StorableVec<Txindex, Height>,
+    pub p2shindex_to_height: IndexedVec<P2SHindex, Height>,
-    pub txindex_to_locktime: StorableVec<Txindex, LockTime>,
+    pub p2shindex_to_p2shaddressbytes: IndexedVec<P2SHindex, P2SHAddressBytes>,
-    pub txindex_to_txid: StorableVec<Txindex, Txid>,
+    pub p2trindex_to_height: IndexedVec<P2TRindex, Height>,
-    pub txindex_to_base_size: StorableVec<Txindex, usize>,
+    pub p2trindex_to_p2traddressbytes: IndexedVec<P2TRindex, P2TRAddressBytes>,
-    pub txindex_to_total_size: StorableVec<Txindex, usize>,
+    pub p2wpkhindex_to_height: IndexedVec<P2WPKHindex, Height>,
-    pub txindex_to_is_explicitly_rbf: StorableVec<Txindex, bool>,
+    pub p2wpkhindex_to_p2wpkhaddressbytes: IndexedVec<P2WPKHindex, P2WPKHAddressBytes>,
-    pub txindex_to_txversion: StorableVec<Txindex, TxVersion>,
+    pub p2wshindex_to_height: IndexedVec<P2WSHindex, Height>,
+    pub p2wshindex_to_p2wshaddressbytes: IndexedVec<P2WSHindex, P2WSHAddressBytes>,
+    pub pushonlyindex_to_height: IndexedVec<Pushonlyindex, Height>,
+    pub txindex_to_base_size: IndexedVec<Txindex, usize>,
+    pub txindex_to_first_txinindex: IndexedVec<Txindex, Txinindex>,
+    pub txindex_to_first_txoutindex: IndexedVec<Txindex, Txoutindex>,
+    pub txindex_to_height: IndexedVec<Txindex, Height>,
+    pub txindex_to_is_explicitly_rbf: IndexedVec<Txindex, bool>,
+    pub txindex_to_locktime: IndexedVec<Txindex, LockTime>,
+    pub txindex_to_total_size: IndexedVec<Txindex, usize>,
+    pub txindex_to_txid: IndexedVec<Txindex, Txid>,
+    pub txindex_to_txversion: IndexedVec<Txindex, TxVersion>,
+    pub txinindex_to_height: IndexedVec<Txinindex, Height>,
     /// If txoutindex == Txoutindex MAX then is it's coinbase
-    pub txinindex_to_txoutindex: StorableVec<Txinindex, Txoutindex>,
+    pub txinindex_to_txoutindex: IndexedVec<Txinindex, Txoutindex>,
-    pub txoutindex_to_addressindex: StorableVec<Txoutindex, Addressindex>,
+    pub txoutindex_to_addressindex: IndexedVec<Txoutindex, Addressindex>,
-    pub txoutindex_to_value: StorableVec<Txoutindex, Sats>,
+    pub txoutindex_to_height: IndexedVec<Txoutindex, Height>,
+    pub txoutindex_to_value: IndexedVec<Txoutindex, Sats>,
+    pub unknownindex_to_height: IndexedVec<Unknownindex, Height>,
 }
 
 impl Vecs {
@@ -69,219 +83,289 @@ impl Vecs {
         fs::create_dir_all(path)?;
 
         Ok(Self {
-            addressindex_to_addresstype: StorableVec::forced_import(
+            addressindex_to_addresstype: IndexedVec::forced_import(
                 &path.join("addressindex_to_addresstype"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            addressindex_to_addresstypeindex: StorableVec::forced_import(
+            addressindex_to_addresstypeindex: IndexedVec::forced_import(
                 &path.join("addressindex_to_addresstypeindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            addressindex_to_height: StorableVec::forced_import(
+            addressindex_to_height: IndexedVec::forced_import(
                 &path.join("addressindex_to_height"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_blockhash: StorableVec::forced_import(
+            height_to_blockhash: IndexedVec::forced_import(
                 &path.join("height_to_blockhash"),
-                Version::ONE,
+                Version::ZERO,
                 Compressed::NO,
             )?,
-            height_to_difficulty: StorableVec::forced_import(
+            height_to_difficulty: IndexedVec::forced_import(
                 &path.join("height_to_difficulty"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_addressindex: StorableVec::forced_import(
+            height_to_first_addressindex: IndexedVec::forced_import(
                 &path.join("height_to_first_addressindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_emptyindex: StorableVec::forced_import(
+            height_to_first_emptyindex: IndexedVec::forced_import(
                 &path.join("height_to_first_emptyindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_multisigindex: StorableVec::forced_import(
+            height_to_first_multisigindex: IndexedVec::forced_import(
                 &path.join("height_to_first_multisigindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_opreturnindex: StorableVec::forced_import(
+            height_to_first_opreturnindex: IndexedVec::forced_import(
                 &path.join("height_to_first_opreturnindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_pushonlyindex: StorableVec::forced_import(
+            height_to_first_pushonlyindex: IndexedVec::forced_import(
                 &path.join("height_to_first_pushonlyindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_txindex: StorableVec::forced_import(
+            height_to_first_txindex: IndexedVec::forced_import(
                 &path.join("height_to_first_txindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_txinindex: StorableVec::forced_import(
+            height_to_first_txinindex: IndexedVec::forced_import(
                 &path.join("height_to_first_txinindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_txoutindex: StorableVec::forced_import(
+            height_to_first_txoutindex: IndexedVec::forced_import(
                 &path.join("height_to_first_txoutindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_unknownindex: StorableVec::forced_import(
+            height_to_first_unknownindex: IndexedVec::forced_import(
                 &path.join("height_to_first_unkownindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_p2pk33index: StorableVec::forced_import(
+            height_to_first_p2pk33index: IndexedVec::forced_import(
                 &path.join("height_to_first_p2pk33index"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_p2pk65index: StorableVec::forced_import(
+            height_to_first_p2pk65index: IndexedVec::forced_import(
                 &path.join("height_to_first_p2pk65index"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_p2pkhindex: StorableVec::forced_import(
+            height_to_first_p2pkhindex: IndexedVec::forced_import(
                 &path.join("height_to_first_p2pkhindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_p2shindex: StorableVec::forced_import(
+            height_to_first_p2shindex: IndexedVec::forced_import(
                 &path.join("height_to_first_p2shindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_p2trindex: StorableVec::forced_import(
+            height_to_first_p2trindex: IndexedVec::forced_import(
                 &path.join("height_to_first_p2trindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_p2wpkhindex: StorableVec::forced_import(
+            height_to_first_p2wpkhindex: IndexedVec::forced_import(
                 &path.join("height_to_first_p2wpkhindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_first_p2wshindex: StorableVec::forced_import(
+            height_to_first_p2wshindex: IndexedVec::forced_import(
                 &path.join("height_to_first_p2wshindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_size: StorableVec::forced_import(
+            height_to_total_size: IndexedVec::forced_import(
-                &path.join("height_to_size"),
+                &path.join("height_to_total_size"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_timestamp: StorableVec::forced_import(
+            height_to_timestamp: IndexedVec::forced_import(
                 &path.join("height_to_timestamp"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            height_to_weight: StorableVec::forced_import(
+            height_to_weight: IndexedVec::forced_import(
                 &path.join("height_to_weight"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            p2pk33index_to_p2pk33addressbytes: StorableVec::forced_import(
+            p2pk33index_to_p2pk33addressbytes: IndexedVec::forced_import(
                 &path.join("p2pk33index_to_p2pk33addressbytes"),
-                Version::ONE,
+                Version::ZERO,
                 Compressed::NO,
             )?,
-            p2pk65index_to_p2pk65addressbytes: StorableVec::forced_import(
+            p2pk65index_to_p2pk65addressbytes: IndexedVec::forced_import(
                 &path.join("p2pk65index_to_p2pk65addressbytes"),
-                Version::ONE,
+                Version::ZERO,
                 Compressed::NO,
             )?,
-            p2pkhindex_to_p2pkhaddressbytes: StorableVec::forced_import(
+            p2pkhindex_to_p2pkhaddressbytes: IndexedVec::forced_import(
                 &path.join("p2pkhindex_to_p2pkhaddressbytes"),
-                Version::ONE,
+                Version::ZERO,
                 Compressed::NO,
             )?,
-            p2shindex_to_p2shaddressbytes: StorableVec::forced_import(
+            p2shindex_to_p2shaddressbytes: IndexedVec::forced_import(
                 &path.join("p2shindex_to_p2shaddressbytes"),
-                Version::ONE,
+                Version::ZERO,
                 Compressed::NO,
             )?,
-            p2trindex_to_p2traddressbytes: StorableVec::forced_import(
+            p2trindex_to_p2traddressbytes: IndexedVec::forced_import(
                 &path.join("p2trindex_to_p2traddressbytes"),
-                Version::ONE,
+                Version::ZERO,
                 Compressed::NO,
             )?,
-            p2wpkhindex_to_p2wpkhaddressbytes: StorableVec::forced_import(
+            p2wpkhindex_to_p2wpkhaddressbytes: IndexedVec::forced_import(
                 &path.join("p2wpkhindex_to_p2wpkhaddressbytes"),
-                Version::ONE,
+                Version::ZERO,
                 Compressed::NO,
             )?,
-            p2wshindex_to_p2wshaddressbytes: StorableVec::forced_import(
+            p2wshindex_to_p2wshaddressbytes: IndexedVec::forced_import(
                 &path.join("p2wshindex_to_p2wshaddressbytes"),
-                Version::ONE,
+                Version::ZERO,
                 Compressed::NO,
             )?,
-            txindex_to_first_txinindex: StorableVec::forced_import(
+            txindex_to_first_txinindex: IndexedVec::forced_import(
                 &path.join("txindex_to_first_txinindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            txindex_to_first_txoutindex: StorableVec::forced_import(
+            txindex_to_first_txoutindex: IndexedVec::forced_import(
                 &path.join("txindex_to_first_txoutindex"),
-                Version::ONE,
+                Version::ZERO,
                 Compressed::NO,
             )?,
-            txindex_to_height: StorableVec::forced_import(
+            txindex_to_height: IndexedVec::forced_import(
                 &path.join("txindex_to_height"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            txindex_to_locktime: StorableVec::forced_import(
+            txindex_to_locktime: IndexedVec::forced_import(
                 &path.join("txindex_to_locktime"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            txindex_to_txid: StorableVec::forced_import(
+            txindex_to_txid: IndexedVec::forced_import(
                 &path.join("txindex_to_txid"),
-                Version::ONE,
+                Version::ZERO,
                 Compressed::NO,
             )?,
-            txindex_to_base_size: StorableVec::forced_import(
+            txindex_to_base_size: IndexedVec::forced_import(
                 &path.join("txindex_to_base_size"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            txindex_to_total_size: StorableVec::forced_import(
+            txindex_to_total_size: IndexedVec::forced_import(
                 &path.join("txindex_to_total_size"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            txindex_to_is_explicitly_rbf: StorableVec::forced_import(
+            txindex_to_is_explicitly_rbf: IndexedVec::forced_import(
                 &path.join("txindex_to_is_explicitly_rbf"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            txindex_to_txversion: StorableVec::forced_import(
+            txindex_to_txversion: IndexedVec::forced_import(
                 &path.join("txindex_to_txversion"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            txinindex_to_txoutindex: StorableVec::forced_import(
+            txinindex_to_txoutindex: IndexedVec::forced_import(
                 &path.join("txinindex_to_txoutindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            txoutindex_to_addressindex: StorableVec::forced_import(
+            txoutindex_to_addressindex: IndexedVec::forced_import(
                 &path.join("txoutindex_to_addressindex"),
-                Version::ONE,
+                Version::ZERO,
                 compressed,
             )?,
-            txoutindex_to_value: StorableVec::forced_import(
+            txoutindex_to_value: IndexedVec::forced_import(
                 &path.join("txoutindex_to_value"),
-                Version::ONE,
+                Version::ZERO,
+                compressed,
+            )?,
+            emptyindex_to_height: IndexedVec::forced_import(
+                &path.join("emptyindex_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            multisigindex_to_height: IndexedVec::forced_import(
+                &path.join("multisigindex_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            opreturnindex_to_height: IndexedVec::forced_import(
+                &path.join("opreturnindex_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            pushonlyindex_to_height: IndexedVec::forced_import(
+                &path.join("pushonlyindex_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            txinindex_to_height: IndexedVec::forced_import(
+                &path.join("txinindex_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            txoutindex_to_height: IndexedVec::forced_import(
+                &path.join("txoutindex_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            unknownindex_to_height: IndexedVec::forced_import(
+                &path.join("unknownindex_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            p2pk33index_to_height: IndexedVec::forced_import(
+                &path.join("p2pk33index_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            p2pk65index_to_height: IndexedVec::forced_import(
+                &path.join("p2pk65index_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            p2pkhindex_to_height: IndexedVec::forced_import(
+                &path.join("p2pkhindex_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            p2shindex_to_height: IndexedVec::forced_import(
+                &path.join("p2shindex_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            p2trindex_to_height: IndexedVec::forced_import(
+                &path.join("p2trindex_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            p2wpkhindex_to_height: IndexedVec::forced_import(
+                &path.join("p2wpkhindex_to_height"),
+                Version::ZERO,
+                compressed,
+            )?,
+            p2wshindex_to_height: IndexedVec::forced_import(
+                &path.join("p2wshindex_to_height"),
+                Version::ZERO,
                 compressed,
             )?,
         })
@@ -290,8 +374,25 @@ impl Vecs {
     pub fn rollback_if_needed(&mut self, starting_indexes: &Indexes) -> brk_vec::Result<()> {
         let saved_height = starting_indexes.height.decremented().unwrap_or_default();
 
-        // We don't want to override the starting indexes so we cut from n + 1
+        let &Indexes {
-        let height = starting_indexes.height.incremented();
+            addressindex,
+            height,
+            p2pk33index,
+            p2pk65index,
+            p2pkhindex,
+            p2shindex,
+            p2trindex,
+            p2wpkhindex,
+            p2wshindex,
+            txindex,
+            txinindex,
+            txoutindex,
+            unknownindex,
+            pushonlyindex,
+            opreturnindex,
+            multisigindex,
+            emptyindex,
+        } = starting_indexes;
 
         self.height_to_first_addressindex
             .truncate_if_needed(height, saved_height)?;
@@ -326,28 +427,11 @@ impl Vecs {
         self.height_to_first_unknownindex
             .truncate_if_needed(height, saved_height)?;
 
-        // Now we can cut everything that's out of date
-        let &Indexes {
-            addressindex,
-            height,
-            p2pk33index,
-            p2pk65index,
-            p2pkhindex,
-            p2shindex,
-            p2trindex,
-            p2wpkhindex,
-            p2wshindex,
-            txindex,
-            txinindex,
-            txoutindex,
-            ..
-        } = starting_indexes;
-
         self.height_to_blockhash
             .truncate_if_needed(height, saved_height)?;
         self.height_to_difficulty
             .truncate_if_needed(height, saved_height)?;
-        self.height_to_size
+        self.height_to_total_size
             .truncate_if_needed(height, saved_height)?;
         self.height_to_timestamp
             .truncate_if_needed(height, saved_height)?;
@@ -403,6 +487,35 @@ impl Vecs {
         self.txoutindex_to_value
             .truncate_if_needed(txoutindex, saved_height)?;
 
+        self.emptyindex_to_height
+            .truncate_if_needed(emptyindex, saved_height)?;
+        self.multisigindex_to_height
+            .truncate_if_needed(multisigindex, saved_height)?;
+        self.opreturnindex_to_height
+            .truncate_if_needed(opreturnindex, saved_height)?;
+        self.pushonlyindex_to_height
+            .truncate_if_needed(pushonlyindex, saved_height)?;
+        self.txinindex_to_height
+            .truncate_if_needed(txinindex, saved_height)?;
+        self.txoutindex_to_height
+            .truncate_if_needed(txoutindex, saved_height)?;
+        self.unknownindex_to_height
+            .truncate_if_needed(unknownindex, saved_height)?;
+        self.p2pk33index_to_height
+            .truncate_if_needed(p2pk33index, saved_height)?;
+        self.p2pk65index_to_height
+            .truncate_if_needed(p2pk65index, saved_height)?;
+        self.p2pkhindex_to_height
+            .truncate_if_needed(p2pkhindex, saved_height)?;
+        self.p2shindex_to_height
+            .truncate_if_needed(p2shindex, saved_height)?;
+        self.p2trindex_to_height
+            .truncate_if_needed(p2trindex, saved_height)?;
+        self.p2wpkhindex_to_height
+            .truncate_if_needed(p2wpkhindex, saved_height)?;
+        self.p2wshindex_to_height
+            .truncate_if_needed(p2wshindex, saved_height)?;
+
         Ok(())
     }
 
@@ -481,7 +594,7 @@ impl Vecs {
         }
     }
 
-    pub fn flush(&mut self, height: Height) -> io::Result<()> {
+    pub fn flush(&mut self, height: Height) -> Result<()> {
         self.as_mut_any_vecs()
             .into_par_iter()
             .try_for_each(|vec| vec.flush(height))
@@ -495,7 +608,7 @@ impl Vecs {
             .unwrap()
     }
 
-    pub fn as_any_vecs(&self) -> Vec<&dyn AnyStorableVec> {
+    pub fn as_any_vecs(&self) -> Vec<&dyn AnyStoredVec> {
         vec![
             self.addressindex_to_addresstype.any_vec(),
             self.addressindex_to_addresstypeindex.any_vec(),
@@ -518,7 +631,7 @@ impl Vecs {
             self.height_to_first_p2trindex.any_vec(),
             self.height_to_first_p2wpkhindex.any_vec(),
             self.height_to_first_p2wshindex.any_vec(),
-            self.height_to_size.any_vec(),
+            self.height_to_total_size.any_vec(),
             self.height_to_timestamp.any_vec(),
             self.height_to_weight.any_vec(),
             self.p2pk33index_to_p2pk33addressbytes.any_vec(),
@@ -540,6 +653,20 @@ impl Vecs {
             self.txinindex_to_txoutindex.any_vec(),
             self.txoutindex_to_addressindex.any_vec(),
             self.txoutindex_to_value.any_vec(),
+            self.emptyindex_to_height.any_vec(),
+            self.multisigindex_to_height.any_vec(),
+            self.opreturnindex_to_height.any_vec(),
+            self.pushonlyindex_to_height.any_vec(),
+            self.txinindex_to_height.any_vec(),
+            self.txoutindex_to_height.any_vec(),
+            self.unknownindex_to_height.any_vec(),
+            self.p2pk33index_to_height.any_vec(),
+            self.p2pk65index_to_height.any_vec(),
+            self.p2pkhindex_to_height.any_vec(),
+            self.p2shindex_to_height.any_vec(),
+            self.p2trindex_to_height.any_vec(),
+            self.p2wpkhindex_to_height.any_vec(),
+            self.p2wshindex_to_height.any_vec(),
         ]
     }
 
@@ -566,7 +693,7 @@ impl Vecs {
             &mut self.height_to_first_p2trindex,
             &mut self.height_to_first_p2wpkhindex,
             &mut self.height_to_first_p2wshindex,
-            &mut self.height_to_size,
+            &mut self.height_to_total_size,
             &mut self.height_to_timestamp,
             &mut self.height_to_weight,
             &mut self.p2pk33index_to_p2pk33addressbytes,
@@ -588,6 +715,20 @@ impl Vecs {
             &mut self.txinindex_to_txoutindex,
             &mut self.txoutindex_to_addressindex,
             &mut self.txoutindex_to_value,
+            &mut self.emptyindex_to_height,
+            &mut self.multisigindex_to_height,
+            &mut self.opreturnindex_to_height,
+            &mut self.pushonlyindex_to_height,
+            &mut self.txinindex_to_height,
+            &mut self.txoutindex_to_height,
+            &mut self.unknownindex_to_height,
+            &mut self.p2pk33index_to_height,
+            &mut self.p2pk65index_to_height,
+            &mut self.p2pkhindex_to_height,
+            &mut self.p2shindex_to_height,
+            &mut self.p2trindex_to_height,
+            &mut self.p2wpkhindex_to_height,
+            &mut self.p2wshindex_to_height,
         ]
     }
 }

crates/brk_logger/README.md

@@ -20,7 +20,7 @@
   <a href="https://deps.rs/crate/brk_logger">
     <img src="https://deps.rs/crate/brk_logger/latest/status.svg" alt="Dependency status">
   </a>
-  <a href="https://discord.gg/Cvrwpv3zEG">
+  <a href="https://discord.gg/HaR3wpH3nr">
     <img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
   </a>
   <a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">

crates/brk_parser/README.md

@@ -20,7 +20,7 @@
   <a href="https://deps.rs/crate/brk_parser">
     <img src="https://deps.rs/crate/brk_parser/latest/status.svg" alt="Dependency status">
   </a>
-  <a href="https://discord.gg/Cvrwpv3zEG">
+  <a href="https://discord.gg/HaR3wpH3nr">
     <img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
   </a>
   <a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">

crates/brk_query/README.md

@@ -20,7 +20,7 @@
   <a href="https://deps.rs/crate/brk_query">
     <img src="https://deps.rs/crate/brk_query/latest/status.svg" alt="Dependency status">
   </a>
-  <a href="https://discord.gg/Cvrwpv3zEG">
+  <a href="https://discord.gg/HaR3wpH3nr">
     <img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
   </a>
   <a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">

crates/brk_query/src/index.rs

@@ -24,10 +24,15 @@ pub enum Index {
     Decadeindex,
     Difficultyepoch,
     Halvingepoch,
+    Emptyindex,
+    Multisigindex,
+    Opreturnindex,
+    Pushonlyindex,
+    Unknownindex,
 }
 
 impl Index {
-    pub fn all() -> [Self; 20] {
+    pub fn all() -> [Self; 25] {
         [
             Self::Height,
             Self::Dateindex,
@@ -49,6 +54,11 @@ impl Index {
             Self::Txindex,
             Self::Txinindex,
             Self::Txoutindex,
+            Self::Emptyindex,
+            Self::Multisigindex,
+            Self::Opreturnindex,
+            Self::Pushonlyindex,
+            Self::Unknownindex,
         ]
     }
 
@@ -75,6 +85,11 @@ impl Index {
             Self::P2TRindex => &["p2tr", "p2trindex"],
             Self::P2WPKHindex => &["p2wpkh", "p2wpkhindex"],
             Self::P2WSHindex => &["p2wsh", "p2wshindex"],
+            Self::Emptyindex => &["empty", "emptyindex"],
+            Self::Multisigindex => &["multisig", "multisigindex"],
+            Self::Opreturnindex => &["opreturn", "opreturnindex"],
+            Self::Pushonlyindex => &["pushonly", "pushonlyindex"],
+            Self::Unknownindex => &["unknown", "unknownindex"],
         }
     }
 
@@ -122,6 +137,12 @@ impl TryFrom<&str> for Index {
             v if (Self::Difficultyepoch).possible_values().contains(&v) => Self::Difficultyepoch,
             v if (Self::Halvingepoch).possible_values().contains(&v) => Self::Halvingepoch,
             v if (Self::Quarterindex).possible_values().contains(&v) => Self::Quarterindex,
+            v if (Self::Quarterindex).possible_values().contains(&v) => Self::Quarterindex,
+            v if (Self::Emptyindex).possible_values().contains(&v) => Self::Emptyindex,
+            v if (Self::Multisigindex).possible_values().contains(&v) => Self::Multisigindex,
+            v if (Self::Opreturnindex).possible_values().contains(&v) => Self::Opreturnindex,
+            v if (Self::Pushonlyindex).possible_values().contains(&v) => Self::Pushonlyindex,
+            v if (Self::Unknownindex).possible_values().contains(&v) => Self::Unknownindex,
             _ => return Err(eyre!("Bad index")),
         })
     }

crates/brk_query/src/lib.rs

@@ -5,7 +5,7 @@
 
 use brk_computer::Computer;
 use brk_indexer::Indexer;
-use brk_vec::AnyStorableVec;
+use brk_vec::AnyStoredVec;
 use tabled::settings::Style;
 
 mod format;
@@ -51,7 +51,7 @@ impl<'a> Query<'a> {
         }
     }
 
-    pub fn search(&self, index: Index, ids: &[&str]) -> Vec<(String, &&dyn AnyStorableVec)> {
+    pub fn search(&self, index: Index, ids: &[&str]) -> Vec<(String, &&dyn AnyStoredVec)> {
         let tuples = ids
             .iter()
             .flat_map(|s| {
@@ -86,7 +86,7 @@ impl<'a> Query<'a> {
 
     pub fn format(
         &self,
-        vecs: Vec<(String, &&dyn AnyStorableVec)>,
+        vecs: Vec<(String, &&dyn AnyStoredVec)>,
         from: Option<i64>,
         to: Option<i64>,
         format: Option<Format>,
@@ -94,7 +94,7 @@ impl<'a> Query<'a> {
         let mut values = vecs
             .iter()
             .map(|(_, vec)| -> brk_vec::Result<Vec<serde_json::Value>> {
-                vec.collect_range_values(from, to)
+                vec.collect_range_serde_json(from, to)
             })
             .collect::<brk_vec::Result<Vec<_>>>()?;
 

crates/brk_query/src/vec_trees.rs

@@ -1,6 +1,6 @@
 use std::collections::BTreeMap;
 
-use brk_vec::AnyStorableVec;
+use brk_vec::AnyStoredVec;
 use derive_deref::{Deref, DerefMut};
 
 use super::index::Index;
@@ -13,10 +13,11 @@ pub struct VecTrees<'a> {
 
 impl<'a> VecTrees<'a> {
     // Not the most performant or type safe but only built once so that's okay
-    pub fn insert(&mut self, vec: &'a dyn AnyStorableVec) {
+    pub fn insert(&mut self, vec: &'a dyn AnyStoredVec) {
         let file_name = vec.file_name();
         let split = file_name.split("_to_").collect::<Vec<_>>();
         if split.len() != 2 {
+            dbg!(&file_name, &split);
             panic!();
         }
         let str = vec
@@ -87,7 +88,7 @@ impl<'a> VecTrees<'a> {
 }
 
 #[derive(Default, Deref, DerefMut)]
-pub struct IndexToVec<'a>(BTreeMap<Index, &'a dyn AnyStorableVec>);
+pub struct IndexToVec<'a>(BTreeMap<Index, &'a dyn AnyStoredVec>);
 
 #[derive(Default, Deref, DerefMut)]
-pub struct IdToVec<'a>(BTreeMap<String, &'a dyn AnyStorableVec>);
+pub struct IdToVec<'a>(BTreeMap<String, &'a dyn AnyStoredVec>);

crates/brk_server/Cargo.toml

@@ -7,7 +7,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
-axum = "0.8.3"
+axum = { workspace = true }
 brk_computer = { workspace = true }
 brk_exit = { workspace = true }
 brk_fetcher = { workspace = true }
@@ -21,10 +21,9 @@ color-eyre = { workspace = true }
 jiff = { workspace = true }
 log = { workspace = true }
 minreq = { workspace = true }
-oxc = { version = "0.62.0", features = ["codegen", "minifier"] }
+oxc = { version = "0.63.0", features = ["codegen", "minifier"] }
 serde = { workspace = true }
-tokio = { version = "1.44.1", features = ["full"] }
+tokio = { version = "1.44.2", features = ["full"] }
 tower-http = { version = "0.6.2", features = ["compression-full", "trace"] }
-zip = "2.5.0"
+zip = "2.6.1"
 tracing = "0.1.41"
-tracing-subscriber = "0.3.19"

crates/brk_server/README.md

@@ -20,7 +20,7 @@
   <a href="https://deps.rs/crate/brk_server">
     <img src="https://deps.rs/crate/brk_server/latest/status.svg" alt="Dependency status">
   </a>
-  <a href="https://discord.gg/Cvrwpv3zEG">
+  <a href="https://discord.gg/HaR3wpH3nr">
     <img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
   </a>
   <a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">

crates/brk_server/examples/main.rs

@@ -10,7 +10,6 @@ use brk_parser::{
     rpc::{self, RpcApi},
 };
 use brk_server::{Server, Website};
-use log::info;
 
 pub fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
@@ -60,14 +59,10 @@ pub fn main() -> color_eyre::Result<()> {
                 loop {
                     let block_count = rpc.get_block_count()?;
 
-                    info!("{block_count} blocks found.");
-
                     let starting_indexes = indexer.index(&parser, rpc, &exit)?;
 
                     computer.compute(&mut indexer, starting_indexes, &exit)?;
 
-                    info!("Waiting for new blocks...");
-
                     while block_count == rpc.get_block_count()? {
                         sleep(Duration::from_secs(1))
                     }

crates/brk_server/src/api/query/dts.rs

@@ -67,7 +67,7 @@ impl DTS for Query<'static> {
             .collect::<Vec<_>>()
             .join("\n");
 
-        contents += "\n\n  return {\n";
+        contents += "\n\n  return /** @type {const} */ ({\n";
 
         self.vec_trees
             .id_to_index_to_vec
@@ -89,7 +89,7 @@ impl DTS for Query<'static> {
                 );
             });
 
-        contents += "  }\n";
+        contents += "  });\n";
         contents.push('}');
 
         contents += "\n/** @typedef {ReturnType<typeof createVecIdToIndexes>} VecIdToIndexes */";

crates/brk_server/src/lib.rs

@@ -49,6 +49,7 @@ pub struct AppState {
 const DEV_PATH: &str = "../..";
 const DOWNLOADS: &str = "downloads";
 const WEBSITES: &str = "websites";
+const VERSION: &str = env!("CARGO_PKG_VERSION");
 
 pub struct Server(AppState);
 
@@ -66,16 +67,14 @@ impl Server {
             } else {
                 let downloads_path = dot_brk_path().join(DOWNLOADS);
 
-                let version = format!("v{}", env!("CARGO_PKG_VERSION"));
+                let downloaded_websites_path =
-
+                    downloads_path.join(format!("brk-{VERSION}")).join(WEBSITES);
-                let downloaded_websites_path = downloads_path.join(&version).join(WEBSITES);
 
                 if !fs::exists(&downloaded_websites_path)? {
                     info!("Downloading websites from Github...");
 
                     let url = format!(
-                        "https://github.com/bitcoinresearchkit/brk/archive/refs/tags/{}.zip",
+                        "https://github.com/bitcoinresearchkit/brk/archive/refs/tags/v{VERSION}.zip",
-                        version
                     );
 
                     let response = minreq::get(url).send()?;
@@ -130,15 +129,14 @@ impl Server {
                     let status = response.status();
                     let uri = response.extensions().get::<Uri>().unwrap();
                     match status {
-                        StatusCode::INTERNAL_SERVER_ERROR => {
-                            error!("{} {} {:?}", status.as_u16().red(), uri, latency)
-                        }
-                        StatusCode::NOT_MODIFIED => {
-                            info!("{} {} {:?}", status.as_u16().bright_black(), uri, latency)
-                        }
                         StatusCode::OK => {
                             info!("{} {} {:?}", status.as_u16().green(), uri, latency)
                         }
+                        StatusCode::NOT_MODIFIED
+                        | StatusCode::TEMPORARY_REDIRECT
+                        | StatusCode::PERMANENT_REDIRECT => {
+                            info!("{} {} {:?}", status.as_u16().bright_black(), uri, latency)
+                        }
                         _ => error!("{} {} {:?}", status.as_u16().red(), uri, latency),
                     }
                 },
@@ -150,7 +148,7 @@ impl Server {
         let router = Router::new()
             .add_api_routes()
             .add_website_routes(state.website)
-            .route("/version", get(Json(env!("CARGO_PKG_VERSION"))))
+            .route("/version", get(Json(VERSION)))
             .with_state(state)
             .layer(compression_layer)
             .layer(response_uri_layer)

crates/brk_vec/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "brk_vec"
-description = "A very small, fast, efficient and simple storable Vec"
+description = "A push-only, truncable, compressable, saveable Vec"
 keywords = ["vec", "disk", "data"]
 categories = ["database"]
 version.workspace = true
@@ -9,6 +9,8 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+axum = { workspace = true }
+arc-swap = "1.7.1"
 memmap2 = "0.9.5"
 rayon = { workspace = true }
 serde = { workspace = true }

crates/brk_vec/README.md

@@ -20,7 +20,7 @@
   <a href="https://deps.rs/crate/brk_vec">
     <img src="https://deps.rs/crate/brk_vec/latest/status.svg" alt="Dependency status">
   </a>
-  <a href="https://discord.gg/Cvrwpv3zEG">
+  <a href="https://discord.gg/HaR3wpH3nr">
     <img src="https://img.shields.io/discord/1350431684562124850?label=discord" alt="Discord" />
   </a>
   <a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
@@ -39,19 +39,3 @@ A `Vec` (an array) that is stored on disk and thus which can be much larger than
 Compared to a key/value store, the data stored is raw byte interpretation of the Vec's values without any overhead which is very efficient. Additionally it uses close to no RAM when caching isn't active and up to 100 MB when it is.
 
 Compression is also available and built on top [`zstd`](https://crates.io/crates/zstd) to save even more space (from 0 to 75%). The tradeoff being slower reading speeds, especially random reading speeds. This is due to the data being stored in compressed pages of 16 KB, which means that if you to read even one value in that page you have to uncompress the whole page.
-
-## Disclaimer
-
-Portability will depend on the type of values.
-
-Non bytes/slices types (`u8`, `u16`, ...) will be read as slice in an unsafe manner (using `std::slice::from_raw_parts`) and thus have the endianness of the system. On the other hand, `&[u8]` should be inserted as is.
-
-If portability is important to you, just create a wrapper struct which has custom `get`, `push`, ... methods and does something like:
-
-```rust
-impl StorableVecU64 {
-    pub fn push(&mut self, value: u64) {
-        self.push(&value.to_be_bytes())
-    }
-}
-```

crates/brk_vec/examples/main.rs

@@ -1,13 +1,16 @@
 use std::{fs, path::Path};
 
-use brk_vec::{Compressed, StorableVec, Version};
+use brk_vec::{Compressed, DynamicVec, GenericVec, StoredVec, Version};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let _ = fs::remove_dir_all("./vec");
 
+    let version = Version::ZERO;
+    let compressed = Compressed::YES;
+
     {
-        let mut vec: StorableVec<usize, u32> =
+        let mut vec: StoredVec<usize, u32> =
-            StorableVec::forced_import(Path::new("./vec"), Version::ONE, Compressed::YES)?;
+            StoredVec::forced_import(Path::new("./vec"), version, compressed)?;
 
         (0..21_u32).for_each(|v| {
             vec.push(v);
@@ -20,8 +23,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     }
 
     {
-        let mut vec: StorableVec<usize, u32> =
+        let mut vec: StoredVec<usize, u32> =
-            StorableVec::forced_import(Path::new("./vec"), Version::ONE, Compressed::YES)?;
+            StoredVec::forced_import(Path::new("./vec"), version, compressed)?;
 
         dbg!(vec.get(0)?);
         dbg!(vec.get(0)?);
@@ -42,10 +45,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     }
 
     {
-        let mut vec: StorableVec<usize, u32> =
+        let mut vec: StoredVec<usize, u32> =
-            StorableVec::forced_import(Path::new("./vec"), Version::ONE, Compressed::YES)?;
+            StoredVec::forced_import(Path::new("./vec"), version, compressed)?;
 
-        vec.enable_large_cache();
+        vec.enable_large_cache_if_needed();
 
         dbg!(vec.get(0)?);
         dbg!(vec.get(20)?);
@@ -58,17 +61,17 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         dbg!(vec.get(5)?);
         dbg!(vec.get(20)?);
 
-        vec.iter(|(_, v)| {
+        vec.iter(|(_, v, ..)| {
             dbg!(v);
             Ok(())
         })?;
 
-        vec.iter_from(5, |(_, v)| {
+        vec.iter_from(5, |(_, v, ..)| {
             dbg!(v);
             Ok(())
         })?;
 
-        dbg!(vec.collect_range(Some(-5), None)?);
+        dbg!(vec.collect_signed_range(Some(-5), None)?);
     }
 
     Ok(())

crates/brk_vec/src/enums/error.rs

@@ -15,13 +15,15 @@ pub enum Error {
     IO(io::Error),
     ZeroCopyError,
     IndexTooHigh,
+    EmptyVec,
     IndexTooLow,
     ExpectFileToHaveIndex,
     ExpectVecToHaveIndex,
     FailedKeyTryIntoUsize,
     UnsupportedUnflushedState,
-    RangeFromAfterTo,
+    RangeFromAfterTo(usize, usize),
     DifferentCompressionMode,
+    ToSerdeJsonValueError(serde_json::Error),
 }
 
 impl From<io::Error> for Error {
@@ -42,6 +44,12 @@ impl<A, B> From<zerocopy::error::SizeError<A, B>> for Error {
     }
 }
 
+impl From<serde_json::Error> for Error {
+    fn from(error: serde_json::Error) -> Self {
+        Self::ToSerdeJsonValueError(error)
+    }
+}
+
 impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
@@ -66,8 +74,10 @@ impl fmt::Display for Error {
                 )
             }
             Error::ZeroCopyError => write!(f, "Zero copy convert error"),
-            Error::RangeFromAfterTo => write!(f, "Range, from is after to"),
+            Error::RangeFromAfterTo(from, to) => write!(f, "Range, from {from} is after to {to}"),
             Error::DifferentCompressionMode => write!(f, "Different compression mode chosen"),
+            Error::EmptyVec => write!(f, "The Vec is empty, maybe wait for a bit"),
+            Error::ToSerdeJsonValueError(error) => Debug::fmt(&error, f),
         }
     }
 }

crates/brk_vec/src/enums/mod.rs

@@ -1,7 +1,5 @@
 mod error;
 mod value;
-mod values;
 
 pub use error::*;
 pub use value::*;
-pub use values::*;

crates/brk_vec/src/enums/values.rs

@@ -1,83 +0,0 @@
-use std::ops::Range;
-
-use memmap2::Mmap;
-use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
-
-use crate::MAX_PAGE_SIZE;
-
-use super::Result;
-
-#[derive(Debug)]
-pub enum Values<T> {
-    Owned(Box<[T]>),
-    Ref(Box<Mmap>),
-}
-
-impl<T> Values<T> {
-    const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T;
-    const SIZE_OF_T: usize = size_of::<T>();
-
-    pub fn get(&self, index: usize) -> Result<Option<&T>>
-    where
-        T: TryFromBytes + IntoBytes + Immutable + KnownLayout,
-    {
-        let index = Self::index_to_decoded_index(index);
-
-        Ok(match self {
-            Self::Owned(a) => a.get(index),
-            Self::Ref(m) => {
-                let range = Self::index_to_byte_range(index);
-                let source = &m[range];
-                Some(T::try_ref_from_bytes(source)?)
-            }
-        })
-    }
-
-    pub fn as_arr(&self) -> &[T] {
-        match self {
-            Self::Owned(a) => a,
-            Self::Ref(_) => unreachable!(),
-        }
-    }
-
-    pub fn as_mmap(&self) -> &Mmap {
-        match self {
-            Self::Owned(_) => unreachable!(),
-            Self::Ref(m) => m,
-        }
-    }
-
-    #[inline]
-    fn index_to_byte_range(index: usize) -> Range<usize> {
-        let index = Self::index_to_byte_index(index) as usize;
-        index..(index + Self::SIZE_OF_T)
-    }
-
-    #[inline]
-    fn index_to_byte_index(index: usize) -> u64 {
-        (index * Self::SIZE_OF_T) as u64
-    }
-
-    #[inline(always)]
-    fn index_to_decoded_index(index: usize) -> usize {
-        index % Self::PER_PAGE
-    }
-}
-
-impl<T> From<Box<[T]>> for Values<T> {
-    fn from(value: Box<[T]>) -> Self {
-        Self::Owned(value)
-    }
-}
-
-impl<T> From<Mmap> for Values<T> {
-    fn from(value: Mmap) -> Self {
-        Self::Ref(Box::new(value))
-    }
-}
-
-impl<T> Default for Values<T> {
-    fn default() -> Self {
-        Self::Owned(vec![].into_boxed_slice())
-    }
-}

crates/brk_vec/src/structs/compressed_pages_meta.rs

@@ -22,8 +22,10 @@ impl CompressedPagesMetadata {
     const PAGE_SIZE: usize = size_of::<CompressedPageMetadata>();
 
     pub fn read(path: &Path) -> Result<CompressedPagesMetadata> {
+        let path = path.join("pages_meta");
+
         let slf = Self {
-            vec: fs::read(path)
+            vec: fs::read(&path)
                 .unwrap_or_default()
                 .chunks(Self::PAGE_SIZE)
                 .map(|bytes| {

crates/brk_vec/src/traits/any.rs

@@ -1,61 +0,0 @@
-use std::{io, path::PathBuf};
-
-use crate::{Result, StorableVec};
-
-use super::{StoredIndex, StoredType};
-
-pub trait AnyStorableVec: Send + Sync {
-    fn file_name(&self) -> String;
-    fn index_type_to_string(&self) -> &str;
-    fn len(&self) -> usize;
-    fn is_empty(&self) -> bool;
-    fn collect_range_values(
-        &self,
-        from: Option<i64>,
-        to: Option<i64>,
-    ) -> Result<Vec<serde_json::Value>>;
-    fn flush(&mut self) -> io::Result<()>;
-    fn path_vec(&self) -> PathBuf;
-}
-
-impl<I, T> AnyStorableVec for StorableVec<I, T>
-where
-    I: StoredIndex,
-    T: StoredType,
-{
-    fn file_name(&self) -> String {
-        self.file_name()
-    }
-
-    fn index_type_to_string(&self) -> &str {
-        self.index_type_to_string()
-    }
-
-    fn len(&self) -> usize {
-        self.len()
-    }
-
-    fn is_empty(&self) -> bool {
-        self.is_empty()
-    }
-
-    fn flush(&mut self) -> io::Result<()> {
-        self.flush()
-    }
-
-    fn collect_range_values(
-        &self,
-        from: Option<i64>,
-        to: Option<i64>,
-    ) -> Result<Vec<serde_json::Value>> {
-        Ok(self
-            .collect_range(from, to)?
-            .into_iter()
-            .map(|v| serde_json::to_value(v).unwrap())
-            .collect::<Vec<_>>())
-    }
-
-    fn path_vec(&self) -> PathBuf {
-        self.base().path_vec()
-    }
-}

crates/brk_vec/src/traits/dynamic.rs

@@ -0,0 +1,118 @@
+use std::{path::Path, sync::Arc};
+
+use arc_swap::{ArcSwap, Guard};
+use memmap2::Mmap;
+
+use crate::{Error, Result, Value};
+
+use super::{StoredIndex, StoredType};
+
+pub trait DynamicVec: Send + Sync {
+    type I: StoredIndex;
+    type T: StoredType;
+
+    #[inline]
+    fn get(&self, index: Self::I) -> Result<Option<Value<Self::T>>> {
+        self.get_(index.to_usize()?)
+    }
+    #[inline]
+    fn cached_get(&mut self, index: Self::I) -> Result<Option<Value<Self::T>>> {
+        self.get_(index.to_usize()?)
+    }
+    #[inline]
+    fn get_(&self, index: usize) -> Result<Option<Value<Self::T>>> {
+        match self.index_to_pushed_index(index) {
+            Ok(index) => {
+                if let Some(index) = index {
+                    return Ok(self.pushed().get(index).map(Value::Ref));
+                }
+            }
+            Err(Error::IndexTooHigh) => return Ok(None),
+            Err(Error::IndexTooLow) => {}
+            Err(error) => return Err(error),
+        }
+
+        Ok(self
+            .get_stored_(index.to_usize()?, self.guard().as_ref().unwrap())?
+            .map(Value::Owned))
+    }
+    fn get_stored_(&self, index: usize, mmap: &Mmap) -> Result<Option<Self::T>>;
+    fn get_last(&self) -> Result<Option<Value<Self::T>>> {
+        let len = self.len();
+        if len == 0 {
+            return Ok(None);
+        }
+        self.get_(len - 1)
+    }
+    #[inline]
+    fn cached_get_(&mut self, index: usize) -> Result<Option<Value<Self::T>>> {
+        match self.index_to_pushed_index(index) {
+            Ok(index) => {
+                if let Some(index) = index {
+                    return Ok(self.pushed().get(index).map(Value::Ref));
+                }
+            }
+            Err(Error::IndexTooHigh) => return Ok(None),
+            Err(Error::IndexTooLow) => {}
+            Err(error) => return Err(error),
+        }
+
+        let mmap = Arc::clone(self.guard().as_ref().unwrap());
+
+        Ok(self
+            .cached_get_stored_(index.to_usize()?, &mmap)?
+            .map(Value::Owned))
+    }
+    fn cached_get_stored_(&mut self, index: usize, mmap: &Mmap) -> Result<Option<Self::T>>;
+    fn cached_get_last(&mut self) -> Result<Option<Value<Self::T>>> {
+        let len = self.len();
+        if len == 0 {
+            return Ok(None);
+        }
+        self.cached_get_(len - 1)
+    }
+
+    #[inline]
+    fn len(&self) -> usize {
+        self.stored_len() + self.pushed_len()
+    }
+    #[inline]
+    fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    fn mmap(&self) -> &ArcSwap<Mmap>;
+
+    fn guard(&self) -> &Option<Guard<Arc<Mmap>>>;
+    fn mut_guard(&mut self) -> &mut Option<Guard<Arc<Mmap>>>;
+
+    fn stored_len(&self) -> usize;
+
+    fn pushed(&self) -> &[Self::T];
+    #[inline]
+    fn pushed_len(&self) -> usize {
+        self.pushed().len()
+    }
+    fn mut_pushed(&mut self) -> &mut Vec<Self::T>;
+    #[inline]
+    fn push(&mut self, value: Self::T) {
+        self.mut_pushed().push(value)
+    }
+    #[inline]
+    fn index_to_pushed_index(&self, index: usize) -> Result<Option<usize>> {
+        let stored_len = self.stored_len();
+
+        if index >= stored_len {
+            let index = index - stored_len;
+            if index >= self.pushed_len() {
+                Err(Error::IndexTooHigh)
+            } else {
+                Ok(Some(index))
+            }
+        } else {
+            Err(Error::IndexTooLow)
+        }
+    }
+
+    fn path(&self) -> &Path;
+}

crates/brk_vec/src/traits/generic.rs

@@ -0,0 +1,206 @@
+use std::{
+    fs::{File, OpenOptions},
+    io::{self, Seek, SeekFrom, Write},
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+use axum::{
+    Json,
+    response::{IntoResponse, Response},
+};
+use memmap2::Mmap;
+use serde_json::Value;
+
+use crate::{Error, Result, Version};
+
+use super::{DynamicVec, StoredIndex, StoredType};
+
+pub trait GenericVec<I, T>: DynamicVec<I = I, T = T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    const SIZE_OF_T: usize = size_of::<Self::T>();
+
+    fn open_file(&self) -> io::Result<File> {
+        Self::open_file_(&self.path_vec())
+    }
+    fn open_file_(path: &Path) -> io::Result<File> {
+        OpenOptions::new()
+            .read(true)
+            .create(true)
+            .truncate(false)
+            .append(true)
+            .open(path)
+    }
+
+    fn file_set_len(&mut self, len: u64) -> Result<()> {
+        let mut file = self.open_file()?;
+        Self::file_set_len_(&mut file, len)?;
+        self.update_mmap(file)
+    }
+    fn file_set_len_(file: &mut File, len: u64) -> Result<()> {
+        file.set_len(len)?;
+        file.seek(SeekFrom::End(0))?;
+        Ok(())
+    }
+
+    fn file_write_all(&mut self, buf: &[u8]) -> Result<()> {
+        let mut file = self.open_file()?;
+        file.write_all(buf)?;
+        self.update_mmap(file)
+    }
+
+    fn file_truncate_and_write_all(&mut self, len: u64, buf: &[u8]) -> Result<()> {
+        let mut file = self.open_file()?;
+        Self::file_set_len_(&mut file, len)?;
+        file.write_all(buf)?;
+        self.update_mmap(file)
+    }
+
+    #[inline]
+    fn reset(&mut self) -> Result<()> {
+        self.file_truncate_and_write_all(0, &[])
+    }
+
+    fn new_mmap(file: File) -> Result<Arc<Mmap>> {
+        Ok(Arc::new(unsafe { Mmap::map(&file)? }))
+    }
+
+    fn update_mmap(&mut self, file: File) -> Result<()> {
+        let mmap = Self::new_mmap(file)?;
+        self.mmap().store(mmap);
+        if self.guard().is_some() {
+            let guard = self.mmap().load();
+            self.mut_guard().replace(guard);
+        } else {
+            unreachable!("This function shouldn't be called in a cloned instance")
+        }
+        Ok(())
+    }
+
+    #[inline]
+    fn is_pushed_empty(&self) -> bool {
+        self.pushed_len() == 0
+    }
+
+    #[inline]
+    fn has(&self, index: Self::I) -> Result<bool> {
+        Ok(self.has_(index.to_usize()?))
+    }
+    #[inline]
+    fn has_(&self, index: usize) -> bool {
+        index < self.len()
+    }
+
+    #[inline]
+    fn index_type_to_string(&self) -> &str {
+        Self::I::to_string()
+    }
+
+    #[inline]
+    fn iter<F>(&mut self, f: F) -> Result<()>
+    where
+        F: FnMut(
+            (
+                Self::I,
+                Self::T,
+                &mut dyn DynamicVec<I = Self::I, T = Self::T>,
+            ),
+        ) -> Result<()>,
+    {
+        self.iter_from(Self::I::default(), f)
+    }
+
+    fn iter_from<F>(&mut self, index: Self::I, f: F) -> Result<()>
+    where
+        F: FnMut(
+            (
+                Self::I,
+                Self::T,
+                &mut dyn DynamicVec<I = Self::I, T = Self::T>,
+            ),
+        ) -> Result<()>;
+
+    fn flush(&mut self) -> Result<()>;
+
+    fn truncate_if_needed(&mut self, index: Self::I) -> Result<()>;
+
+    fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<Self::T>>;
+
+    #[inline]
+    fn collect_inclusive_range(&self, from: I, to: I) -> Result<Vec<Self::T>> {
+        self.collect_range(Some(from.to_usize()?), Some(to.to_usize()? + 1))
+    }
+
+    #[inline]
+    fn i64_to_usize(i: i64, len: usize) -> usize {
+        if i >= 0 {
+            i as usize
+        } else {
+            let v = len as i64 + i;
+            if v < 0 { 0 } else { v as usize }
+        }
+    }
+
+    fn collect_signed_range(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<Self::T>> {
+        let len = self.len();
+        let from = from.map(|i| Self::i64_to_usize(i, len));
+        let to = to.map(|i| Self::i64_to_usize(i, len));
+        self.collect_range(from, to)
+    }
+
+    #[inline]
+    fn collect_range_axum_json(
+        &self,
+        from: Option<i64>,
+        to: Option<i64>,
+    ) -> Result<Json<Vec<Self::T>>> {
+        Ok(Json(self.collect_signed_range(from, to)?))
+    }
+
+    #[inline]
+    fn collect_range_serde_json(&self, from: Option<i64>, to: Option<i64>) -> Result<Vec<Value>> {
+        self.collect_signed_range(from, to)?
+            .into_iter()
+            .map(|v| serde_json::to_value(v).map_err(Error::from))
+            .collect::<Result<Vec<_>>>()
+    }
+
+    #[inline]
+    fn collect_range_response(&self, from: Option<i64>, to: Option<i64>) -> Result<Response> {
+        Ok(self.collect_range_axum_json(from, to)?.into_response())
+    }
+
+    #[inline]
+    fn path_vec(&self) -> PathBuf {
+        Self::path_vec_(self.path())
+    }
+    #[inline]
+    fn path_vec_(path: &Path) -> PathBuf {
+        path.join("vec")
+    }
+
+    #[inline]
+    fn path_version_(path: &Path) -> PathBuf {
+        path.join("version")
+    }
+
+    #[inline]
+    fn path_compressed_(path: &Path) -> PathBuf {
+        path.join("compressed")
+    }
+
+    #[inline]
+    fn file_name(&self) -> String {
+        self.path()
+            .file_name()
+            .unwrap()
+            .to_str()
+            .unwrap()
+            .to_owned()
+    }
+
+    fn version(&self) -> Version;
+}

crates/brk_vec/src/traits/mod.rs

@@ -1,8 +1,9 @@
-mod any;
+mod dynamic;
-// mod bytes;
+mod generic;
 mod stored_index;
 mod stored_type;
 
-pub use any::*;
+pub use dynamic::*;
+pub use generic::*;
 pub use stored_index::*;
 pub use stored_type::*;

crates/brk_vec/src/traits/stored_type.rs

@@ -5,10 +5,28 @@ use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes};
 
 pub trait StoredType
 where
-    Self: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout + Send + Sync + Serialize,
+    Self: Sized
+        + Debug
+        + Clone
+        + TryFromBytes
+        + IntoBytes
+        + Immutable
+        + KnownLayout
+        + Send
+        + Sync
+        + Serialize,
 {
 }
 impl<T> StoredType for T where
-    T: Sized + Debug + Clone + TryFromBytes + IntoBytes + Immutable + KnownLayout + Send + Sync + Serialize
+    T: Sized
+        + Debug
+        + Clone
+        + TryFromBytes
+        + IntoBytes
+        + Immutable
+        + KnownLayout
+        + Send
+        + Sync
+        + Serialize
 {
 }

crates/brk_vec/src/variants/compressed.rs

@@ -0,0 +1,526 @@
+use std::{
+    fs::{self, File},
+    mem,
+    path::Path,
+    sync::{Arc, OnceLock},
+};
+
+use arc_swap::{ArcSwap, Guard};
+use memmap2::Mmap;
+use rayon::prelude::*;
+use zstd::DEFAULT_COMPRESSION_LEVEL;
+
+use crate::{
+    CompressedPageMetadata, CompressedPagesMetadata, DynamicVec, Error, GenericVec, RawVec, Result,
+    StoredIndex, StoredType, UnsafeSlice, Version,
+};
+
+const ONE_KIB: usize = 1024;
+const ONE_MIB: usize = ONE_KIB * ONE_KIB;
+pub const MAX_CACHE_SIZE: usize = 100 * ONE_MIB;
+pub const MAX_PAGE_SIZE: usize = 16 * ONE_KIB;
+
+#[derive(Debug)]
+pub struct CompressedVec<I, T> {
+    inner: RawVec<I, T>,
+    decoded_page: Option<(usize, Vec<T>)>,
+    decoded_pages: Option<Vec<OnceLock<Vec<T>>>>,
+    pages_meta: Arc<ArcSwap<CompressedPagesMetadata>>,
+    // pages: Option<Vec<OnceLock<Values<T>>>>,
+    // page: Option<(usize, Values<T>)>,
+    // length: Length
+}
+
+impl<I, T> CompressedVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    pub const PER_PAGE: usize = MAX_PAGE_SIZE / Self::SIZE_OF_T;
+    pub const PAGE_SIZE: usize = Self::PER_PAGE * Self::SIZE_OF_T;
+    pub const CACHE_LENGTH: usize = MAX_CACHE_SIZE / Self::PAGE_SIZE;
+
+    /// Same as import but will reset the folder under certain errors, so be careful !
+    pub fn forced_import(path: &Path, version: Version) -> Result<Self> {
+        let res = Self::import(path, version);
+        match res {
+            Err(Error::WrongEndian)
+            | Err(Error::DifferentVersion { .. })
+            | Err(Error::DifferentCompressionMode) => {
+                fs::remove_dir_all(path)?;
+                Self::import(path, version)
+            }
+            _ => res,
+        }
+    }
+
+    pub fn import(path: &Path, version: Version) -> Result<Self> {
+        fs::create_dir_all(path)?;
+
+        let vec_exists = fs::exists(Self::path_vec_(path)).is_ok_and(|b| b);
+        let compressed_path = Self::path_compressed_(path);
+        let compressed_exists = fs::exists(&compressed_path).is_ok_and(|b| b);
+
+        if vec_exists && !compressed_exists {
+            return Err(Error::DifferentCompressionMode);
+        }
+
+        if !vec_exists && !compressed_exists {
+            File::create(&compressed_path)?;
+        }
+
+        Ok(Self {
+            inner: RawVec::import(path, version)?,
+            decoded_page: None,
+            decoded_pages: None,
+            pages_meta: Arc::new(ArcSwap::new(Arc::new(CompressedPagesMetadata::read(path)?))),
+        })
+    }
+
+    fn cached_get_stored__(
+        index: usize,
+        mmap: &Mmap,
+        stored_len: usize,
+        decoded_page: &mut Option<(usize, Vec<T>)>,
+        compressed_pages_meta: &CompressedPagesMetadata,
+    ) -> Result<Option<T>> {
+        let page_index = Self::index_to_page_index(index);
+
+        if decoded_page.as_ref().is_none_or(|b| b.0 != page_index) {
+            let values = Self::decode_page_(stored_len, page_index, mmap, compressed_pages_meta)?;
+            decoded_page.replace((page_index, values));
+        }
+
+        Ok(decoded_page
+            .as_ref()
+            .unwrap()
+            .1
+            .get(index % Self::PER_PAGE)
+            .cloned())
+    }
+
+    fn decode_page(&self, page_index: usize, mmap: &Mmap) -> Result<Vec<T>> {
+        Self::decode_page_(self.stored_len(), page_index, mmap, &self.pages_meta.load())
+    }
+
+    fn decode_page_(
+        stored_len: usize,
+        page_index: usize,
+        mmap: &Mmap,
+        compressed_pages_meta: &CompressedPagesMetadata,
+    ) -> Result<Vec<T>> {
+        if Self::page_index_to_index(page_index) >= stored_len {
+            return Err(Error::IndexTooHigh);
+        } else if compressed_pages_meta.len() <= page_index {
+            return Err(Error::ExpectVecToHaveIndex);
+        }
+
+        let page = compressed_pages_meta.get(page_index).unwrap();
+        let len = page.bytes_len as usize;
+        let offset = page.start as usize;
+
+        Ok(zstd::decode_all(&mmap[offset..offset + len])
+            .inspect_err(|_| {
+                dbg!((len, offset, page_index, &mmap[..], &mmap.len()));
+            })?
+            .chunks(Self::SIZE_OF_T)
+            .map(|slice| T::try_read_from_bytes(slice).unwrap())
+            .collect::<Vec<_>>())
+    }
+
+    fn compress_page(chunk: &[T]) -> Vec<u8> {
+        if chunk.len() > Self::PER_PAGE {
+            panic!();
+        }
+
+        let mut bytes: Vec<u8> = vec![0; chunk.len() * Self::SIZE_OF_T];
+
+        let unsafe_bytes = UnsafeSlice::new(&mut bytes);
+
+        chunk
+            .into_par_iter()
+            .enumerate()
+            .for_each(|(i, v)| unsafe_bytes.copy_slice(i * Self::SIZE_OF_T, v.as_bytes()));
+
+        zstd::encode_all(bytes.as_slice(), DEFAULT_COMPRESSION_LEVEL).unwrap()
+    }
+
+    pub fn enable_large_cache(&mut self) {
+        self.decoded_pages.replace(vec![]);
+        self.reset_large_cache();
+    }
+
+    pub fn disable_large_cache(&mut self) {
+        self.decoded_pages.take();
+    }
+
+    fn reset_large_cache(&mut self) {
+        let stored_len = self.stored_len();
+
+        if let Some(pages) = self.decoded_pages.as_mut() {
+            pages.par_iter_mut().for_each(|lock| {
+                lock.take();
+            });
+
+            let len = (stored_len as f64 / Self::PER_PAGE as f64).ceil() as usize;
+            let len = Self::CACHE_LENGTH.min(len);
+
+            if pages.len() != len {
+                pages.resize_with(len, Default::default);
+            }
+        }
+    }
+
+    pub fn large_cache_len(&self) -> usize {
+        self.decoded_pages.as_ref().map_or(0, |v| v.len())
+    }
+
+    fn reset_small_cache(&mut self) {
+        self.decoded_page.take();
+    }
+
+    fn reset_caches(&mut self) {
+        self.reset_small_cache();
+        self.reset_large_cache();
+    }
+
+    #[inline(always)]
+    fn index_to_page_index(index: usize) -> usize {
+        index / Self::PER_PAGE
+    }
+
+    #[inline(always)]
+    fn page_index_to_index(page_index: usize) -> usize {
+        page_index * Self::PER_PAGE
+    }
+}
+
+impl<I, T> DynamicVec for CompressedVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    type I = I;
+    type T = T;
+
+    #[inline]
+    fn get_stored_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
+        let cached_start = self
+            .stored_len()
+            .checked_sub(Self::CACHE_LENGTH)
+            .unwrap_or_default();
+
+        let decoded_index = index % Self::PER_PAGE;
+
+        if index >= cached_start {
+            let trimmed_index = index - cached_start;
+            if let Some(decoded_pages) = self.decoded_pages.as_ref() {
+                let decoded_page = decoded_pages
+                    .get(Self::index_to_page_index(trimmed_index))
+                    .unwrap();
+
+                return Ok(decoded_page
+                    .get_or_init(|| {
+                        self.decode_page(Self::index_to_page_index(index), mmap)
+                            .unwrap()
+                    })
+                    .get(decoded_index)
+                    .cloned());
+            }
+        }
+
+        let page_index = Self::index_to_page_index(index);
+
+        Ok(self
+            .decode_page(page_index, mmap)?
+            .get(decoded_index)
+            .cloned())
+    }
+    #[inline]
+    fn cached_get_stored_(&mut self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
+        Self::cached_get_stored__(
+            index,
+            mmap,
+            self.stored_len(),
+            &mut self.decoded_page,
+            &self.pages_meta.load(),
+        )
+    }
+
+    #[inline]
+    fn mmap(&self) -> &ArcSwap<Mmap> {
+        self.inner.mmap()
+    }
+
+    #[inline]
+    fn guard(&self) -> &Option<Guard<Arc<Mmap>>> {
+        self.inner.guard()
+    }
+    #[inline]
+    fn mut_guard(&mut self) -> &mut Option<Guard<Arc<Mmap>>> {
+        self.inner.mut_guard()
+    }
+
+    fn stored_len(&self) -> usize {
+        let pages_meta = self.pages_meta.load();
+        if let Some(last) = pages_meta.last() {
+            (pages_meta.len() - 1) * Self::PER_PAGE + last.values_len as usize
+        } else {
+            0
+        }
+    }
+
+    #[inline]
+    fn pushed(&self) -> &[T] {
+        self.inner.pushed()
+    }
+    #[inline]
+    fn mut_pushed(&mut self) -> &mut Vec<T> {
+        self.inner.mut_pushed()
+    }
+
+    #[inline]
+    fn path(&self) -> &Path {
+        self.inner.path()
+    }
+}
+
+impl<I, T> GenericVec<I, T> for CompressedVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    fn iter_from<F>(&mut self, index: I, mut f: F) -> Result<()>
+    where
+        F: FnMut((I, T, &mut dyn DynamicVec<I = Self::I, T = Self::T>)) -> Result<()>,
+    {
+        if !self.is_pushed_empty() {
+            return Err(Error::UnsupportedUnflushedState);
+        }
+
+        let start = index.to_usize()?;
+
+        let stored_len = self.stored_len();
+        if start >= stored_len {
+            return Ok(());
+        }
+
+        let mmap = self.mmap().load();
+        let pages_meta = self.pages_meta.load();
+
+        (start..stored_len).try_for_each(|index| {
+            let v = Self::cached_get_stored__(
+                index,
+                &mmap,
+                stored_len,
+                &mut self.decoded_page,
+                &pages_meta,
+            )?
+            .unwrap();
+            f((I::from(index), v, self as &mut dyn DynamicVec<I = I, T = T>))
+        })
+    }
+
+    fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<Self::T>> {
+        if !self.is_pushed_empty() {
+            return Err(Error::UnsupportedUnflushedState);
+        }
+
+        let stored_len = self.stored_len();
+
+        let from = from.unwrap_or_default();
+        let to = to.map_or(stored_len, |i| i.min(stored_len));
+
+        if from >= stored_len {
+            return Ok(vec![]);
+        }
+
+        let mmap = self.mmap().load();
+        let pages_meta = self.pages_meta.load();
+        let mut decoded_page: Option<(usize, Vec<T>)> = None;
+
+        (from..to)
+            .map(|index| {
+                Self::cached_get_stored__(index, &mmap, stored_len, &mut decoded_page, &pages_meta)
+                    .map(|opt| opt.unwrap())
+            })
+            .collect::<Result<Vec<_>>>()
+    }
+
+    fn flush(&mut self) -> Result<()> {
+        let pushed_len = self.pushed_len();
+
+        if pushed_len == 0 {
+            return Ok(());
+        }
+
+        let stored_len = self.stored_len();
+
+        let mut pages_meta = (**self.pages_meta.load()).clone();
+
+        let mut starting_page_index = pages_meta.len();
+        let mut values = vec![];
+        let mut truncate_at = None;
+
+        if self.stored_len() % Self::PER_PAGE != 0 {
+            if pages_meta.is_empty() {
+                unreachable!()
+            }
+
+            let last_page_index = pages_meta.len() - 1;
+
+            values = if let Some(values) = self
+                .decoded_pages
+                .as_mut()
+                .and_then(|v| v.last_mut().and_then(|lock| lock.take()))
+            {
+                values
+            } else if self
+                .decoded_page
+                .as_ref()
+                .is_some_and(|(page_index, _)| *page_index == last_page_index)
+            {
+                self.decoded_page.take().unwrap().1
+            } else {
+                Self::decode_page_(
+                    stored_len,
+                    last_page_index,
+                    self.guard().as_ref().unwrap(),
+                    &pages_meta,
+                )
+                .inspect_err(|_| {
+                    dbg!(last_page_index, &pages_meta);
+                })
+                .unwrap()
+            };
+
+            truncate_at.replace(pages_meta.pop().unwrap().start);
+            starting_page_index = last_page_index;
+        }
+
+        let compressed = values
+            .into_par_iter()
+            .chain(mem::take(self.mut_pushed()).into_par_iter())
+            .chunks(Self::PER_PAGE)
+            .map(|chunk| (Self::compress_page(chunk.as_ref()), chunk.len()))
+            .collect::<Vec<_>>();
+
+        compressed
+            .iter()
+            .enumerate()
+            .for_each(|(i, (compressed_bytes, values_len))| {
+                let page_index = starting_page_index + i;
+
+                let start = if page_index != 0 {
+                    let prev = pages_meta.get(page_index - 1).unwrap();
+                    prev.start + prev.bytes_len as u64
+                } else {
+                    0
+                };
+
+                let bytes_len = compressed_bytes.len() as u32;
+                let values_len = *values_len as u32;
+
+                let page = CompressedPageMetadata::new(start, bytes_len, values_len);
+
+                pages_meta.push(page_index, page);
+            });
+
+        let buf = compressed
+            .into_iter()
+            .flat_map(|(v, _)| v)
+            .collect::<Vec<_>>();
+
+        pages_meta.write()?;
+
+        if let Some(truncate_at) = truncate_at {
+            self.file_set_len(truncate_at)?;
+        }
+
+        self.file_write_all(&buf)?;
+
+        self.pages_meta.store(Arc::new(pages_meta));
+
+        self.reset_caches();
+
+        Ok(())
+    }
+
+    fn reset(&mut self) -> Result<()> {
+        let mut pages_meta = (**self.pages_meta.load()).clone();
+        pages_meta.truncate(0);
+        pages_meta.write()?;
+        self.pages_meta.store(Arc::new(pages_meta));
+        self.reset_caches();
+        self.file_truncate_and_write_all(0, &[])
+    }
+
+    fn truncate_if_needed(&mut self, index: I) -> Result<()> {
+        let index = index.to_usize()?;
+
+        if index >= self.stored_len() {
+            return Ok(());
+        }
+
+        if index == 0 {
+            self.reset()?;
+            return Ok(());
+        }
+
+        let mut pages_meta = (**self.pages_meta.load()).clone();
+
+        let page_index = Self::index_to_page_index(index);
+
+        let guard = self.guard().as_ref().unwrap();
+
+        let values = self.decode_page(page_index, guard)?;
+        let mut buf = vec![];
+
+        let mut page = pages_meta.truncate(page_index).unwrap();
+
+        let len = page.start;
+
+        let decoded_index = index % Self::PER_PAGE;
+
+        if decoded_index != 0 {
+            let chunk = &values[..decoded_index];
+
+            buf = Self::compress_page(chunk);
+
+            page.values_len = chunk.len() as u32;
+            page.bytes_len = buf.len() as u32;
+
+            pages_meta.push(page_index, page);
+        }
+
+        pages_meta.write()?;
+
+        self.pages_meta.store(Arc::new(pages_meta));
+
+        self.file_truncate_and_write_all(len, &buf)?;
+
+        self.reset_caches();
+
+        Ok(())
+    }
+
+    #[inline]
+    fn version(&self) -> Version {
+        self.inner.version()
+    }
+}
+
+impl<I, T> Clone for CompressedVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+            decoded_page: None,
+            decoded_pages: None,
+            pages_meta: self.pages_meta.clone(),
+        }
+    }
+}

crates/brk_vec/src/variants/mod.rs

@@ -0,0 +1,5 @@
+mod compressed;
+mod raw;
+
+pub use compressed::*;
+pub use raw::*;

crates/brk_vec/src/variants/raw.rs

@@ -0,0 +1,241 @@
+use std::{
+    fs,
+    marker::PhantomData,
+    mem,
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+use arc_swap::{ArcSwap, Guard};
+use memmap2::Mmap;
+use rayon::prelude::*;
+
+use crate::{DynamicVec, Error, GenericVec, Result, StoredIndex, StoredType, UnsafeSlice, Version};
+
+#[derive(Debug)]
+pub struct RawVec<I, T> {
+    version: Version,
+    pathbuf: PathBuf,
+    // Consider  Arc<ArcSwap<Option<Mmap>>> for dataraces when reorg ?
+    mmap: Arc<ArcSwap<Mmap>>,
+    guard: Option<Guard<Arc<Mmap>>>,
+    pushed: Vec<T>,
+    phantom: PhantomData<I>,
+}
+
+impl<I, T> RawVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    /// Same as import but will reset the folder under certain errors, so be careful !
+    pub fn forced_import(path: &Path, version: Version) -> Result<Self> {
+        let res = Self::import(path, version);
+        match res {
+            Err(Error::WrongEndian) | Err(Error::DifferentVersion { .. }) => {
+                fs::remove_dir_all(path)?;
+                Self::import(path, version)
+            }
+            _ => res,
+        }
+    }
+
+    pub fn import(path: &Path, version: Version) -> Result<Self> {
+        fs::create_dir_all(path)?;
+
+        let version_path = Self::path_version_(path);
+        version.validate(version_path.as_ref())?;
+        version.write(version_path.as_ref())?;
+
+        let file = Self::open_file_(Self::path_vec_(path).as_path())?;
+        let mmap = Arc::new(ArcSwap::new(Self::new_mmap(file)?));
+        let guard = Some(mmap.load());
+
+        Ok(Self {
+            mmap,
+            guard,
+            version,
+            pathbuf: path.to_owned(),
+            pushed: vec![],
+            phantom: PhantomData,
+        })
+    }
+}
+
+impl<I, T> DynamicVec for RawVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    type I = I;
+    type T = T;
+
+    #[inline]
+    fn get_stored_(&self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
+        let index = index * Self::SIZE_OF_T;
+        let slice = &mmap[index..(index + Self::SIZE_OF_T)];
+        Self::T::try_read_from_bytes(slice)
+            .map(|v| Some(v))
+            .map_err(Error::from)
+    }
+    #[inline]
+    fn cached_get_stored_(&mut self, index: usize, mmap: &Mmap) -> Result<Option<T>> {
+        self.get_stored_(index, mmap)
+    }
+
+    #[inline]
+    fn mmap(&self) -> &ArcSwap<Mmap> {
+        &self.mmap
+    }
+
+    #[inline]
+    fn guard(&self) -> &Option<Guard<Arc<Mmap>>> {
+        &self.guard
+    }
+    #[inline]
+    fn mut_guard(&mut self) -> &mut Option<Guard<Arc<Mmap>>> {
+        &mut self.guard
+    }
+
+    #[inline]
+    fn stored_len(&self) -> usize {
+        if let Some(guard) = self.guard() {
+            guard.len() / Self::SIZE_OF_T
+        } else {
+            self.mmap.load().len() / Self::SIZE_OF_T
+        }
+    }
+
+    #[inline]
+    fn pushed(&self) -> &[T] {
+        self.pushed.as_slice()
+    }
+    #[inline]
+    fn mut_pushed(&mut self) -> &mut Vec<T> {
+        &mut self.pushed
+    }
+
+    #[inline]
+    fn path(&self) -> &Path {
+        self.pathbuf.as_path()
+    }
+}
+
+impl<I, T> GenericVec<I, T> for RawVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    fn iter_from<F>(&mut self, index: I, mut f: F) -> Result<()>
+    where
+        F: FnMut((I, T, &mut dyn DynamicVec<I = Self::I, T = Self::T>)) -> Result<()>,
+    {
+        if !self.is_pushed_empty() {
+            return Err(Error::UnsupportedUnflushedState);
+        }
+
+        let start = index.to_usize()?;
+
+        let stored_len = self.stored_len();
+        if start >= stored_len {
+            return Ok(());
+        }
+
+        let guard = self.mmap.load();
+
+        (start..stored_len).try_for_each(|index| {
+            let v = self.get_stored_(index, &guard)?.unwrap();
+            f((I::from(index), v, self as &mut dyn DynamicVec<I = I, T = T>))
+        })
+    }
+
+    fn collect_range(&self, from: Option<usize>, to: Option<usize>) -> Result<Vec<T>> {
+        if !self.is_pushed_empty() {
+            return Err(Error::UnsupportedUnflushedState);
+        }
+
+        let stored_len = self.stored_len();
+
+        let from = from.unwrap_or_default();
+        let to = to.map_or(stored_len, |i| i.min(stored_len));
+
+        if from >= stored_len {
+            return Ok(vec![]);
+        }
+
+        let mmap = self.mmap.load();
+
+        (from..to)
+            .map(|index| self.get_stored_(index, &mmap).map(|opt| opt.unwrap()))
+            .collect::<Result<Vec<_>>>()
+    }
+
+    fn flush(&mut self) -> Result<()> {
+        let pushed_len = self.pushed_len();
+
+        if pushed_len == 0 {
+            return Ok(());
+        }
+
+        let bytes = {
+            let pushed = &mut self.pushed;
+
+            let mut bytes: Vec<u8> = vec![0; pushed.len() * Self::SIZE_OF_T];
+
+            let unsafe_bytes = UnsafeSlice::new(&mut bytes);
+
+            mem::take(pushed)
+                .into_par_iter()
+                .enumerate()
+                .for_each(|(i, v)| unsafe_bytes.copy_slice(i * Self::SIZE_OF_T, v.as_bytes()));
+
+            bytes
+        };
+
+        self.file_write_all(&bytes)?;
+
+        Ok(())
+    }
+
+    fn truncate_if_needed(&mut self, index: I) -> Result<()> {
+        let index = index.to_usize()?;
+
+        if index >= self.stored_len() {
+            return Ok(());
+        }
+
+        if index == 0 {
+            self.reset()?;
+            return Ok(());
+        }
+
+        let len = index * Self::SIZE_OF_T;
+
+        self.file_set_len(len as u64)?;
+
+        Ok(())
+    }
+
+    #[inline]
+    fn version(&self) -> Version {
+        self.version
+    }
+}
+
+impl<I, T> Clone for RawVec<I, T>
+where
+    I: StoredIndex,
+    T: StoredType,
+{
+    fn clone(&self) -> Self {
+        Self {
+            version: self.version,
+            pathbuf: self.pathbuf.clone(),
+            // Consider  Arc<ArcSwap<Option<Mmap>>> for dataraces when reorg ?
+            mmap: self.mmap.clone(),
+            guard: None,
+            pushed: vec![],
+            phantom: PhantomData,
+        }
+    }
+}

publish.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-cargo clean
-
-cargo build --all-targets
-
-cd crates/brk
-
-cd ../brk_core
-cargo publish
-
-cd ../brk_exit
-cargo publish
-
-cd ../brk_vec
-cargo publish
-
-cd ../brk_logger
-cargo publish
-
-cd ../brk_indexer
-cargo publish
-
-cd ../brk_parser
-cargo publish
-
-cd ../brk_fetcher
-cargo publish
-
-cd ../brk_computer
-cargo publish
-
-cd ../brk_query
-cargo publish
-
-cd ../brk_server
-cargo publish
-
-cd ../brk_cli
-cargo publish
-
-cd ../brk
-cargo publish
-
-cd ../..

update.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+cargo clean
 rustup update
 cargo upgrade --incompatible
 cargo update
+cargo build --all-targets

websites/kibo.money/index.html

@@ -877,9 +877,10 @@
         background-image: linear-gradient(
           to bottom,
           transparent,
+          var(--background-color) 90%,
           var(--background-color)
         );
-        height: 24rem;
+        height: 21rem;
         bottom: 0;
         left: 0;
         right: 0;

websites/kibo.money/packages/lightweight-charts/wrapper.js

@@ -57,9 +57,6 @@ export default import("./v5.0.5-treeshaked/script.js").then((lc) => {
         vertLines: { visible: false },
         horzLines: { visible: false },
       },
-      timeScale: {
-        minBarSpacing: 2.1,
-      },
       localization: {
         priceFormatter: utils.locale.numberToShortUSFormat,
         locale: "en-us",
@@ -942,7 +939,7 @@ function createPriceScaleSelectorIfNeeded({
       /** @typedef {(typeof choices)[number]} Choices */
       const serializedValue = signals.createSignal(
         /** @satisfies {Choices} */ (
-          unit === "US Dollars" && seriesType !== "Baseline" ? "log" : "lin"
+          unit === "USD" && seriesType !== "Baseline" ? "log" : "lin"
         ),
         {
           save: {

websites/kibo.money/scripts/chart.js

@@ -87,7 +87,7 @@ export function init({
       const candles = chart.addCandlestickSeries({
         vecId: "ohlc",
         name: "Price",
-        unit: "US Dollars",
+        unit: "USD",
       });
       signals.createEffect(webSockets.kraken1dCandle.latest, (latest) => {
         if (!latest) return;
@@ -103,7 +103,10 @@ export function init({
         { blueprints: option.bottom, paneIndex: 1 },
       ].forEach(({ blueprints, paneIndex }) => {
         blueprints?.forEach((blueprint) => {
-          if (vecIdToIndexes[blueprint.key].includes(index)) {
+          const indexes = /** @type {readonly number[]} */ (
+            vecIdToIndexes[blueprint.key]
+          );
+          if (indexes.includes(index)) {
             chart.addLineSeries({
               vecId: blueprint.key,
               color: blueprint.color,
@@ -178,7 +181,7 @@ function createIndexSelector({ elements, signals, utils }) {
   elements.charts.append(fieldset);
 
   const index = signals.createMemo(
-    /** @returns {Index} */ () => {
+    /** @returns {ChartableIndex} */ () => {
       switch (serializedIndex()) {
         case "timestamp":
           return /** @satisfies {Height} */ (0);

websites/kibo.money/scripts/main.js

@@ -1,7 +1,7 @@
 // @ts-check
 
 /**
- * @import { Option, PartialChartOption, ChartOption, AnyPartialOption, ProcessedOptionAddons, OptionsTree, SimulationOption, Unit, AnySeriesBlueprint } from "./options"
+ * @import { Option, PartialChartOption, ChartOption, AnyPartialOption, ProcessedOptionAddons, OptionsTree, SimulationOption, Unit, AnySeriesBlueprint, ChartableIndex } from "./options"
  * @import {Valued,  SingleValueData, CandlestickData, ChartData, OHLCTuple} from "../packages/lightweight-charts/wrapper"
  * @import * as _ from "../packages/ufuzzy/v1.0.14/types"
  * @import { createChart as CreateClassicChart, LineStyleOptions, DeepPartial, ChartOptions, IChartApi, IHorzScaleBehavior, WhitespaceData, ISeriesApi, Time, LineData, LogicalRange, SeriesType, BaselineStyleOptions, SeriesOptionsCommon, BaselineData, CandlestickStyleOptions } from "../packages/lightweight-charts/v5.0.5-treeshaked/types"
@@ -436,7 +436,7 @@ function createUtils() {
     createInputDollar({ id, title, signal, signals }) {
       return this.createInputNumberElement({
         id,
-        placeholder: "US Dollars",
+        placeholder: "USD",
         min: 0,
         title,
         signal,
@@ -767,13 +767,13 @@ function createUtils() {
         return numberToUSFormat(value, 0);
       } else if (absoluteValue < 1_000_000) {
         return `${numberToUSFormat(value / 1_000, 1)}K`;
-      } else if (absoluteValue >= 9_000_000_000_000_000) {
+      } else if (absoluteValue >= 900_000_000_000_000_000) {
         return "Inf.";
       }
 
       const log = Math.floor(Math.log10(absoluteValue) - 6);
 
-      const suffices = ["M", "B", "T", "Q"];
+      const suffices = ["M", "B", "T", "P", "E"];
       const letterIndex = Math.floor(log / 3);
       const letter = suffices[letterIndex];
 
@@ -1216,9 +1216,10 @@ function createUtils() {
       /**
        * @param {Index} index
        * @param {VecId} vecId
+       * @param {number} from
        */
-      genUrl(index, vecId) {
+      genUrl(index, vecId, from) {
-        return `/api${genPath(index, vecId)}`;
+        return `/api${genPath(index, vecId, from)}`;
       },
       /**
        * @template {number | OHLCTuple} [T=number]
@@ -1284,8 +1285,10 @@ function createVecsResources(signals, utils) {
       let loading = false;
       let at = /** @type {Date | null} */ (null);
 
+      const from = -10_000;
+
       return {
-        url: utils.api.genUrl(index, id),
+        url: utils.api.genUrl(index, id, from),
         fetched,
         async fetch() {
           if (loading) return fetched();
@@ -1302,7 +1305,7 @@ function createVecsResources(signals, utils) {
               },
               index,
               id,
-              -10_000,
+              from,
             )
           );
           at = new Date();

websites/kibo.money/scripts/simulation.js

@@ -557,7 +557,7 @@ export function init({
     utils,
     config: [
       {
-        unit: "US Dollars",
+        unit: "USD",
         blueprints: [
           {
             title: "Bitcoin Value",
@@ -600,7 +600,7 @@ export function init({
     utils,
     config: [
       {
-        unit: "Bitcoin",
+        unit: "BTC",
         blueprints: [
           {
             title: "Bitcoin Stack",
@@ -623,7 +623,7 @@ export function init({
     utils,
     config: [
       {
-        unit: "US Dollars",
+        unit: "USD",
         blueprints: [
           {
             title: "Bitcoin Price",
@@ -652,7 +652,7 @@ export function init({
     utils,
     config: [
       {
-        unit: "US Dollars",
+        unit: "USD",
         blueprints: [
           {
             title: "Return Of Investment",

websites/kibo.money/scripts/vecid-to-indexes.js

@@ -22,8 +22,13 @@
 /** @typedef {17} Txindex */
 /** @typedef {18} Txinindex */
 /** @typedef {19} Txoutindex */
+/** @typedef {20} Emptyindex */
+/** @typedef {21} Multisigindex */
+/** @typedef {22} Opreturnindex */
+/** @typedef {23} Pushonlyindex */
+/** @typedef {24} Unknownindex */
 
-/** @typedef {Height | Dateindex | Weekindex | Difficultyepoch | Monthindex | Quarterindex | Yearindex | Decadeindex | Halvingepoch | Addressindex | P2PK33index | P2PK65index | P2PKHindex | P2SHindex | P2TRindex | P2WPKHindex | P2WSHindex | Txindex | Txinindex | Txoutindex} Index */
+/** @typedef {Height | Dateindex | Weekindex | Difficultyepoch | Monthindex | Quarterindex | Yearindex | Decadeindex | Halvingepoch | Addressindex | P2PK33index | P2PK65index | P2PKHindex | P2SHindex | P2TRindex | P2WPKHindex | P2WSHindex | Txindex | Txinindex | Txoutindex | Emptyindex | Multisigindex | Opreturnindex | Pushonlyindex | Unknownindex} Index */
 
 export function createVecIdToIndexes() {
   const Height = /** @satisfies {Height} */ (0);
@@ -46,14 +51,19 @@ export function createVecIdToIndexes() {
   const Txindex = /** @satisfies {Txindex} */ (17);
   const Txinindex = /** @satisfies {Txinindex} */ (18);
   const Txoutindex = /** @satisfies {Txoutindex} */ (19);
+  const Emptyindex = /** @satisfies {Emptyindex} */ (20);
+  const Multisigindex = /** @satisfies {Multisigindex} */ (21);
+  const Opreturnindex = /** @satisfies {Opreturnindex} */ (22);
+  const Pushonlyindex = /** @satisfies {Pushonlyindex} */ (23);
+  const Unknownindex = /** @satisfies {Unknownindex} */ (24);
 
-  return {
+  return /** @type {const} */ ({
     addressindex: [Txoutindex],
     addresstype: [Addressindex],
     addresstypeindex: [Addressindex],
     "base-size": [Txindex],
-    "block-count": [Dateindex],
+    "block-count": [Height],
-    "block-interval": [Height],
+    "block-count-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
     "block-interval-10p": [Dateindex],
     "block-interval-25p": [Dateindex],
     "block-interval-75p": [Dateindex],
@@ -62,14 +72,46 @@ export function createVecIdToIndexes() {
     "block-interval-max": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
     "block-interval-median": [Dateindex],
     "block-interval-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "block-size-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "block-vbytes-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "block-weight-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
     blockhash: [Height],
     close: [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
     "close-in-cents": [Dateindex, Height],
+    coinbase: [Height],
+    "coinbase-10p": [Dateindex],
+    "coinbase-25p": [Dateindex],
+    "coinbase-75p": [Dateindex],
+    "coinbase-90p": [Dateindex],
+    "coinbase-average": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "coinbase-max": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "coinbase-median": [Dateindex],
+    "coinbase-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "coinbase-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
     date: [Dateindex],
     dateindex: [Dateindex, Height],
     decadeindex: [Yearindex, Decadeindex],
     difficulty: [Height],
     difficultyepoch: [Height, Difficultyepoch],
+    fee: [Txindex],
+    "fee-10p": [Height],
+    "fee-25p": [Height],
+    "fee-75p": [Height],
+    "fee-90p": [Height],
+    "fee-average": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "fee-max": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "fee-median": [Height],
+    "fee-min": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "fee-sum": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    feerate: [Txindex],
+    "feerate-10p": [Height],
+    "feerate-25p": [Height],
+    "feerate-75p": [Height],
+    "feerate-90p": [Height],
+    "feerate-average": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "feerate-max": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "feerate-median": [Height],
+    "feerate-min": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
     "first-addressindex": [Height],
     "first-dateindex": [Weekindex, Monthindex],
     "first-emptyindex": [Height],
@@ -93,10 +135,23 @@ export function createVecIdToIndexes() {
     "fixed-date": [Height],
     "fixed-timestamp": [Height],
     halvingepoch: [Height, Halvingepoch],
-    height: [Addressindex, Height, Txindex],
+    height: [Addressindex, Height, P2PK33index, P2PK65index, P2PKHindex, P2SHindex, P2TRindex, P2WPKHindex, P2WSHindex, Txindex, Txinindex, Txoutindex, Emptyindex, Multisigindex, Opreturnindex, Pushonlyindex, Unknownindex],
     high: [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
     "high-in-cents": [Dateindex, Height],
-    "inputs-count": [Txindex],
+    "input-count": [Txindex],
+    "input-count-10p": [Height],
+    "input-count-25p": [Height],
+    "input-count-75p": [Height],
+    "input-count-90p": [Height],
+    "input-count-average": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "input-count-max": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "input-count-median": [Height],
+    "input-count-min": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "input-count-sum": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "input-value": [Txindex],
+    "input-value-average": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "input-value-sum": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    interval: [Height],
     "is-coinbase": [Txindex],
     "is-explicitly-rbf": [Txindex],
     "last-dateindex": [Weekindex, Monthindex],
@@ -114,7 +169,19 @@ export function createVecIdToIndexes() {
     "ohlc-in-cents": [Dateindex, Height],
     open: [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
     "open-in-cents": [Dateindex, Height],
-    "outputs-count": [Txindex],
+    "output-count": [Txindex],
+    "output-count-10p": [Height],
+    "output-count-25p": [Height],
+    "output-count-75p": [Height],
+    "output-count-90p": [Height],
+    "output-count-average": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "output-count-max": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "output-count-median": [Height],
+    "output-count-min": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "output-count-sum": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "output-value": [Txindex],
+    "output-value-average": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "output-value-sum": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
     p2pk33addressbytes: [P2PK33index],
     p2pk65addressbytes: [P2PK65index],
     p2pkhaddressbytes: [P2PKHindex],
@@ -124,19 +191,76 @@ export function createVecIdToIndexes() {
     p2wshaddressbytes: [P2WSHindex],
     quarterindex: [Monthindex, Quarterindex],
     "real-date": [Height],
-    "sats-per-dollar": [Dateindex, Height],
+    "sats-per-dollar": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
-    size: [Height],
+    subsidy: [Height],
+    "subsidy-10p": [Dateindex],
+    "subsidy-25p": [Dateindex],
+    "subsidy-75p": [Dateindex],
+    "subsidy-90p": [Dateindex],
+    "subsidy-average": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "subsidy-max": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "subsidy-median": [Dateindex],
+    "subsidy-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "subsidy-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
     timestamp: [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch, Halvingepoch],
-    "total-block-count": [Dateindex],
+    "total-block-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
-    "total-size": [Txindex],
+    "total-block-size": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-block-vbytes": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-block-weight": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-coinbase": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-fee": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-input-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-input-value": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-output-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-output-value": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-size": [Height, Txindex],
+    "total-subsidy": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-tx-count": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-tx-v1": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-tx-v2": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "total-tx-v3": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-count": [Height],
+    "tx-count-10p": [Dateindex],
+    "tx-count-25p": [Dateindex],
+    "tx-count-75p": [Dateindex],
+    "tx-count-90p": [Dateindex],
+    "tx-count-average": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-count-max": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-count-median": [Dateindex],
+    "tx-count-min": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-count-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-v1": [Height],
+    "tx-v1-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-v2": [Height],
+    "tx-v2-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-v3": [Height],
+    "tx-v3-sum": [Dateindex, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-vsize-10p": [Height],
+    "tx-vsize-25p": [Height],
+    "tx-vsize-75p": [Height],
+    "tx-vsize-90p": [Height],
+    "tx-vsize-average": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-vsize-max": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-vsize-median": [Height],
+    "tx-vsize-min": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-weight-10p": [Height],
+    "tx-weight-25p": [Height],
+    "tx-weight-75p": [Height],
+    "tx-weight-90p": [Height],
+    "tx-weight-average": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-weight-max": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
+    "tx-weight-median": [Height],
+    "tx-weight-min": [Dateindex, Height, Weekindex, Monthindex, Quarterindex, Yearindex, Decadeindex, Difficultyepoch],
     txid: [Txindex],
     txoutindex: [Txinindex],
     txversion: [Txindex],
-    value: [Txoutindex],
+    value: [Txinindex, Txoutindex],
+    vbytes: [Height],
+    vsize: [Txindex],
     weekindex: [Dateindex, Weekindex],
-    weight: [Height],
+    weight: [Height, Txindex],
     yearindex: [Monthindex, Yearindex],
-  }
+  });
 }
 /** @typedef {ReturnType<typeof createVecIdToIndexes>} VecIdToIndexes */
 /** @typedef {keyof VecIdToIndexes} VecId */