release: v0.0.111

.claude/prompts/changelog.md

@@ -1,137 +1,221 @@
-# Changelog Generation Prompt
+# Changelog Generation for Claude Code
 
-Update docs/CHANGELOG.md for ALL latest releases not present in the file. Use ONLY git commands - no other sources.
+**TASK**: Update docs/CHANGELOG.md for ALL latest releases missing from the file.
 
-## MANDATORY PROCESS - FOLLOW EXACTLY - NO EXCEPTIONS:
-1. Run `git tag --list --sort=version:refname` to get releases in order
-2. Process EXACTLY ONE release at a time
-3. For EACH SINGLE release: run `git diff [previous-tag]..[current-tag]`
-4. **MANDATORY ANALYSIS STEP**: Before writing ANY changelog entry, you MUST:
-   - Analyze each file change and explain what the code is doing
-   - Identify the purpose and impact of each modification
-   - Group related changes together logically
-   - State clearly what functionality is being added, removed, or modified
-   - If you cannot understand what a change does from the diff, explicitly say so
-5. Only AFTER completing the analysis, write the detailed changelog entry
-6. Update the CHANGELOG.md file with that ONE entry
-7. STOP. Ask me if you should continue to the next release.
+## ⚠️ CRITICAL FAILURE MODE TO AVOID ⚠️
+**THE #1 FAILURE**: Ignoring most changes and only documenting a few
+**ABSOLUTELY FORBIDDEN**: Skipping changes, summarizing with "and other updates", or being incomplete
+**YOU MUST DOCUMENT EVERY SINGLE MEANINGFUL CHANGE - NO EXCEPTIONS**
 
-## CRITICAL CONSTRAINTS:
-- **NEVER EVER** process multiple releases in one go, even if there are many
-- **NEVER** say "let me continue more efficiently by processing multiple releases"
-- **NEVER** batch releases together for any reason
-- If you feel tempted to process multiple releases, **STOP** and process only one
-- Context window concerns do **NOT** justify batching - process one release only
+## CORE WORKFLOW - EXECUTE EXACTLY:
 
-## ABSOLUTE REQUIREMENTS:
-- **NEVER** read commit messages, PR descriptions, existing changelog, or any text documentation
-- Use **ONLY** the actual code changes shown in git diff output
-- Process releases **ONE BY ONE** - I don't care if there are 100 releases
-- **MANDATORY**: Before writing changelog entries, demonstrate understanding by analyzing what each code change accomplishes
-- Be **HIGHLY DESCRIPTIVE** about what each code change does and why it matters
-- Don't be conservative - write detailed explanations of the impact and purpose of changes
-- **If you don't understand a change from the code diff alone, DO NOT GUESS - say so explicitly**
+### Step 1: Get Release Information
+```bash
+git tag --list --sort=version:refname
+```
 
-## SOURCE OF TRUTH:
-- `git diff` output is the **ONLY** source of truth
-- If you can't determine what a change does from the code diff alone, say so explicitly
-- Ignore **ALL** text/documentation - focus purely on code additions, deletions, and modifications
+### Step 2: Process ONE Release at a Time
+For each missing release, execute these commands to get complete information:
 
-## CHANGELOG FILE REQUIREMENTS:
-- Add a header at the top of the CHANGELOG.md file: `<!-- This changelog was generated by Claude Code -->`
-- Ensure this header appears before any changelog entries
+**First, get the file list:**
+```bash
+git diff --name-only [previous-tag]..[current-tag]
+```
 
-## CHANGELOG WRITING RULES:
+**Then, get changes excluding Cargo.lock:**
+```bash
+git diff [previous-tag]..[current-tag] -- . ':(exclude)Cargo.lock'
+```
 
-### RELEASE TITLE FORMAT:
-**MUST** use this exact format: `## [vX.Y.Z](https://github.com/bitcoinresearchkit/brk/releases/tag/vX.Y.Z) - YYYY-MM-DD`
-Use the actual release date from git tag information
+**If output is too large, examine files individually:**
+```bash
+git diff [previous-tag]..[current-tag] -- path/to/specific/file.rs
+```
 
-### ABSOLUTELY FORBIDDEN PATTERNS:
-- **NEVER** mention line counts (e.g., "with 138 lines", "1,290 lines removed")
-- **NEVER** use vague action words: "Enhanced", "Improved", "Updated", "Expanded", "Restructured", "Refactored", "Modified", "Adjusted"
-- **NEVER** write sections about Cargo.lock or dependency updates unless they represent major functional changes
-- **NEVER** use the format "Action: File with vague description"
-- **NEVER** mention version bumps of local crates (e.g., "Updated all crate versions from 0.0.61 to 0.0.62") - this is implied by the release version
-- **NEVER** mention dependency version changes in external crates unless they enable new functionality visible in the code
-- **NEVER** write entries like "Updated dependencies" or "Cargo.lock maintenance"
+**For summary of changes per file (if needed):**
+```bash
+git diff --stat [previous-tag]..[current-tag]
+```
 
-### REQUIRED WRITING STYLE:
-- Write what the code **actually DOES**, not that it was "enhanced" or "improved"
-- Be **specific about functionality**: "Added transaction validation logic", "Implemented caching for API responses"
-- Focus on **business/functional impact**: "Enables users to...", "Fixes issue where...", "Adds support for..."
-- **Mandatory structure**: Group by: Breaking Changes, New Features, Bug Fixes, Internal Changes
-- Include GitHub file links for major changes (max 5 per entry)
-- **Skip entirely**: minor dependency bumps, Cargo.lock changes, and local crate version bumps
+### Step 3: Analyze Before Writing
+**MANDATORY**: Before writing ANY changelog entry, analyze the diff output and explain:
+- **Use `git diff --name-only` to see ALL changed files** - this prevents truncation issues
+- **Use `git diff -- . ':(exclude)Cargo.lock'` to see actual changes** without Cargo.lock noise
+- **If output is large, examine key files individually** with `git diff [tags] -- path/to/file`
+- **Identify every functional change** - what new capabilities, fixes, or modifications were made
+- What each code change accomplishes functionally
+- The user-facing or system impact of modifications
+- **Which crate each change belongs to** (based on file paths)
+- How changes group together logically within and across crates
+- What functionality is added/removed/modified per crate
 
-### MANDATORY ANALYSIS WORKFLOW:
-**BEFORE writing any changelog entry, you MUST:**
+**COMPLETENESS CHECK**: State "I have analyzed X files and identified Y distinct functional changes to document"
 
-1. **Code Comprehension Check**: Go through each modified file and explain:
-   - What specific functionality is being added/removed/changed
-   - What the new/modified functions/structs/methods do
-   - How the changes affect the overall system behavior
+### Step 4: Write Changelog Entry
+Only after analysis, update CHANGELOG.md with ONE release entry.
 
-2. **Impact Assessment**: For each change, determine:
-   - Is this a new feature, bug fix, breaking change, or internal improvement?
-   - What user-facing or system behavior changes result from this code?
-   - What problem does this change solve?
+**REQUIRED**: End each release entry with a comparison link:
+```markdown
+[View changes](https://github.com/bitcoinresearchkit/brk/compare/vPREVIOUS...vCURRENT)
+```
+Where PREVIOUS is the previous release tag and CURRENT is the current release tag.
 
-3. **Logical Grouping**: Organize related changes together:
-   - Group files that work together to implement a single feature
-   - Separate breaking changes from additions
-   - Distinguish user-facing changes from internal refactoring
+### Step 5: Stop and Confirm
+**CRITICAL**: Process only ONE release, then ask for confirmation to continue.
 
-4. **Understanding Verification**: Before writing changelog text, state:
-   - "I understand this change does X because the code shows Y"
-   - If unclear: "I cannot determine the purpose of this change from the diff alone"
+---
 
-**ONLY AFTER completing this analysis should you write the changelog entry.**
+## STRICT RULES
 
-### WHAT TO FOCUS ON (IN ORDER OF PRIORITY):
-1. **New functionality** - What can users now do that they couldn't before?
-2. **Breaking changes** - What existing functionality changed or was removed?
-3. **Bug fixes** - What specific problems were resolved?
-4. **Internal changes** - New modules, significant refactoring, architecture changes
-5. **Skip completely** - Dependency updates, version bumps, Cargo.lock changes
+### FORBIDDEN - NEVER MENTION:
+- **Cargo.lock** (ignore completely)
+- **Line counts** ("added 50 lines", "removed 200 lines")
+- **Dependency version bumps** (unless enabling major new features)
+- **Local crate version changes** (0.0.61 → 0.0.62)
+- **Vague words**: "Enhanced", "Improved", "Updated", "Refactored"
 
-### VERBOSITY REQUIREMENTS:
-- **Minimum 3-4 bullet points per section** when changes exist
-- **Each bullet point should be 1-2 sentences** explaining both what changed and why it matters
-- **For new features**: Explain what the feature does and what problem it solves
-- **For bug fixes**: Describe the problem that was fixed (inferred from the code changes)
-- **For internal changes**: Explain the architectural or structural improvement
+### FORBIDDEN PHRASES:
+- "and other changes"
+- "various updates"
+- "along with minor improvements"
+- "among other enhancements"
+- **"along with other modifications"**
+- **"plus additional changes"**
+- **"including other updates"**
+- **"and more"**
+- **ANY phrase that suggests you're skipping changes**
 
-### EXAMPLES OF GOOD vs BAD:
+### REQUIRED - MUST INCLUDE:
+- **Every meaningful change** in the diff (no shortcuts)
+- **Specific functionality** descriptions
+- **Business impact** of each change
+- **Complete coverage** - if 20 changes exist, document all 20
+- **Source links** for significant changes (new features, breaking changes, major bug fixes)
 
-#### ❌ BAD EXAMPLES:
-- "Enhanced: Chain analysis with sophisticated blockchain processing capabilities"
-- "Updated: brk_rolldown from 0.0.1 to 0.1.0 with comprehensive bundling improvements"
-- "Version Bump: Updated all crate versions from 0.0.61 to 0.0.62"
-- "Improved error handling"
-- "Refactored codebase"
-- "Updated dependencies"
+## WORKSPACE-SPECIFIC RULES
 
-#### ✅ GOOD EXAMPLES WITH ANALYSIS:
+### Crate Identification:
+- **Determine crate from file paths** in the diff (e.g., `crates/brk-core/src/lib.rs` → `brk-core`)
+- **Group all changes by their crate** before writing changelog entries
+- **Use crate name as subheading** under each change type section
+- **For root-level files**: Use `workspace` as the crate name
 
-**Analysis**: "Looking at the diff, I see a new `TransactionAnalyzer` struct was added with methods `calculate_fee()` and `is_coinbase()`. The struct takes transaction data and provides analysis methods. This enables users to programmatically analyze transaction properties."
+### Cross-Crate Changes:
+- **When changes span multiple crates** for one feature, mention the relationship
+- **Example**: "Added new transaction API in `brk-core` with corresponding HTTP endpoints in `brk-api`"
 
-**Changelog**: "Added new `TransactionAnalyzer` struct that provides methods for computing transaction fees and detecting coinbase transactions"
+### Crate Naming:
+- **Use backticks** around crate names: `brk-core`, `brk-api`
+- **Use workspace structure** as shown in file paths, not display names
 
-**Analysis**: "The diff shows error handling was added around block parsing where previously there was an unwrap(). Now it returns a Result and handles the empty block case explicitly. This prevents panics when processing malformed blocks."
+### File Header (if missing):
+```markdown
+<!-- This changelog was generated by Claude Code -->
+```
 
-**Changelog**: "Fixed panic when processing blocks with zero transactions by adding explicit empty block handling and proper error propagation"
+### Release Entry Format:
+```markdown
+## [vX.Y.Z](https://github.com/bitcoinresearchkit/brk/releases/tag/vX.Y.Z) - YYYY-MM-DD
 
-**Analysis**: "I see a new caching layer was implemented with a HashMap storing block hashes as keys and block data as values. The API endpoints now check this cache before making network requests. This should improve performance for repeated queries."
+### Breaking Changes
+#### `crate-name`
+- Specific change with functional impact explanation ([source](https://github.com/bitcoinresearchkit/brk/blob/vX.Y.Z/path/to/file.rs))
 
-**Changelog**: "Implemented new caching layer for blockchain queries, reducing API response time by storing frequently accessed block data in memory"
+### New Features
+#### `crate-name`
+- Feature description with user benefit ([source](https://github.com/bitcoinresearchkit/brk/blob/vX.Y.Z/path/to/main/file.rs))
+- Another feature with implementation details
 
-#### ❌ BAD EXAMPLES (NO UNDERSTANDING):
-- "Enhanced error handling" (What specific errors? How were they enhanced?)
-- "Improved performance" (What was improved? How?)
-- "Updated transaction logic" (What specific logic? What changed?)
+#### `another-crate`
+- Feature specific to this crate
 
-## FINAL REMINDER:
-**PROCESS ONLY ONE RELEASE. THEN STOP AND WAIT FOR MY CONFIRMATION.**
+### Bug Fixes
+#### `crate-name`
+- Specific problem that was resolved ([source](https://github.com/bitcoinresearchkit/brk/blob/vX.Y.Z/path/to/file.rs))
+- Another bug fix with impact description
 
-You must be thorough and verbose - if there are code changes, there should be substantial changelog content explaining what those changes accomplish.
+### Internal Changes
+#### `crate-name`
+- Architectural improvement with purpose
+- Code organization change with benefit
+
+[View changes](https://github.com/bitcoinresearchkit/brk/compare/vPREVIOUS...vCURRENT)
+```
+
+---
+
+## ANALYSIS REQUIREMENTS
+
+**Before writing changelog text, you MUST state:**
+
+1. **File discovery**: "Using git diff --name-only, I see X files were modified"
+2. **Cargo.lock check**: "After excluding Cargo.lock, Y files contain actual changes"
+3. **Functional changes identified**: "I identified these distinct changes: A, B, C, D..."
+4. **What each change does**: "Change A: adds new API endpoints, Change B: fixes memory leak, etc."
+5. **Which crates are affected**: "Changes span crates: A, B, C"
+6. **What this means for users**: "Users can now do X, bug Y is fixed, etc."
+7. **How changes group together**: "Changes A and B work together to implement feature W"
+8. **Cross-crate dependencies**: "Crate A's new feature requires the new API in crate B"
+9. **Any unclear changes**: "I cannot determine the purpose of change X from the diff"
+10. **FINAL CHECK**: "I will now document all X functional changes without listing files"
+
+**MANDATORY**: The changelog should describe FUNCTIONALITY, not files. But you must capture ALL functionality changes.
+
+**Only after this analysis, write the changelog entry.**
+
+---
+
+## EXAMPLES
+
+### ❌ BAD (Vague, incomplete):
+```markdown
+### New Features
+- Enhanced blockchain processing capabilities
+- Improved error handling and various other updates
+```
+
+### ✅ GOOD (Specific, complete, grouped by crate, with source links):
+```markdown
+### New Features
+#### `brk-core`
+- Added `TransactionAnalyzer` struct with fee calculation and coinbase detection methods ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.0.108/crates/brk-core/src/analyzer.rs))
+- Implemented in-memory caching layer for blockchain queries using HashMap storage
+
+#### `brk-api`
+- Added three new API endpoints: `/api/blocks/{hash}`, `/api/transactions/search`, and `/api/stats/network` ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.0.108/crates/brk-api/src/routes.rs))
+- Implemented standardized error responses with error codes and descriptions
+
+### Bug Fixes
+#### `brk-core`
+- Fixed panic when processing blocks with zero transactions by adding explicit empty block validation ([source](https://github.com/bitcoinresearchkit/brk/blob/v0.0.108/crates/brk-core/src/block.rs))
+
+#### `brk-api`
+- Resolved memory leak in connection pool by implementing proper cleanup in Drop trait
+
+[View changes](https://github.com/bitcoinresearchkit/brk/compare/v0.0.107...v0.0.108)
+```
+
+---
+
+## SUCCESS CRITERIA
+
+✅ **COUNT files modified for internal verification only**
+✅ **Identify EVERY distinct functional change**
+✅ **Document ALL functionality changes (no "other changes")**
+✅ **Changelog describes WHAT users can do, not which files changed**
+✅ **Never mention Cargo.lock or line counts**
+✅ **Use specific, functional descriptions**
+✅ **Complete analysis before writing**
+✅ **Stop and ask for confirmation after each release**
+
+**FAILURE INDICATORS - If you do any of these, you FAILED:**
+❌ "and other changes"
+❌ "various updates"
+❌ "among other improvements"
+❌ **Diff shows new API endpoints but changelog doesn't mention them**
+❌ **Diff shows bug fixes but changelog misses some**
+❌ **Diff shows new structs/functions but changelog ignores them**
+❌ Missing obvious functional changes from the diff
+❌ Summarizing instead of listing each distinct functionality change
+
+**KEY PRINCIPLE**: Count files internally to ensure you don't miss changes, but write about FUNCTIONALITY for users.

.zed/settings.json

@@ -12,9 +12,11 @@
     "**/.settings",
     // custom
     "**/lean-qr/*/index.mjs",
-    "uFuzzy.mjs",
-    "lightweight-charts.standalone.production.mjs",
     "**/modern-screenshot/*/index.mjs",
-    "**/solidjs-signals/*/dist/prod.js"
+    "**/solidjs-signals/*/dist/prod.js",
+    "uFuzzy.mjs",
+    "lightweight-charts.standalone.production.mjs"
+    // "scripts/packages",
+    // "dist"
   ]
 }

Cargo.toml

@@ -4,7 +4,7 @@ members = ["crates/*"]
 package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node"
 package.license = "MIT"
 package.edition = "2024"
-package.version = "0.0.109"
+package.version = "0.0.111"
 package.homepage = "https://bitcoinresearchkit.org"
 package.repository = "https://github.com/bitcoinresearchkit/brk"
 package.readme = "README.md"
@@ -42,22 +42,23 @@ debug-assertions = false
 
 [workspace.dependencies]
 allocative = { version = "0.3.4", features = ["parking_lot"] }
-axum = "0.8.4"
+axum = "0.8.6"
 bitcoin = { version = "0.32.7", features = ["serde"] }
 bitcoincore-rpc = "0.19.0"
-brk_bundler = { version = "0.0.109", path = "crates/brk_bundler" }
-brk_cli = { version = "0.0.109", path = "crates/brk_cli" }
-brk_computer = { version = "0.0.109", path = "crates/brk_computer" }
-brk_error = { version = "0.0.109", path = "crates/brk_error" }
-brk_fetcher = { version = "0.0.109", path = "crates/brk_fetcher" }
-brk_indexer = { version = "0.0.109", path = "crates/brk_indexer" }
-brk_interface = { version = "0.0.109", path = "crates/brk_interface" }
-brk_logger = { version = "0.0.109", path = "crates/brk_logger" }
-brk_mcp = { version = "0.0.109", path = "crates/brk_mcp" }
-brk_parser = { version = "0.0.109", path = "crates/brk_parser" }
-brk_server = { version = "0.0.109", path = "crates/brk_server" }
-brk_store = { version = "0.0.109", path = "crates/brk_store" }
-brk_structs = { version = "0.0.109", path = "crates/brk_structs" }
+brk_binder = { version = "0.0.111", path = "crates/brk_binder" }
+brk_bundler = { version = "0.0.111", path = "crates/brk_bundler" }
+brk_cli = { version = "0.0.111", path = "crates/brk_cli" }
+brk_computer = { version = "0.0.111", path = "crates/brk_computer" }
+brk_error = { version = "0.0.111", path = "crates/brk_error" }
+brk_fetcher = { version = "0.0.111", path = "crates/brk_fetcher" }
+brk_indexer = { version = "0.0.111", path = "crates/brk_indexer" }
+brk_interface = { version = "0.0.111", path = "crates/brk_interface" }
+brk_logger = { version = "0.0.111", path = "crates/brk_logger" }
+brk_mcp = { version = "0.0.111", path = "crates/brk_mcp" }
+brk_parser = { version = "0.0.111", path = "crates/brk_parser" }
+brk_server = { version = "0.0.111", path = "crates/brk_server" }
+brk_store = { version = "0.0.111", path = "crates/brk_store" }
+brk_structs = { version = "0.0.111", path = "crates/brk_structs" }
 byteview = "=0.6.1"
 derive_deref = "1.1.1"
 fjall = "2.11.2"
@@ -67,11 +68,12 @@ minreq = { version = "2.14.1", features = ["https", "serde_json"] }
 parking_lot = "0.12.4"
 quick_cache = "0.6.16"
 rayon = "1.11.0"
-serde = "1.0.225"
+schemars = "1.0.4"
+serde = "1.0.228"
 serde_bytes = "0.11.19"
-serde_derive = "1.0.225"
+serde_derive = "1.0.228"
 serde_json = { version = "1.0.145", features = ["float_roundtrip"] }
-sonic-rs = "0.5.4"
+sonic-rs = "0.5.5"
 tokio = { version = "1.47.1", features = ["rt-multi-thread"] }
 # vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]}
 vecdb = { version = "0.2.16", features = ["derive"]}
@@ -90,4 +92,3 @@ ci = "github"
 allow-dirty = ["ci"]
 installers = []
 targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]
-rust-toolchain-version = "1.89"

crates/brk/Cargo.toml

@@ -11,6 +11,7 @@ build = "build.rs"
 
 [features]
 full = [
+    "binder",
     "bundler",
     "computer",
     "error",
@@ -24,6 +25,7 @@ full = [
     "store",
     "structs",
 ]
+binder = ["brk_binder"]
 bundler = ["brk_bundler"]
 computer = ["brk_computer"]
 error = ["brk_error"]
@@ -38,6 +40,7 @@ store = ["brk_store"]
 structs = ["brk_structs"]
 
 [dependencies]
+brk_binder = { workspace = true, optional = true }
 brk_bundler = { workspace = true, optional = true }
 brk_cli = { workspace = true }
 brk_computer = { workspace = true, optional = true }

crates/brk/src/lib.rs

@@ -1,5 +1,9 @@
 #![doc = include_str!("../README.md")]
 
+#[cfg(feature = "binder")]
+#[doc(inline)]
+pub use brk_binder as binder;
+
 #[cfg(feature = "bundler")]
 #[doc(inline)]
 pub use brk_bundler as bundler;

crates/brk_binder/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "brk_binder"
+description = "A generator of binding files for other languages"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+build = "build.rs"
+
+[dependencies]
+brk_interface = { workspace = true }
+brk_structs = { workspace = true }

crates/brk_binder/README.md

@@ -0,0 +1 @@
+# brk_binder

crates/brk_binder/build.rs

@@ -0,0 +1,8 @@
+fn main() {
+    let profile = std::env::var("PROFILE").unwrap_or_default();
+
+    if profile == "release" {
+        println!("cargo:rustc-flag=-C");
+        println!("cargo:rustc-flag=target-cpu=native");
+    }
+}

crates/brk_binder/src/js.rs

@@ -0,0 +1,250 @@
+use std::{
+    collections::{BTreeMap, HashMap},
+    fs, io,
+    path::Path,
+};
+
+use brk_interface::{Index, Interface};
+use brk_structs::pools;
+
+use super::VERSION;
+
+const AUTO_GENERATED_DISCLAIMER: &str = "//
+// File auto-generated, any modifications will be overwritten
+//";
+
+#[allow(clippy::upper_case_acronyms)]
+pub trait Bridge {
+    fn generate_js_files(&self, modules_path: &Path) -> io::Result<()>;
+}
+
+impl Bridge for Interface<'static> {
+    fn generate_js_files(&self, modules_path: &Path) -> io::Result<()> {
+        let path = modules_path.join("brk-client");
+
+        if !fs::exists(&path)? {
+            return Ok(());
+        }
+
+        let path = path.join("generated");
+        fs::create_dir_all(&path)?;
+
+        generate_version_file(&path)?;
+        generate_metrics_file(self, &path)?;
+        generate_pools_file(&path)
+    }
+}
+
+fn generate_version_file(parent: &Path) -> io::Result<()> {
+    let path = parent.join(Path::new("version.js"));
+
+    let contents = format!(
+        "{AUTO_GENERATED_DISCLAIMER}
+
+export const VERSION = \"v{VERSION}\";
+"
+    );
+
+    fs::write(path, contents)
+}
+
+fn generate_pools_file(parent: &Path) -> io::Result<()> {
+    let path = parent.join(Path::new("pools.js"));
+
+    let pools = pools();
+
+    let mut contents = format!("{AUTO_GENERATED_DISCLAIMER}\n");
+
+    contents += "
+/**
+ * @typedef {typeof POOL_ID_TO_POOL_NAME} PoolIdToPoolName
+ * @typedef {keyof PoolIdToPoolName} PoolId
+ */
+
+export const POOL_ID_TO_POOL_NAME = /** @type {const} */ ({
+";
+
+    let mut sorted_pools: Vec<_> = pools.iter().collect();
+    sorted_pools.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase()));
+
+    contents += &sorted_pools
+        .iter()
+        .map(|pool| {
+            let id = pool.serialized_id();
+            format!("  {id}: \"{}\",", pool.name)
+        })
+        .collect::<Vec<_>>()
+        .join("\n");
+
+    contents += "\n});\n";
+
+    fs::write(path, contents)
+}
+
+fn generate_metrics_file(interface: &Interface<'static>, parent: &Path) -> io::Result<()> {
+    let path = parent.join(Path::new("metrics.js"));
+
+    let indexes = Index::all();
+
+    let mut contents = format!(
+        "{AUTO_GENERATED_DISCLAIMER}
+
+export const INDEXES = /** @type {{const}} */ ([
+{}
+]);
+
+/**
+ * @typedef {{typeof INDEXES[number]}} IndexName
+ */
+",
+        indexes
+            .iter()
+            .map(|i| format!("  \"{}\"", i.serialize_long()))
+            .collect::<Vec<_>>()
+            .join(",\n")
+    );
+
+    // contents += &indexes
+    //     .iter()
+    //     .map(|i| format!(" * @typedef {{\"{}\"}} {i}", i.serialize_long()))
+    //     .collect::<Vec<_>>()
+    //     .join("\n");
+
+    //     contents += &format!(
+    //         "
+    //  * @typedef {{{}}} Index
+    //  */
+    // ",
+    //         indexes
+    //             .iter()
+    //             .map(|i| i.to_string())
+    //             .collect::<Vec<_>>()
+    //             .join(" | ")
+    //     );
+
+    let mut unique_index_groups = BTreeMap::new();
+
+    let mut word_to_freq: BTreeMap<_, usize> = BTreeMap::new();
+    interface
+        .metric_to_index_to_vec()
+        .keys()
+        .for_each(|metric| {
+            metric.split("_").for_each(|word| {
+                *word_to_freq.entry(word).or_default() += 1;
+            });
+        });
+    let mut word_to_freq = word_to_freq.into_iter().collect::<Vec<_>>();
+    word_to_freq.sort_unstable_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0)));
+    let words = word_to_freq
+        .into_iter()
+        .map(|(str, _)| str)
+        .collect::<Vec<_>>();
+
+    contents += &format!(
+        "
+export const INDEX_TO_WORD = [
+  {}
+];
+
+",
+        words
+            .iter()
+            .enumerate()
+            .map(|(index, word)| format!("\"{word}\", // {}", index_to_letters(index)))
+            .collect::<Vec<_>>()
+            .join("\n  ")
+    );
+
+    let word_to_base62 = words
+        .into_iter()
+        .enumerate()
+        .map(|(i, w)| (w, index_to_letters(i)))
+        .collect::<HashMap<_, _>>();
+
+    let mut ser_metric_to_indexes = "
+/** @type {Record<string, IndexName[]>} */
+export const COMPRESSED_METRIC_TO_INDEXES = {
+"
+    .to_string();
+
+    interface
+        .metric_to_index_to_vec()
+        .iter()
+        .for_each(|(metric, index_to_vec)| {
+            let indexes = index_to_vec
+                .keys()
+                .map(|i| format!("\"{}\"", i.serialize_long()))
+                .collect::<Vec<_>>()
+                .join(", ");
+            let indexes = format!("[{indexes}]");
+            let unique = unique_index_groups.len();
+            let index = index_to_letters(*unique_index_groups.entry(indexes).or_insert(unique));
+
+            let compressed_metric = metric.split('_').fold(String::new(), |mut acc, w| {
+                if !acc.is_empty() {
+                    acc.push('_');
+                }
+                acc.push_str(&word_to_base62[w]);
+                acc
+            });
+
+            ser_metric_to_indexes += &format!("  {compressed_metric}: {index},\n");
+        });
+
+    ser_metric_to_indexes += "};
+";
+
+    let mut sorted_groups: Vec<_> = unique_index_groups.into_iter().collect();
+    sorted_groups.sort_by_key(|(_, index)| *index);
+    sorted_groups.into_iter().for_each(|(group, index)| {
+        let index = index_to_letters(index);
+        contents += &format!("/** @type {{IndexName[]}} */\nconst {index} = {group};\n");
+    });
+
+    contents += &ser_metric_to_indexes;
+
+    fs::write(path, contents)
+}
+
+fn index_to_letters(mut index: usize) -> String {
+    if index < 52 {
+        return (index_to_char(index) as char).to_string();
+    }
+    let mut result = [0u8; 8];
+    let mut pos = 8;
+    loop {
+        pos -= 1;
+        result[pos] = index_to_char(index % 52);
+        index /= 52;
+        if index == 0 {
+            break;
+        }
+        index -= 1;
+    }
+    unsafe { String::from_utf8_unchecked(result[pos..].to_vec()) }
+}
+
+fn index_to_char(index: usize) -> u8 {
+    match index {
+        0..=25 => b'A' + index as u8,
+        26..=51 => b'a' + (index - 26) as u8,
+        _ => unreachable!(),
+    }
+}
+
+// fn letters_to_index(s: &str) -> usize {
+//     let mut result = 0;
+//     for byte in s.bytes() {
+//         let value = char_to_index(byte) as usize;
+//         result = result * 52 + value + 1;
+//     }
+//     result - 1
+// }
+
+// fn char_to_index(byte: u8) -> u8 {
+//     match byte {
+//         b'A'..=b'Z' => byte - b'A',
+//         b'a'..=b'z' => byte - b'a' + 26,
+//         _ => 255, // Invalid
+//     }
+// }

crates/brk_binder/src/lib.rs

@@ -0,0 +1,5 @@
+mod js;
+
+pub use js::Bridge;
+
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");

crates/brk_binder/src/python.rs

@@ -0,0 +1 @@
+// TODO ?

crates/brk_bundler/Cargo.toml

@@ -12,7 +12,7 @@ build = "build.rs"
 [dependencies]
 log = { workspace = true }
 notify = "8.2.0"
-brk_rolldown = "0.1.5"
+brk_rolldown = "0.2.3"
 # brk_rolldown = { path = "../../../rolldown/crates/rolldown"}
 sugar_path = "1.2.0"
 tokio = { workspace = true }

crates/brk_bundler/src/lib.rs

@@ -6,7 +6,10 @@ use std::{
     sync::Arc,
 };
 
-use brk_rolldown::{Bundler, BundlerOptions, RawMinifyOptions, SourceMapType};
+use brk_rolldown::{
+    Bundler, BundlerOptions, InlineConstConfig, InlineConstMode, InlineConstOption,
+    OptimizationOption, RawMinifyOptions, SourceMapType,
+};
 use log::error;
 use notify::{EventKind, RecursiveMode, Watcher};
 use sugar_path::SugarPath;
@@ -14,49 +17,88 @@ use tokio::sync::Mutex;
 
 const VERSION: &str = env!("CARGO_PKG_VERSION");
 
-pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> io::Result<PathBuf> {
-    let source_path = websites_path.join(source_folder);
-    let dist_path = websites_path.join("dist");
-
-    let _ = fs::remove_dir_all(&dist_path);
-    copy_dir_all(&source_path, &dist_path)?;
-
-    let source_scripts = format!("./{source_folder}/scripts");
-    let source_entry = format!("{source_scripts}/entry.js");
+pub async fn bundle(
+    modules_path: &Path,
+    websites_path: &Path,
+    source_folder: &str,
+    watch: bool,
+) -> io::Result<PathBuf> {
+    let relative_modules_path = modules_path;
+    let relative_source_path = websites_path.join(source_folder);
+    let relative_dist_path = websites_path.join("dist");
 
+    let absolute_modules_path = relative_modules_path.absolutize();
+    let absolute_modules_path_clone = absolute_modules_path.clone();
     let absolute_websites_path = websites_path.absolutize();
+    let absolute_websites_path_clone = absolute_websites_path.clone();
+
+    let absolute_source_path = relative_source_path.absolutize();
+    let absolute_source_index_path = absolute_source_path.join("index.html");
+    let absolute_source_index_path_clone = absolute_source_index_path.clone();
+    let absolute_source_scripts_path = absolute_source_path.join("scripts");
+    let absolute_source_scripts_modules_path = absolute_source_scripts_path.join("modules");
+    let absolute_source_sw_path = absolute_source_path.join("service-worker.js");
+    let absolute_source_sw_path_clone = absolute_source_sw_path.clone();
+
+    let absolute_dist_path = relative_dist_path.absolutize();
+    let absolute_dist_scripts_path = absolute_dist_path.join("scripts");
+    let absolute_dist_scripts_entry_path = absolute_dist_scripts_path.join("entry.js");
+    let absolute_dist_scripts_entry_path_clone = absolute_dist_scripts_entry_path.clone();
+    let absolute_dist_index_path = absolute_dist_path.join("index.html");
+    let absolute_dist_sw_path = absolute_dist_path.join("service-worker.js");
+
+    let _ = fs::remove_dir_all(&absolute_dist_path);
+    let _ = fs::remove_dir_all(&absolute_source_scripts_modules_path);
+    copy_dir_all(
+        &absolute_modules_path,
+        &absolute_source_scripts_modules_path,
+    )?;
+    copy_dir_all(&absolute_source_path, &absolute_dist_path)?;
+    fs::remove_dir_all(&absolute_dist_scripts_path)?;
+    fs::create_dir(&absolute_dist_scripts_path)?;
+
+    // dbg!(BundlerOptions::default());
 
     let mut bundler = Bundler::new(BundlerOptions {
-        input: Some(vec![source_entry.into()]),
+        input: Some(vec![format!("./{source_folder}/scripts/entry.js").into()]),
         dir: Some("./dist/scripts".to_string()),
         cwd: Some(absolute_websites_path),
         minify: Some(RawMinifyOptions::Bool(true)),
         sourcemap: Some(SourceMapType::File),
+        // advanced_chunks: Some(AdvancedChunksOptions {
+        //     // min_size: Some(1000.0),
+        //     min_share_count: Some(20),
+        //     // min_module_size: S
+        //     // include_dependencies_recursively: Some(true),
+        //     ..Default::default()
+        // }),
+        //
+        // inline_dynamic_imports
+        // experimental: Some(ExperimentalOptions {
+        //     strict_execution_order: Some(true),
+        //     ..Default::default()
+        // }),
+        optimization: Some(OptimizationOption {
+            inline_const: Some(InlineConstOption::Config(InlineConstConfig {
+                mode: Some(InlineConstMode::All),
+                ..Default::default()
+            })),
+            // Needs benchmarks
+            // pife_for_module_wrappers: Some(true),
+            ..Default::default()
+        }),
         ..Default::default()
-    });
+    })
+    .unwrap();
 
     if let Err(error) = bundler.write().await {
         error!("{error:?}");
     }
 
-    let absolute_source_index_path = source_path.join("index.html").absolutize();
-    let absolute_source_index_path_clone = absolute_source_index_path.clone();
-    let absolute_source_path = source_path.absolutize();
-    let absolute_source_path_clone = absolute_source_path.clone();
-    let absolute_source_scripts_path = websites_path.join(source_scripts).absolutize();
-    let absolute_source_sw_path = source_path.join("service-worker.js").absolutize();
-    let absolute_source_sw_path_clone = absolute_source_sw_path.clone();
-
-    let absolute_dist_entry_path = dist_path.join("scripts/entry.js").absolutize();
-    let absolute_dist_index_path = dist_path.join("index.html").absolutize();
-    let absolute_dist_path = dist_path.absolutize();
-    let absolute_dist_path_clone = absolute_dist_path.clone();
-    let absolute_dist_sw_path = dist_path.join("service-worker.js").absolutize();
-
-    let write_index = move || {
+    let update_dist_index = move || {
         let mut contents = fs::read_to_string(&absolute_source_index_path).unwrap();
 
-        if let Ok(entry) = fs::read_to_string(absolute_dist_path_clone.join("scripts/entry.js"))
+        if let Ok(entry) = fs::read_to_string(&absolute_dist_scripts_entry_path_clone)
             && let Some(start) = entry.find("main")
             && let Some(end) = entry.find(".js")
         {
@@ -67,36 +109,22 @@ pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> i
         let _ = fs::write(&absolute_dist_index_path, contents);
     };
 
-    let write_sw = move || {
+    let update_source_sw = move || {
         let contents = fs::read_to_string(&absolute_source_sw_path)
             .unwrap()
             .replace("__VERSION__", &format!("v{VERSION}"));
         let _ = fs::write(&absolute_dist_sw_path, contents);
     };
 
-    write_index();
-    write_sw();
+    update_dist_index();
+    update_source_sw();
 
     if !watch {
-        return Ok(dist_path);
+        return Ok(relative_dist_path);
     }
 
     tokio::spawn(async move {
-        let write_index_clone = write_index.clone();
-
-        let mut entry_watcher = notify::recommended_watcher(
-            move |res: Result<notify::Event, notify::Error>| match res {
-                Ok(_) => write_index_clone(),
-                Err(e) => error!("watch error: {e:?}"),
-            },
-        )
-        .unwrap();
-
-        entry_watcher
-            .watch(&absolute_dist_entry_path, RecursiveMode::Recursive)
-            .unwrap();
-
-        let mut source_watcher = notify::recommended_watcher(
+        let mut event_watcher = notify::recommended_watcher(
             move |res: Result<notify::Event, notify::Error>| match res {
                 Ok(event) => match event.kind {
                     EventKind::Create(_) => event.paths,
@@ -104,18 +132,30 @@ pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> i
                     _ => vec![],
                 }
                 .into_iter()
-                .filter(|path| path.starts_with(&absolute_source_path))
-                .filter(|path| !path.starts_with(&absolute_source_scripts_path))
-                .for_each(|source_path| {
-                    let suffix = source_path.strip_prefix(&absolute_source_path).unwrap();
-                    let dist_path = absolute_dist_path.join(suffix);
+                .for_each(|path| {
+                    let path = path.absolutize();
 
-                    if source_path == absolute_source_index_path_clone {
-                        write_index();
-                    } else if source_path == absolute_source_sw_path_clone {
-                        write_sw();
-                    } else {
-                        let _ = fs::copy(&source_path, &dist_path);
+                    if path == absolute_dist_scripts_entry_path
+                        || path == absolute_source_index_path_clone
+                    {
+                        update_dist_index();
+                    } else if path == absolute_source_sw_path_clone {
+                        update_source_sw();
+                    } else if let Ok(suffix) = path.strip_prefix(&absolute_modules_path) {
+                        let source_modules_path = absolute_source_scripts_modules_path.join(suffix);
+                        if path.is_file() {
+                            let _ = fs::create_dir_all(path.parent().unwrap());
+                            let _ = fs::copy(&path, &source_modules_path);
+                        }
+                    } else if let Ok(suffix) = path.strip_prefix(&absolute_source_path)
+                        // scripts are handled by rolldown
+                        && !path.starts_with(&absolute_source_scripts_path)
+                    {
+                        let dist_path = absolute_dist_path.join(suffix);
+                        if path.is_file() {
+                            let _ = fs::create_dir_all(path.parent().unwrap());
+                            let _ = fs::copy(&path, &dist_path);
+                        }
                     }
                 }),
                 Err(e) => error!("watch error: {e:?}"),
@@ -123,8 +163,11 @@ pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> i
         )
         .unwrap();
 
-        source_watcher
-            .watch(&absolute_source_path_clone, RecursiveMode::Recursive)
+        event_watcher
+            .watch(&absolute_websites_path_clone, RecursiveMode::Recursive)
+            .unwrap();
+        event_watcher
+            .watch(&absolute_modules_path_clone, RecursiveMode::Recursive)
             .unwrap();
 
         let watcher =
@@ -133,7 +176,7 @@ pub async fn bundle(websites_path: &Path, source_folder: &str, watch: bool) -> i
         watcher.start().await;
     });
 
-    Ok(dist_path)
+    Ok(relative_dist_path)
 }
 
 fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {

crates/brk_cli/Cargo.toml

@@ -11,15 +11,16 @@ build = "build.rs"
 
 [dependencies]
 bitcoincore-rpc = { workspace = true }
+brk_binder = { workspace = true }
 brk_bundler = { workspace = true }
 brk_computer = { workspace = true }
+brk_error = { workspace = true }
 brk_fetcher = { workspace = true }
 brk_indexer = { workspace = true }
 brk_interface = { workspace = true }
 brk_logger = { workspace = true }
 brk_parser = { workspace = true }
 brk_server = { workspace = true }
-brk_structs = { workspace = true }
 vecdb = { workspace = true }
 clap = { version = "4.5.48", features = ["derive", "string"] }
 color-eyre = "0.6.5"

crates/brk_cli/src/bridge.rs

@@ -1,154 +0,0 @@
-use std::{fs, io, path::Path};
-
-use brk_interface::{Index, Interface};
-use brk_server::VERSION;
-use brk_structs::pools;
-
-use crate::website::Website;
-
-const BRIDGE_PATH: &str = "scripts/bridge";
-
-#[allow(clippy::upper_case_acronyms)]
-pub trait Bridge {
-    fn generate_bridge_files(&self, website: Website, websites_path: &Path) -> io::Result<()>;
-}
-
-impl Bridge for Interface<'static> {
-    fn generate_bridge_files(&self, website: Website, websites_path: &Path) -> io::Result<()> {
-        if website.is_none() {
-            return Ok(());
-        }
-
-        let path = websites_path.join(website.to_folder_name());
-
-        if !fs::exists(&path)? {
-            return Ok(());
-        }
-
-        let path = path.join(BRIDGE_PATH);
-
-        fs::create_dir_all(&path)?;
-
-        generate_vecs_file(self, &path)?;
-        generate_pools_file(&path)
-    }
-}
-
-fn generate_pools_file(parent: &Path) -> io::Result<()> {
-    let path = parent.join(Path::new("pools.js"));
-
-    let pools = pools();
-
-    let mut contents = "//
-// File auto-generated, any modifications will be overwritten
-//
-"
-    .to_string();
-
-    contents += "
-/** @typedef {ReturnType<typeof createPools>} Pools */
-/** @typedef {keyof Pools} Pool */
-
-export function createPools() {
-return /** @type {const} */ ({
-";
-
-    let mut sorted_pools: Vec<_> = pools.iter().collect();
-    sorted_pools.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase()));
-
-    contents += &sorted_pools
-        .iter()
-        .map(|pool| {
-            let id = pool.serialized_id();
-            format!("    {id}: \"{}\",", pool.name)
-        })
-        .collect::<Vec<_>>()
-        .join("\n");
-
-    contents += "\n  });\n}\n";
-
-    fs::write(path, contents)
-}
-
-fn generate_vecs_file(interface: &Interface<'static>, parent: &Path) -> io::Result<()> {
-    let path = parent.join(Path::new("vecs.js"));
-
-    let indexes = Index::all();
-
-    let mut contents = format!(
-        "//
-// File auto-generated, any modifications will be overwritten
-//
-
-export const VERSION = \"v{VERSION}\";
-
-"
-    );
-
-    contents += &indexes
-        .iter()
-        .enumerate()
-        .map(|(i_of_i, i)| {
-            // let lowered = i.to_string().to_lowercase();
-            format!("/** @typedef {{{i_of_i}}} {i} */",)
-        })
-        .collect::<Vec<_>>()
-        .join("\n");
-
-    contents += &format!(
-        "\n\n/** @typedef {{{}}} Index */\n",
-        indexes
-            .iter()
-            .map(|i| i.to_string())
-            .collect::<Vec<_>>()
-            .join(" | ")
-    );
-
-    contents += "
-/** @typedef {ReturnType<typeof createIndexes>} Indexes */
-
-export function createIndexes() {
-return {
-";
-
-    contents += &indexes
-        .iter()
-        .enumerate()
-        .map(|(i_of_i, i)| {
-            let lowered = i.to_string().to_lowercase();
-            format!("    {lowered}: /** @satisfies {{{i}}} */ ({i_of_i}),",)
-        })
-        .collect::<Vec<_>>()
-        .join("\n");
-
-    contents += "  };\n}\n";
-
-    contents += "
-/** @typedef {ReturnType<typeof createVecIdToIndexes>} VecIdToIndexes
-/** @typedef {keyof VecIdToIndexes} VecId */
-
-/**
-* @returns {Record<any, number[]>}
-*/
-export function createVecIdToIndexes() {
-return {
-";
-
-    interface
-        .id_to_index_to_vec()
-        .iter()
-        .for_each(|(id, index_to_vec)| {
-            let indexes = index_to_vec
-                .keys()
-                .map(|i| (*i as u8).to_string())
-                // .map(|i| i.to_string())
-                .collect::<Vec<_>>()
-                .join(", ");
-
-            contents += &format!("    \"{id}\": [{indexes}],\n");
-        });
-
-    contents += "  };\n}\n";
-
-    fs::write(path, contents)
-}

crates/brk_cli/src/lib.rs

@@ -9,8 +9,10 @@ use std::{
 };
 
 use bitcoincore_rpc::{self, RpcApi};
+use brk_binder::Bridge;
 use brk_bundler::bundle;
 use brk_computer::Computer;
+use brk_error::Result;
 use brk_indexer::Indexer;
 use brk_interface::Interface;
 use brk_parser::Parser;
@@ -18,12 +20,11 @@ use brk_server::{Server, VERSION};
 use log::info;
 use vecdb::Exit;
 
-mod bridge;
 mod config;
 mod paths;
 mod website;
 
-use crate::{bridge::Bridge, config::Config, paths::*};
+use crate::{config::Config, paths::*};
 
 pub fn main() -> color_eyre::Result<()> {
     color_eyre::install()?;
@@ -51,97 +52,116 @@ pub fn run() -> color_eyre::Result<()> {
 
     let mut indexer = Indexer::forced_import(&config.brkdir())?;
 
-    let wait_for_synced_node = |rpc_client: &bitcoincore_rpc::Client| -> color_eyre::Result<()> {
-        let is_synced = || -> color_eyre::Result<bool> {
-            let info = rpc_client.get_blockchain_info()?;
-            Ok(info.headers == info.blocks)
-        };
-
-        if !is_synced()? {
-            info!("Waiting for node to be synced...");
-            while !is_synced()? {
-                sleep(Duration::from_secs(1))
-            }
-        }
-
-        Ok(())
-    };
-
     let mut computer = Computer::forced_import(&config.brkdir(), &indexer, config.fetcher())?;
 
-    tokio::runtime::Builder::new_multi_thread()
-        .enable_all()
-        .build()?
-        .block_on(async {
-            let interface = Interface::build(&parser, &indexer, &computer);
+    let interface = Interface::build(&parser, &indexer, &computer);
 
-            let website = config.website();
+    let website = config.website();
 
-            let downloads_path = config.downloads_dir();
+    let downloads_path = config.downloads_dir();
 
-            let bundle_path = if website.is_some() {
-                let websites_dev_path = Path::new("../../websites");
+    let future = async move {
+        let bundle_path = if website.is_some() {
+            let websites_dev_path = Path::new("../../websites");
+            let modules_dev_path = Path::new("../../modules");
 
-                let websites_path = if fs::exists(websites_dev_path)? {
-                    websites_dev_path.to_path_buf()
-                } else {
-                    let downloaded_websites_path =
-                        downloads_path.join(format!("brk-{VERSION}")).join("websites");
+            let websites_path;
+            let modules_path;
 
-                    if !fs::exists(&downloaded_websites_path)? {
-                        info!("Downloading websites from Github...");
-
-                        let url = format!(
-                            "https://github.com/bitcoinresearchkit/brk/archive/refs/tags/v{VERSION}.zip",
-                        );
-
-                        let response = minreq::get(url).send()?;
-                        let bytes = response.as_bytes();
-                        let cursor = Cursor::new(bytes);
-
-                        let mut zip = zip::ZipArchive::new(cursor).unwrap();
-
-                        zip.extract(downloads_path).unwrap();
-                    }
-
-                    downloaded_websites_path
-                };
-
-                interface.generate_bridge_files(website, websites_path.as_path())?;
-
-                Some(bundle(&websites_path, website.to_folder_name(), true).await?)
+            if fs::exists(websites_dev_path)? && fs::exists(modules_dev_path)? {
+                websites_path = websites_dev_path.to_path_buf();
+                modules_path = modules_dev_path.to_path_buf();
             } else {
-                None
-            };
+                let downloaded_brk_path = downloads_path.join(format!("brk-{VERSION}"));
 
-            let server = Server::new(
-                interface,
-                bundle_path,
-            );
+                let downloaded_websites_path = downloaded_brk_path.join("websites");
+                let downloaded_modules_path = downloaded_brk_path.join("modules");
 
-            tokio::spawn(async move {
-                server.serve(true).await.unwrap();
-            });
+                if !fs::exists(&downloaded_websites_path)? {
+                    info!("Downloading source from Github...");
 
-            sleep(Duration::from_secs(1));
+                    let url = format!(
+                        "https://github.com/bitcoinresearchkit/brk/archive/refs/tags/v{VERSION}.zip",
+                    );
 
-            loop {
-                wait_for_synced_node(rpc)?;
+                    let response = minreq::get(url).send()?;
+                    let bytes = response.as_bytes();
+                    let cursor = Cursor::new(bytes);
 
-                let block_count = rpc.get_block_count()?;
+                    let mut zip = zip::ZipArchive::new(cursor).unwrap();
 
-                info!("{} blocks found.", block_count + 1);
-
-                let starting_indexes =
-                    indexer.index(&parser, rpc, &exit, config.check_collisions()).unwrap();
-
-                computer.compute(&indexer, starting_indexes, &parser, &exit).unwrap();
-
-                info!("Waiting for new blocks...");
-
-                while block_count == rpc.get_block_count()? {
-                    sleep(Duration::from_secs(1))
+                    zip.extract(downloads_path).unwrap();
                 }
+
+                websites_path = downloaded_websites_path;
+                modules_path = downloaded_modules_path;
             }
-        })
+
+            interface.generate_js_files(&modules_path)?;
+
+            Some(
+                bundle(
+                    &modules_path,
+                    &websites_path,
+                    website.to_folder_name(),
+                    true,
+                )
+                .await?,
+            )
+        } else {
+            None
+        };
+
+        let server = Server::new(interface, bundle_path);
+
+        tokio::spawn(async move {
+            server.serve(true).await.unwrap();
+        });
+
+        Ok(()) as Result<()>
+    };
+
+    let runtime = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()?;
+
+    let _handle = runtime.spawn(future);
+
+    loop {
+        wait_for_synced_node(rpc)?;
+
+        let block_count = rpc.get_block_count()?;
+
+        info!("{} blocks found.", block_count + 1);
+
+        let starting_indexes = indexer
+            .index(&parser, rpc, &exit, config.check_collisions())
+            .unwrap();
+
+        computer
+            .compute(&indexer, starting_indexes, &parser, &exit)
+            .unwrap();
+
+        info!("Waiting for new blocks...");
+
+        while block_count == rpc.get_block_count()? {
+            sleep(Duration::from_secs(1))
+        }
+    }
+}
+
+fn wait_for_synced_node(rpc_client: &bitcoincore_rpc::Client) -> color_eyre::Result<()> {
+    let is_synced = || -> color_eyre::Result<bool> {
+        let info = rpc_client.get_blockchain_info()?;
+        Ok(info.headers == info.blocks)
+    };
+
+    if !is_synced()? {
+        info!("Waiting for node to sync...");
+        while !is_synced()? {
+            sleep(Duration::from_secs(1))
+        }
+    }
+
+    Ok(())
 }

crates/brk_computer/src/blks.rs

@@ -3,7 +3,7 @@ use std::path::Path;
 use brk_error::Result;
 use brk_indexer::Indexer;
 use brk_parser::Parser;
-use brk_structs::{BlkPosition, Height, StoredU32, TxIndex, Version};
+use brk_structs::{BlkPosition, Height, TxIndex, Version};
 use vecdb::{
     AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CompressedVec, Database, Exit,
     GenericStoredVec, PAGE_SIZE, VecIterator,
@@ -16,9 +16,7 @@ pub struct Vecs {
     db: Database,
 
     pub height_to_position: CompressedVec<Height, BlkPosition>,
-    pub height_to_len: CompressedVec<Height, StoredU32>,
     pub txindex_to_position: CompressedVec<TxIndex, BlkPosition>,
-    pub txindex_to_len: CompressedVec<TxIndex, StoredU32>,
 }
 
 impl Vecs {
@@ -34,13 +32,11 @@ impl Vecs {
                 "position",
                 version + Version::TWO,
             )?,
-            height_to_len: CompressedVec::forced_import(&db, "len", version + Version::TWO)?,
             txindex_to_position: CompressedVec::forced_import(
                 &db,
                 "position",
                 version + Version::TWO,
             )?,
-            txindex_to_len: CompressedVec::forced_import(&db, "len", version + Version::TWO)?,
 
             db,
         };
@@ -104,9 +100,6 @@ impl Vecs {
                     exit,
                 )?;
 
-                self.height_to_len
-                    .forced_push_at(height, block.metadata().len().into(), exit)?;
-
                 let txindex = height_to_first_txindex_iter.unwrap_get_inner(height);
 
                 block.tx_metadata().iter().enumerate().try_for_each(
@@ -117,8 +110,6 @@ impl Vecs {
                             metadata.position(),
                             exit,
                         )?;
-                        self.txindex_to_len
-                            .forced_push_at(txindex, metadata.len().into(), exit)?;
                         Ok(())
                     },
                 )?;
@@ -126,18 +117,14 @@ impl Vecs {
                 if *height % 1_000 == 0 {
                     let _lock = exit.lock();
                     self.height_to_position.flush()?;
-                    self.height_to_len.flush()?;
                     self.txindex_to_position.flush()?;
-                    self.txindex_to_len.flush()?;
                 }
                 Ok(())
             })?;
 
         let _lock = exit.lock();
         self.height_to_position.flush()?;
-        self.height_to_len.flush()?;
         self.txindex_to_position.flush()?;
-        self.txindex_to_len.flush()?;
 
         Ok(())
     }
@@ -146,9 +133,7 @@ impl Vecs {
         Box::new(
             [
                 &self.height_to_position as &dyn AnyCollectableVec,
-                &self.height_to_len,
                 &self.txindex_to_position,
-                &self.txindex_to_len,
             ]
             .into_iter(),
         )

crates/brk_computer/src/stateful/common.rs

@@ -24,7 +24,6 @@ pub struct Vecs {
     pub height_to_supply: EagerVec<Height, Sats>,
     pub height_to_utxo_count: EagerVec<Height, StoredU64>,
     // Single
-    pub dateindex_to_supply_breakeven: Option<EagerVec<DateIndex, Sats>>,
     pub dateindex_to_supply_in_loss: Option<EagerVec<DateIndex, Sats>>,
     pub dateindex_to_supply_in_profit: Option<EagerVec<DateIndex, Sats>>,
     pub dateindex_to_unrealized_loss: Option<EagerVec<DateIndex, Dollars>>,
@@ -35,7 +34,6 @@ pub struct Vecs {
     pub height_to_min_price_paid: Option<EagerVec<Height, Dollars>>,
     pub height_to_realized_loss: Option<EagerVec<Height, Dollars>>,
     pub height_to_realized_profit: Option<EagerVec<Height, Dollars>>,
-    pub height_to_supply_breakeven: Option<EagerVec<Height, Sats>>,
     pub height_to_supply_in_loss: Option<EagerVec<Height, Sats>>,
     pub height_to_supply_in_profit: Option<EagerVec<Height, Sats>>,
     pub height_to_unrealized_loss: Option<EagerVec<Height, Dollars>>,
@@ -129,24 +127,17 @@ pub struct Vecs {
     pub indexes_to_realized_profit_rel_to_realized_cap: Option<ComputedVecsFromHeight<StoredF32>>,
     pub indexes_to_realized_loss_rel_to_realized_cap: Option<ComputedVecsFromHeight<StoredF32>>,
     pub indexes_to_net_realized_pnl_rel_to_realized_cap: Option<ComputedVecsFromHeight<StoredF32>>,
-    pub height_to_supply_breakeven_value: Option<ComputedHeightValueVecs>,
     pub height_to_supply_in_loss_value: Option<ComputedHeightValueVecs>,
     pub height_to_supply_in_profit_value: Option<ComputedHeightValueVecs>,
-    pub indexes_to_supply_breakeven: Option<ComputedValueVecsFromDateIndex>,
     pub indexes_to_supply_in_loss: Option<ComputedValueVecsFromDateIndex>,
     pub indexes_to_supply_in_profit: Option<ComputedValueVecsFromDateIndex>,
-    pub height_to_supply_breakeven_rel_to_own_supply: Option<EagerVec<Height, StoredF64>>,
     pub height_to_supply_in_loss_rel_to_own_supply: Option<EagerVec<Height, StoredF64>>,
     pub height_to_supply_in_profit_rel_to_own_supply: Option<EagerVec<Height, StoredF64>>,
-    pub indexes_to_supply_breakeven_rel_to_own_supply: Option<ComputedVecsFromDateIndex<StoredF64>>,
     pub indexes_to_supply_in_loss_rel_to_own_supply: Option<ComputedVecsFromDateIndex<StoredF64>>,
     pub indexes_to_supply_in_profit_rel_to_own_supply: Option<ComputedVecsFromDateIndex<StoredF64>>,
     pub indexes_to_supply_rel_to_circulating_supply: Option<ComputedVecsFromHeight<StoredF64>>,
-    pub height_to_supply_breakeven_rel_to_circulating_supply: Option<EagerVec<Height, StoredF64>>,
     pub height_to_supply_in_loss_rel_to_circulating_supply: Option<EagerVec<Height, StoredF64>>,
     pub height_to_supply_in_profit_rel_to_circulating_supply: Option<EagerVec<Height, StoredF64>>,
-    pub indexes_to_supply_breakeven_rel_to_circulating_supply:
-        Option<ComputedVecsFromDateIndex<StoredF64>>,
     pub indexes_to_supply_in_loss_rel_to_circulating_supply:
         Option<ComputedVecsFromDateIndex<StoredF64>>,
     pub indexes_to_supply_in_profit_rel_to_circulating_supply:
@@ -191,16 +182,6 @@ impl Vecs {
             .unwrap()
         });
 
-        let dateindex_to_supply_breakeven = compute_dollars.then(|| {
-            EagerVec::forced_import(
-                db,
-                &suffix("supply_breakeven"),
-                version + Version::ZERO,
-                format,
-            )
-            .unwrap()
-        });
-
         let dateindex_to_supply_in_loss = compute_dollars.then(|| {
             EagerVec::forced_import(
                 db,
@@ -257,31 +238,6 @@ impl Vecs {
                 .unwrap()
             }),
             dateindex_to_supply_in_profit,
-            height_to_supply_breakeven: compute_dollars.then(|| {
-                EagerVec::forced_import(
-                    db,
-                    &suffix("supply_breakeven"),
-                    version + Version::ZERO,
-                    format,
-                )
-                .unwrap()
-            }),
-            indexes_to_supply_breakeven: compute_dollars.then(|| {
-                ComputedValueVecsFromDateIndex::forced_import(
-                    db,
-                    &suffix("supply_breakeven"),
-                    dateindex_to_supply_breakeven
-                        .as_ref()
-                        .map(|v| v.boxed_clone())
-                        .into(),
-                    version + Version::ZERO,
-                    VecBuilderOptions::default().add_last(),
-                    compute_dollars,
-                    indexes,
-                )
-                .unwrap()
-            }),
-            dateindex_to_supply_breakeven,
             height_to_supply_in_loss: compute_dollars.then(|| {
                 EagerVec::forced_import(
                     db,
@@ -1125,17 +1081,6 @@ impl Vecs {
                 )
                 .unwrap()
             }),
-            height_to_supply_breakeven_value: compute_dollars.then(|| {
-                ComputedHeightValueVecs::forced_import(
-                    db,
-                    &suffix("supply_breakeven"),
-                    Source::None,
-                    version + Version::ZERO,
-                    format,
-                    compute_dollars,
-                )
-                .unwrap()
-            }),
             height_to_supply_in_loss_value: compute_dollars.then(|| {
                 ComputedHeightValueVecs::forced_import(
                     db,
@@ -1158,15 +1103,6 @@ impl Vecs {
                 )
                 .unwrap()
             }),
-            height_to_supply_breakeven_rel_to_own_supply: compute_dollars.then(|| {
-                EagerVec::forced_import(
-                    db,
-                    &suffix("supply_breakeven_rel_to_own_supply"),
-                    version + Version::ONE,
-                    format,
-                )
-                .unwrap()
-            }),
             height_to_supply_in_loss_rel_to_own_supply: compute_dollars.then(|| {
                 EagerVec::forced_import(
                     db,
@@ -1185,17 +1121,6 @@ impl Vecs {
                 )
                 .unwrap()
             }),
-            indexes_to_supply_breakeven_rel_to_own_supply: compute_dollars.then(|| {
-                ComputedVecsFromDateIndex::forced_import(
-                    db,
-                    &suffix("supply_breakeven_rel_to_own_supply"),
-                    Source::Compute,
-                    version + Version::ONE,
-                    indexes,
-                    VecBuilderOptions::default().add_last(),
-                )
-                .unwrap()
-            }),
             indexes_to_supply_in_loss_rel_to_own_supply: compute_dollars.then(|| {
                 ComputedVecsFromDateIndex::forced_import(
                     db,
@@ -1229,17 +1154,6 @@ impl Vecs {
                 )
                 .unwrap()
             }),
-            height_to_supply_breakeven_rel_to_circulating_supply: (compute_rel_to_all
-                && compute_dollars)
-                .then(|| {
-                    EagerVec::forced_import(
-                        db,
-                        &suffix("supply_breakeven_rel_to_circulating_supply"),
-                        version + Version::ONE,
-                        format,
-                    )
-                    .unwrap()
-                }),
             height_to_supply_in_loss_rel_to_circulating_supply: (compute_rel_to_all
                 && compute_dollars)
                 .then(|| {
@@ -1262,19 +1176,6 @@ impl Vecs {
                     )
                     .unwrap()
                 }),
-            indexes_to_supply_breakeven_rel_to_circulating_supply: (compute_rel_to_all
-                && compute_dollars)
-                .then(|| {
-                    ComputedVecsFromDateIndex::forced_import(
-                        db,
-                        &suffix("supply_breakeven_rel_to_circulating_supply"),
-                        Source::Compute,
-                        version + Version::ONE,
-                        indexes,
-                        VecBuilderOptions::default().add_last(),
-                    )
-                    .unwrap()
-                }),
             indexes_to_supply_in_loss_rel_to_circulating_supply: (compute_rel_to_all
                 && compute_dollars)
                 .then(|| {
@@ -1406,9 +1307,6 @@ impl Vecs {
             self.height_to_supply_in_loss
                 .as_ref()
                 .map_or(usize::MAX, |v| v.len()),
-            self.height_to_supply_breakeven
-                .as_ref()
-                .map_or(usize::MAX, |v| v.len()),
             self.height_to_unrealized_profit
                 .as_ref()
                 .map_or(usize::MAX, |v| v.len()),
@@ -1551,17 +1449,6 @@ impl Vecs {
                 .validate_computed_version_or_reset(
                     base_version + height_to_supply_in_loss_inner_version,
                 )?;
-            let height_to_supply_breakeven_inner_version = self
-                .height_to_supply_breakeven
-                .as_ref()
-                .unwrap()
-                .inner_version();
-            self.height_to_supply_breakeven
-                .as_mut()
-                .unwrap()
-                .validate_computed_version_or_reset(
-                    base_version + height_to_supply_breakeven_inner_version,
-                )?;
             let height_to_unrealized_profit_inner_version = self
                 .height_to_unrealized_profit
                 .as_ref()
@@ -1606,17 +1493,6 @@ impl Vecs {
                 .validate_computed_version_or_reset(
                     base_version + dateindex_to_supply_in_loss_inner_version,
                 )?;
-            let dateindex_to_supply_breakeven_inner_version = self
-                .dateindex_to_supply_breakeven
-                .as_ref()
-                .unwrap()
-                .inner_version();
-            self.dateindex_to_supply_breakeven
-                .as_mut()
-                .unwrap()
-                .validate_computed_version_or_reset(
-                    base_version + dateindex_to_supply_breakeven_inner_version,
-                )?;
             let dateindex_to_unrealized_profit_inner_version = self
                 .dateindex_to_unrealized_profit
                 .as_ref()
@@ -1790,10 +1666,6 @@ impl Vecs {
             let (height_unrealized_state, date_unrealized_state) =
                 state.compute_unrealized_states(height_price, date_price.unwrap());
 
-            self.height_to_supply_breakeven
-                .as_mut()
-                .unwrap()
-                .forced_push_at(height, height_unrealized_state.supply_breakeven, exit)?;
             self.height_to_supply_in_profit
                 .as_mut()
                 .unwrap()
@@ -1814,10 +1686,6 @@ impl Vecs {
             if let Some(date_unrealized_state) = date_unrealized_state {
                 let dateindex = dateindex.unwrap();
 
-                self.dateindex_to_supply_breakeven
-                    .as_mut()
-                    .unwrap()
-                    .forced_push_at(dateindex, date_unrealized_state.supply_breakeven, exit)?;
                 self.dateindex_to_supply_in_profit
                     .as_mut()
                     .unwrap()
@@ -1877,10 +1745,6 @@ impl Vecs {
                 .as_mut()
                 .unwrap()
                 .safe_flush(exit)?;
-            self.height_to_supply_breakeven
-                .as_mut()
-                .unwrap()
-                .safe_flush(exit)?;
             self.height_to_unrealized_profit
                 .as_mut()
                 .unwrap()
@@ -1897,10 +1761,6 @@ impl Vecs {
                 .as_mut()
                 .unwrap()
                 .safe_flush(exit)?;
-            self.dateindex_to_supply_breakeven
-                .as_mut()
-                .unwrap()
-                .safe_flush(exit)?;
             self.dateindex_to_unrealized_profit
                 .as_mut()
                 .unwrap()
@@ -2085,18 +1945,6 @@ impl Vecs {
                         .as_slice(),
                     exit,
                 )?;
-            self.height_to_supply_breakeven
-                .as_mut()
-                .unwrap()
-                .compute_sum_of_others(
-                    starting_indexes.height,
-                    others
-                        .iter()
-                        .map(|v| v.height_to_supply_breakeven.as_ref().unwrap())
-                        .collect::<Vec<_>>()
-                        .as_slice(),
-                    exit,
-                )?;
             self.height_to_unrealized_profit
                 .as_mut()
                 .unwrap()
@@ -2145,18 +1993,6 @@ impl Vecs {
                         .as_slice(),
                     exit,
                 )?;
-            self.dateindex_to_supply_breakeven
-                .as_mut()
-                .unwrap()
-                .compute_sum_of_others(
-                    starting_indexes.dateindex,
-                    others
-                        .iter()
-                        .map(|v| v.dateindex_to_supply_breakeven.as_ref().unwrap())
-                        .collect::<Vec<_>>()
-                        .as_slice(),
-                    exit,
-                )?;
             self.dateindex_to_unrealized_profit
                 .as_mut()
                 .unwrap()
@@ -2588,15 +2424,6 @@ impl Vecs {
                     exit,
                     Some(self.dateindex_to_supply_in_loss.as_ref().unwrap()),
                 )?;
-            self.indexes_to_supply_breakeven
-                .as_mut()
-                .unwrap()
-                .compute_rest(
-                    price,
-                    starting_indexes,
-                    exit,
-                    Some(self.dateindex_to_supply_breakeven.as_ref().unwrap()),
-                )?;
             self.indexes_to_unrealized_profit
                 .as_mut()
                 .unwrap()
@@ -3109,15 +2936,6 @@ impl Vecs {
                     Ok(())
                 })?;
 
-            self.height_to_supply_breakeven_value
-                .as_mut()
-                .unwrap()
-                .compute_rest(
-                    price,
-                    starting_indexes,
-                    exit,
-                    Some(self.height_to_supply_breakeven.as_ref().unwrap()),
-                )?;
             self.height_to_supply_in_loss_value
                 .as_mut()
                 .unwrap()
@@ -3136,19 +2954,6 @@ impl Vecs {
                     exit,
                     Some(self.height_to_supply_in_profit.as_ref().unwrap()),
                 )?;
-            self.height_to_supply_breakeven_rel_to_own_supply
-                .as_mut()
-                .unwrap()
-                .compute_percentage(
-                    starting_indexes.height,
-                    &self
-                        .height_to_supply_breakeven_value
-                        .as_ref()
-                        .unwrap()
-                        .bitcoin,
-                    &self.height_to_supply_value.bitcoin,
-                    exit,
-                )?;
             self.height_to_supply_in_loss_rel_to_own_supply
                 .as_mut()
                 .unwrap()
@@ -3175,24 +2980,6 @@ impl Vecs {
                     &self.height_to_supply_value.bitcoin,
                     exit,
                 )?;
-            self.indexes_to_supply_breakeven_rel_to_own_supply
-                .as_mut()
-                .unwrap()
-                .compute_all(starting_indexes, exit, |v| {
-                    v.compute_percentage(
-                        starting_indexes.dateindex,
-                        self.indexes_to_supply_breakeven
-                            .as_ref()
-                            .unwrap()
-                            .bitcoin
-                            .dateindex
-                            .as_ref()
-                            .unwrap(),
-                        self.indexes_to_supply.bitcoin.dateindex.as_ref().unwrap(),
-                        exit,
-                    )?;
-                    Ok(())
-                })?;
             self.indexes_to_supply_in_loss_rel_to_own_supply
                 .as_mut()
                 .unwrap()
@@ -3283,20 +3070,11 @@ impl Vecs {
                     Ok(())
                 })?;
 
-            if let Some(height_to_supply_breakeven_rel_to_circulating_supply) = self
-                .height_to_supply_breakeven_rel_to_circulating_supply
+            if self
+                .height_to_supply_in_profit_rel_to_circulating_supply
                 .as_mut()
+                .is_some()
             {
-                height_to_supply_breakeven_rel_to_circulating_supply.compute_percentage(
-                    starting_indexes.height,
-                    &self
-                        .height_to_supply_breakeven_value
-                        .as_ref()
-                        .unwrap()
-                        .bitcoin,
-                    height_to_supply,
-                    exit,
-                )?;
                 self.height_to_supply_in_loss_rel_to_circulating_supply
                     .as_mut()
                     .unwrap()
@@ -3323,24 +3101,6 @@ impl Vecs {
                         height_to_supply,
                         exit,
                     )?;
-                self.indexes_to_supply_breakeven_rel_to_circulating_supply
-                    .as_mut()
-                    .unwrap()
-                    .compute_all(starting_indexes, exit, |v| {
-                        v.compute_percentage(
-                            starting_indexes.dateindex,
-                            self.indexes_to_supply_breakeven
-                                .as_ref()
-                                .unwrap()
-                                .bitcoin
-                                .dateindex
-                                .as_ref()
-                                .unwrap(),
-                            dateindex_to_supply,
-                            exit,
-                        )?;
-                        Ok(())
-                    })?;
                 self.indexes_to_supply_in_loss_rel_to_circulating_supply
                     .as_mut()
                     .unwrap()
@@ -3555,13 +3315,6 @@ impl Vecs {
                     .map(|v| v as &dyn AnyCollectableVec),
             ),
         );
-        iter = Box::new(
-            iter.chain(
-                self.dateindex_to_supply_breakeven
-                    .iter()
-                    .map(|v| v as &dyn AnyCollectableVec),
-            ),
-        );
         iter = Box::new(
             iter.chain(
                 self.dateindex_to_supply_in_loss
@@ -3695,34 +3448,6 @@ impl Vecs {
                     .map(|v| v as &dyn AnyCollectableVec),
             ),
         );
-        iter = Box::new(
-            iter.chain(
-                self.height_to_supply_breakeven
-                    .iter()
-                    .map(|v| v as &dyn AnyCollectableVec),
-            ),
-        );
-        iter = Box::new(
-            iter.chain(
-                self.height_to_supply_breakeven_rel_to_circulating_supply
-                    .iter()
-                    .map(|v| v as &dyn AnyCollectableVec),
-            ),
-        );
-        iter = Box::new(
-            iter.chain(
-                self.height_to_supply_breakeven_rel_to_own_supply
-                    .iter()
-                    .map(|v| v as &dyn AnyCollectableVec),
-            ),
-        );
-        iter = Box::new(
-            iter.chain(
-                self.height_to_supply_breakeven_value
-                    .iter()
-                    .flat_map(|v| v.iter_any_collectable()),
-            ),
-        );
         iter = Box::new(iter.chain(self.height_to_supply_half_value.iter_any_collectable()));
         iter = Box::new(
             iter.chain(
@@ -4064,27 +3789,6 @@ impl Vecs {
             ),
         );
         iter = Box::new(iter.chain(self.indexes_to_supply.iter_any_collectable()));
-        iter = Box::new(
-            iter.chain(
-                self.indexes_to_supply_breakeven
-                    .iter()
-                    .flat_map(|v| v.iter_any_collectable()),
-            ),
-        );
-        iter = Box::new(
-            iter.chain(
-                self.indexes_to_supply_breakeven_rel_to_circulating_supply
-                    .iter()
-                    .flat_map(|v| v.iter_any_collectable()),
-            ),
-        );
-        iter = Box::new(
-            iter.chain(
-                self.indexes_to_supply_breakeven_rel_to_own_supply
-                    .iter()
-                    .flat_map(|v| v.iter_any_collectable()),
-            ),
-        );
         iter = Box::new(iter.chain(self.indexes_to_supply_half.iter_any_collectable()));
         iter = Box::new(
             iter.chain(

crates/brk_computer/src/states/cohorts/common.rs

@@ -257,10 +257,11 @@ impl CohortState {
 
         let update_state =
             |price: Dollars, current_price: Dollars, sats: Sats, state: &mut UnrealizedState| {
-                match price.cmp(&current_price) {
-                    Ordering::Less => {
+                let cmp = price.cmp(&current_price);
+                match cmp {
+                    Ordering::Equal | Ordering::Less => {
                         state.supply_in_profit += sats;
-                        if price > Dollars::ZERO && current_price > Dollars::ZERO {
+                        if !cmp.is_eq() && price > Dollars::ZERO && current_price > Dollars::ZERO {
                             let diff = current_price.checked_sub(price).unwrap();
                             // Add back once in a while to verify, but generally not needed
                             // if diff <= Dollars::ZERO {
@@ -282,9 +283,6 @@ impl CohortState {
                             state.unrealized_loss += diff * sats;
                         }
                     }
-                    Ordering::Equal => {
-                        state.supply_breakeven += sats;
-                    }
                 }
             };
 

crates/brk_computer/src/states/unrealized.rs

@@ -3,7 +3,6 @@ use brk_structs::{Dollars, Sats};
 #[derive(Debug, Default, Clone)]
 pub struct UnrealizedState {
     pub supply_in_profit: Sats,
-    pub supply_breakeven: Sats,
     pub supply_in_loss: Sats,
     pub unrealized_profit: Dollars,
     pub unrealized_loss: Dollars,
@@ -12,7 +11,6 @@ pub struct UnrealizedState {
 impl UnrealizedState {
     pub const NAN: Self = Self {
         supply_in_profit: Sats::ZERO,
-        supply_breakeven: Sats::ZERO,
         supply_in_loss: Sats::ZERO,
         unrealized_profit: Dollars::NAN,
         unrealized_loss: Dollars::NAN,
@@ -20,7 +18,6 @@ impl UnrealizedState {
 
     pub const ZERO: Self = Self {
         supply_in_profit: Sats::ZERO,
-        supply_breakeven: Sats::ZERO,
         supply_in_loss: Sats::ZERO,
         unrealized_profit: Dollars::ZERO,
         unrealized_loss: Dollars::ZERO,

crates/brk_error/Cargo.toml

@@ -15,6 +15,6 @@ bitcoincore-rpc = { workspace = true }
 fjall = { workspace = true }
 jiff = { workspace = true }
 minreq = { workspace = true }
-serde_json = { workspace = true }
+sonic-rs = { workspace = true }
 vecdb = { workspace = true }
 zerocopy = { workspace = true }

crates/brk_error/src/lib.rs

@@ -19,7 +19,7 @@ pub enum Error {
     SystemTimeError(time::SystemTimeError),
     BitcoinConsensusEncode(bitcoin::consensus::encode::Error),
     BitcoinBip34Error(bitcoin::block::Bip34Error),
-    SerdeJson(serde_json::Error),
+    SonicRS(sonic_rs::Error),
     ZeroCopyError,
     Vecs(vecdb::Error),
 
@@ -49,9 +49,9 @@ impl From<time::SystemTimeError> for Error {
     }
 }
 
-impl From<serde_json::Error> for Error {
-    fn from(error: serde_json::Error) -> Self {
-        Self::SerdeJson(error)
+impl From<sonic_rs::Error> for Error {
+    fn from(error: sonic_rs::Error) -> Self {
+        Self::SonicRS(error)
     }
 }
 
@@ -126,7 +126,7 @@ impl fmt::Display for Error {
             Error::Jiff(error) => Display::fmt(&error, f),
             Error::Minreq(error) => Display::fmt(&error, f),
             Error::SeqDB(error) => Display::fmt(&error, f),
-            Error::SerdeJson(error) => Display::fmt(&error, f),
+            Error::SonicRS(error) => Display::fmt(&error, f),
             Error::SystemTimeError(error) => Display::fmt(&error, f),
             Error::VecDB(error) => Display::fmt(&error, f),
             Error::Vecs(error) => Display::fmt(&error, f),

crates/brk_fetcher/Cargo.toml

@@ -15,4 +15,4 @@ brk_logger = { workspace = true }
 brk_structs = { workspace = true }
 log = { workspace = true }
 minreq = { workspace = true }
-serde_json = { workspace = true }
+sonic-rs = { workspace = true }

crates/brk_fetcher/src/binance.rs

@@ -3,13 +3,12 @@ use std::{
     fs::{self, File},
     io::BufReader,
     path::{Path, PathBuf},
-    str::FromStr,
 };
 
 use brk_error::{Error, Result};
 use brk_structs::{Cents, OHLCCents, Timestamp};
 use log::info;
-use serde_json::Value;
+use sonic_rs::{JsonContainerTrait, JsonValueTrait, Value};
 
 use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, default_retry};
 
@@ -17,7 +16,7 @@ use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, default_retry};
 pub struct Binance {
     path: Option<PathBuf>,
     _1mn: Option<BTreeMap<Timestamp, OHLCCents>>,
-    pub _1d: Option<BTreeMap<Date, OHLCCents>>,
+    _1d: Option<BTreeMap<Date, OHLCCents>>,
     har: Option<BTreeMap<Timestamp, OHLCCents>>,
 }
 
@@ -69,11 +68,11 @@ impl Binance {
         info!("Fetching 1mn prices from Binance...");
 
         default_retry(|_| {
-            Self::json_to_timestamp_to_ohlc(
-                &minreq::get(Self::url("interval=1m&limit=1000"))
+            Self::json_to_timestamp_to_ohlc(&sonic_rs::from_str(
+                minreq::get(Self::url("interval=1m&limit=1000"))
                     .send()?
-                    .json()?,
-            )
+                    .as_str()?,
+            )?)
         })
     }
 
@@ -94,7 +93,9 @@ impl Binance {
         info!("Fetching daily prices from Binance...");
 
         default_retry(|_| {
-            Self::json_to_date_to_ohlc(&minreq::get(Self::url("interval=1d")).send()?.json()?)
+            Self::json_to_date_to_ohlc(&sonic_rs::from_str(
+                minreq::get(Self::url("interval=1d")).send()?.as_str()?,
+            )?)
         })
     }
 
@@ -119,7 +120,7 @@ impl Binance {
 
         let reader = BufReader::new(file);
 
-        let json: BTreeMap<String, Value> = if let Ok(json) = serde_json::from_reader(reader) {
+        let json: BTreeMap<String, Value> = if let Ok(json) = sonic_rs::from_reader(reader) {
             json
         } else {
             return Ok(Default::default());
@@ -129,7 +130,7 @@ impl Binance {
             .ok_or(Error::Str("Expect object to have log attribute"))?
             .as_object()
             .ok_or(Error::Str("Expect to be an object"))?
-            .get("entries")
+            .get(&"entries")
             .ok_or(Error::Str("Expect object to have entries"))?
             .as_array()
             .ok_or(Error::Str("Expect to be an array"))?
@@ -138,11 +139,11 @@ impl Binance {
                 entry
                     .as_object()
                     .unwrap()
-                    .get("request")
+                    .get(&"request")
                     .unwrap()
                     .as_object()
                     .unwrap()
-                    .get("url")
+                    .get(&"url")
                     .unwrap()
                     .as_str()
                     .unwrap()
@@ -152,14 +153,14 @@ impl Binance {
                 let response = entry
                     .as_object()
                     .unwrap()
-                    .get("response")
+                    .get(&"response")
                     .unwrap()
                     .as_object()
                     .unwrap();
 
-                let content = response.get("content").unwrap().as_object().unwrap();
+                let content = response.get(&"content").unwrap().as_object().unwrap();
 
-                let text = content.get("text");
+                let text = content.get(&"text");
 
                 if text.is_none() {
                     return Ok(BTreeMap::new());
@@ -167,7 +168,7 @@ impl Binance {
 
                 let text = text.unwrap().as_str().unwrap();
 
-                Self::json_to_timestamp_to_ohlc(&serde_json::Value::from_str(text).unwrap())
+                Self::json_to_timestamp_to_ohlc(&sonic_rs::from_str(text).unwrap())
             })
             .try_fold(BTreeMap::default(), |mut all, res| {
                 all.append(&mut res?);

crates/brk_fetcher/src/brk.rs

@@ -3,7 +3,7 @@ use std::collections::BTreeMap;
 use brk_error::{Error, Result};
 use brk_structs::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents};
 use log::info;
-use serde_json::Value;
+use sonic_rs::{JsonContainerTrait, JsonValueTrait, Value};
 
 use crate::{Close, Dollars, High, Low, Open, default_retry};
 
@@ -51,7 +51,7 @@ impl BRK {
                 height + CHUNK_SIZE
             );
 
-            let body: Value = minreq::get(url).send()?.json()?;
+            let body: Value = sonic_rs::from_str(minreq::get(url).send()?.as_str()?)?;
 
             body.as_array()
                 .ok_or(Error::Str("Expect to be an array"))?
@@ -96,7 +96,7 @@ impl BRK {
                 dateindex + CHUNK_SIZE
             );
 
-            let body: Value = minreq::get(url).send()?.json()?;
+            let body: Value = sonic_rs::from_str(minreq::get(url).send()?.as_str()?)?;
 
             body.as_array()
                 .ok_or(Error::Str("Expect to be an array"))?

crates/brk_fetcher/src/kraken.rs

@@ -3,7 +3,7 @@ use std::collections::BTreeMap;
 use brk_error::{Error, Result};
 use brk_structs::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Timestamp};
 use log::info;
-use serde_json::Value;
+use sonic_rs::{JsonContainerTrait, JsonValueTrait, Value};
 
 use crate::{Fetcher, default_retry};
 
@@ -36,7 +36,9 @@ impl Kraken {
         info!("Fetching 1mn prices from Kraken...");
 
         default_retry(|_| {
-            Self::json_to_timestamp_to_ohlc(&minreq::get(Self::url(1)).send()?.json()?)
+            Self::json_to_timestamp_to_ohlc(&sonic_rs::from_str(
+                minreq::get(Self::url(1)).send()?.as_str()?,
+            )?)
         })
     }
 
@@ -55,7 +57,11 @@ impl Kraken {
     pub fn fetch_1d() -> Result<BTreeMap<Date, OHLCCents>> {
         info!("Fetching daily prices from Kraken...");
 
-        default_retry(|_| Self::json_to_date_to_ohlc(&minreq::get(Self::url(1440)).send()?.json()?))
+        default_retry(|_| {
+            Self::json_to_date_to_ohlc(&sonic_rs::from_str(
+                minreq::get(Self::url(1440)).send()?.as_str()?,
+            )?)
+        })
     }
 
     fn json_to_timestamp_to_ohlc(json: &Value) -> Result<BTreeMap<Timestamp, OHLCCents>> {
@@ -73,11 +79,11 @@ impl Kraken {
     {
         json.as_object()
             .ok_or(Error::Str("Expect to be an object"))?
-            .get("result")
+            .get(&"result")
             .ok_or(Error::Str("Expect object to have result"))?
             .as_object()
             .ok_or(Error::Str("Expect to be an object"))?
-            .get("XXBTZUSD")
+            .get(&"XXBTZUSD")
             .ok_or(Error::Str("Expect to have XXBTZUSD"))?
             .as_array()
             .ok_or(Error::Str("Expect to be an array"))?

crates/brk_fetcher/src/retry.rs

@@ -22,6 +22,7 @@ where
         if i == retries || res.is_ok() {
             return res;
         } else {
+            let _ = dbg!(res);
             info!("Failed, waiting {sleep_in_s} seconds...");
             sleep(Duration::from_secs(sleep_in_s));
         }

crates/brk_indexer/src/indexes.rs

@@ -1,6 +1,5 @@
 use bitcoincore_rpc::Client;
 use brk_error::{Error, Result};
-use brk_parser::NUMBER_OF_UNSAFE_BLOCKS;
 use brk_structs::{
     BlockHash, CheckedSub, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
     OutputType, P2AAddressIndex, P2MSOutputIndex, P2PK33AddressIndex, P2PK65AddressIndex,
@@ -13,6 +12,8 @@ use vecdb::{
 
 use crate::{Stores, Vecs};
 
+const NUMBER_OF_UNSAFE_BLOCKS: usize = 100;
+
 #[derive(Debug, Default, Clone)]
 pub struct Indexes {
     pub emptyoutputindex: EmptyOutputIndex,
@@ -34,7 +35,7 @@ pub struct Indexes {
 }
 
 impl Indexes {
-    pub fn typeindex(&self, outputtype: OutputType) -> TypeIndex {
+    pub fn to_typeindex(&self, outputtype: OutputType) -> TypeIndex {
         match outputtype {
             OutputType::Empty => *self.emptyoutputindex,
             OutputType::OpReturn => *self.opreturnindex,

crates/brk_indexer/src/lib.rs

@@ -1,10 +1,9 @@
 #![doc = include_str!("../README.md")]
 
-use std::{collections::BTreeMap, path::Path, str::FromStr, thread, time::Instant};
+use std::{collections::BTreeMap, path::Path, str::FromStr, time::Instant};
 
 use bitcoin::{Transaction, TxIn, TxOut};
 use brk_error::{Error, Result};
-
 use brk_parser::Parser;
 use brk_store::AnyStore;
 use brk_structs::{
@@ -24,9 +23,8 @@ pub use stores::*;
 pub use vecs::*;
 
 // One version for all data sources
-// Increment on change OR addition
+// Increment on **change _OR_ addition**
 const VERSION: Version = Version::new(21);
-
 const SNAPSHOT_BLOCK_RANGE: usize = 1_000;
 const COLLISIONS_CHECKED_UP_TO: Height = Height::new(909_150);
 
@@ -92,6 +90,7 @@ impl Indexer {
         let export =
             |stores: &mut Stores, vecs: &mut Vecs, height: Height, exit: &Exit| -> Result<()> {
                 info!("Exporting...");
+                // std::process::exit(0);
                 let _lock = exit.lock();
                 let i = Instant::now();
                 stores.commit(height).unwrap();
@@ -184,7 +183,7 @@ impl Indexer {
                 let p2aaddressindex_to_p2abytes_reader = p2aaddressindex_to_p2abytes_reader_opt.as_ref().unwrap();
 
                 // Used to check rapidhash collisions
-                let check_collisions = check_collisions && height > COLLISIONS_CHECKED_UP_TO ;
+                let check_collisions = check_collisions && height > COLLISIONS_CHECKED_UP_TO;
 
                 let blockhash_prefix = BlockHashPrefix::from(blockhash);
 
@@ -215,275 +214,241 @@ impl Indexer {
                 vecs.height_to_total_size.push_if_needed(height, block.total_size().into())?;
                 vecs.height_to_weight.push_if_needed(height, block.weight().into())?;
 
-                let (
-                    txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle,
-                    input_source_vec_handle,
-                    outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle,
-                ) = thread::scope(|scope| {
-                    let txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle =
-                        scope.spawn(|| -> Result<_> {
-                            block
-                                .txdata
-                                .iter()
-                                .enumerate()
-                                .map(|(index, tx)| {
-                                    let txid = Txid::from(tx.compute_txid());
+                let txid_prefix_to_txid_and_block_txindex_and_prev_txindex = block
+                    .txdata
+                    .par_iter()
+                    .enumerate()
+                    .map(|(index, tx)| {
+                        let txid = Txid::from(tx.compute_txid());
 
-                                    let txid_prefix = TxidPrefix::from(&txid);
+                        let txid_prefix = TxidPrefix::from(&txid);
 
-                                    let prev_txindex_opt =
-                                        if check_collisions && stores.txidprefix_to_txindex.needs(height) {
-                                            // Should only find collisions for two txids (duplicates), see below
-                                            stores.txidprefix_to_txindex.get(&txid_prefix)?.map(|v| *v)
-                                        } else {
-                                            None
-                                        };
+                        let prev_txindex_opt =
+                            if check_collisions && stores.txidprefix_to_txindex.needs(height) {
+                                // Should only find collisions for two txids (duplicates), see below
+                                stores.txidprefix_to_txindex.get(&txid_prefix)?.map(|v| *v)
+                            } else {
+                                None
+                            };
 
-                                    Ok((txid_prefix, (tx, txid, TxIndex::from(index), prev_txindex_opt)))
-                                })
-                                .collect::<Result<BTreeMap<_, _>>>()
-                        });
+                        Ok((txid_prefix, (tx, txid, TxIndex::from(index), prev_txindex_opt)))
+                    })
+                    .collect::<Result<BTreeMap<_, _>>>()?;
 
-                    let input_source_vec_handle = scope.spawn(|| {
-                        let inputs = block
-                            .txdata
+                let inputs = block
+                    .txdata
+                    .iter()
+                    .enumerate()
+                    .flat_map(|(index, tx)| {
+                        tx.input
                             .iter()
                             .enumerate()
-                            .flat_map(|(index, tx)| {
-                                tx.input
-                                    .iter()
-                                    .enumerate()
-                                    .map(move |(vin, txin)| (TxIndex::from(index), Vin::from(vin), txin, tx))
-                            })
-                            .collect::<Vec<_>>();
+                            .map(move |(vin, txin)| (TxIndex::from(index), Vin::from(vin), txin, tx))
+                    })
+                    .collect::<Vec<_>>();
 
-                        inputs
-                            .into_par_iter()
+                let input_source_vec = inputs
+                    .into_par_iter()
+                    .enumerate()
+                    .map(|(block_inputindex, (block_txindex, vin, txin, tx))| -> Result<(InputIndex, InputSource)> {
+                        let txindex = idxs.txindex + block_txindex;
+                        let inputindex = idxs.inputindex + InputIndex::from(block_inputindex);
+
+                        let outpoint = txin.previous_output;
+                        let txid = Txid::from(outpoint.txid);
+
+                        if tx.is_coinbase() {
+                            return Ok((inputindex, InputSource::SameBlock((tx, txindex, txin, vin))));
+                        }
+
+                        let prev_txindex = if let Some(txindex) = stores
+                            .txidprefix_to_txindex
+                            .get(&TxidPrefix::from(&txid))?
+                            .map(|v| *v)
+                            .and_then(|txindex| {
+                                // Checking if not finding txindex from the future
+                                (txindex < idxs.txindex).then_some(txindex)
+                            }) {
+                            txindex
+                        } else {
+                            // dbg!(indexes.txindex + block_txindex, txindex, txin, vin);
+                            return Ok((inputindex, InputSource::SameBlock((tx, txindex, txin, vin))));
+                        };
+
+                        let vout = Vout::from(outpoint.vout);
+
+                        let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, txindex_to_first_outputindex_reader)?
+                            .ok_or(Error::Str("Expect outputindex to not be none"))
+                            .inspect_err(|_| {
+                                dbg!(outpoint.txid, prev_txindex, vout);
+                            })?.into_owned()
+                            + vout;
+
+                        Ok((inputindex, InputSource::PreviousBlock((
+                            vin,
+                            txindex,
+                            outputindex,
+                        ))))
+                    })
+                    .try_fold(BTreeMap::new, |mut map, tuple| -> Result<_> {
+                        let (key, value) = tuple?;
+                        map.insert(key, value);
+                        Ok(map)
+                    })
+                    .try_reduce(BTreeMap::new, |mut map, mut map2| {
+                        if map.len() > map2.len() {
+                            map.append(&mut map2);
+                            Ok(map)
+                        } else {
+                            map2.append(&mut map);
+                            Ok(map2)
+                        }
+                    })?;
+
+                let outputs = block
+                    .txdata
+                    .iter()
+                    .enumerate()
+                    .flat_map(|(index, tx)| {
+                        tx.output
+                            .iter()
                             .enumerate()
-                            .map(|(block_inputindex, (block_txindex, vin, txin, tx))| -> Result<(InputIndex, InputSource)> {
-                                let txindex = idxs.txindex + block_txindex;
-                                let inputindex = idxs.inputindex + InputIndex::from(block_inputindex);
+                            .map(move |(vout, txout)| (TxIndex::from(index), Vout::from(vout), txout, tx))
+                    }).collect::<Vec<_>>();
 
-                                let outpoint = txin.previous_output;
-                                let txid = Txid::from(outpoint.txid);
+                let outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt = outputs.into_par_iter()
+                    .enumerate()
+                    .map(
+                        #[allow(clippy::type_complexity)]
+                        |(block_outputindex, (block_txindex, vout, txout, tx))| -> Result<(
+                            OutputIndex,
+                            (
+                                &TxOut,
+                                TxIndex,
+                                Vout,
+                                OutputType,
+                                Result<AddressBytes>,
+                                Option<TypeIndex>,
+                                &Transaction,
+                            ),
+                        )> {
+                            let txindex = idxs.txindex + block_txindex;
+                            let outputindex = idxs.outputindex + OutputIndex::from(block_outputindex);
 
-                                if tx.is_coinbase() {
-                                    return Ok((inputindex, InputSource::SameBlock((tx, txindex, txin, vin))));
-                                }
+                            let script = &txout.script_pubkey;
 
-                                let prev_txindex = if let Some(txindex) = stores
-                                    .txidprefix_to_txindex
-                                    .get(&TxidPrefix::from(&txid))?
-                                    .map(|v| *v)
-                                    .and_then(|txindex| {
-                                        // Checking if not finding txindex from the future
-                                        (txindex < idxs.txindex).then_some(txindex)
-                                    }) {
-                                    txindex
-                                } else {
-                                    // dbg!(indexes.txindex + block_txindex, txindex, txin, vin);
-                                    return Ok((inputindex, InputSource::SameBlock((tx, txindex, txin, vin))));
-                                };
+                            let outputtype = OutputType::from(script);
 
-                                let vout = Vout::from(outpoint.vout);
-
-                                let outputindex = vecs.txindex_to_first_outputindex.get_or_read(prev_txindex, txindex_to_first_outputindex_reader)?
-                                    .ok_or(Error::Str("Expect outputindex to not be none"))
-                                    .inspect_err(|_| {
-                                        dbg!(outpoint.txid, prev_txindex, vout);
-                                    })?.into_owned()
-                                    + vout;
-
-                                Ok((inputindex, InputSource::PreviousBlock((
-                                    vin,
-                                    txindex,
-                                    outputindex,
-                                ))))
-                            })
-                            .try_fold(BTreeMap::new, |mut map, tuple| -> Result<_> {
-                                let (key, value) = tuple?;
-                                map.insert(key, value);
-                                Ok(map)
-                            })
-                            .try_reduce(BTreeMap::new, |mut map, mut map2| {
-                                if map.len() > map2.len() {
-                                    map.append(&mut map2);
-                                    Ok(map)
-                                } else {
-                                    map2.append(&mut map);
-                                    Ok(map2)
-                                }
-                            })
-                    });
-
-                    let outputs = block
-                        .txdata
-                        .iter()
-                        .enumerate()
-                        .flat_map(|(index, tx)| {
-                            tx.output
-                                .iter()
-                                .enumerate()
-                                .map(move |(vout, txout)| (TxIndex::from(index), Vout::from(vout), txout, tx))
-                        }).collect::<Vec<_>>();
-
-                    let outputindex_to_txout_outputtype_addressbytes_res_addressindex = outputs.into_par_iter()
-                        .enumerate()
-                        .map(
-                            #[allow(clippy::type_complexity)]
-                            |(block_outputindex, (block_txindex, vout, txout, tx))| -> Result<(
-                                OutputIndex,
-                                (
-                                    &TxOut,
-                                    TxIndex,
-                                    Vout,
-                                    OutputType,
-                                    Result<AddressBytes>,
-                                    Option<TypeIndex>,
-                                    &Transaction,
-                                ),
-                            )> {
-                                let txindex = idxs.txindex + block_txindex;
-                                let outputindex = idxs.outputindex + OutputIndex::from(block_outputindex);
-
-                                let script = &txout.script_pubkey;
-
-                                let outputtype = OutputType::from(script);
-
-                                let address_bytes_res =
-                                    AddressBytes::try_from((script, outputtype)).inspect_err(|_| {
-                                        // dbg!(&txout, height, txi, &tx.compute_txid());
-                                    });
-
-                                let typeindex_opt = address_bytes_res.as_ref().ok().and_then(|addressbytes| {
-                                    stores
-                                        .addressbyteshash_to_typeindex
-                                        .get(&AddressBytesHash::from((addressbytes, outputtype)))
-                                        .unwrap()
-                                        .map(|v| *v)
-                                        // Checking if not in the future
-                                        .and_then(|typeindex_local| {
-                                            (typeindex_local < idxs.typeindex(outputtype)).then_some(typeindex_local)
-                                        })
+                            let address_bytes_res =
+                                AddressBytes::try_from((script, outputtype)).inspect_err(|_| {
+                                    // dbg!(&txout, height, txi, &tx.compute_txid());
                                 });
 
-                                if let Some(Some(typeindex)) = check_collisions.then_some(typeindex_opt) {
-                                    let addressbytes = address_bytes_res.as_ref().unwrap();
+                            let typeindex_opt = address_bytes_res.as_ref().ok().and_then(|addressbytes| {
+                                stores
+                                    .addressbyteshash_to_typeindex
+                                    .get(&AddressBytesHash::from((addressbytes, outputtype)))
+                                    .unwrap()
+                                    .map(|v| *v)
+                                    // Checking if not in the future
+                                    .and_then(|typeindex_local| {
+                                        (typeindex_local < idxs.to_typeindex(outputtype)).then_some(typeindex_local)
+                                    })
+                            });
 
-                                    let prev_addressbytes_opt = match outputtype {
-                                        OutputType::P2PK65 => vecs
-                                            .p2pk65addressindex_to_p2pk65bytes
-                                            .get_or_read(typeindex.into(), p2pk65addressindex_to_p2pk65bytes_reader)?
-                                            .map(|v| AddressBytes::from(v.into_owned())),
-                                        OutputType::P2PK33 => vecs
-                                            .p2pk33addressindex_to_p2pk33bytes
-                                            .get_or_read(typeindex.into(), p2pk33addressindex_to_p2pk33bytes_reader)?
-                                            .map(|v| AddressBytes::from(v.into_owned())),
-                                        OutputType::P2PKH => vecs
-                                            .p2pkhaddressindex_to_p2pkhbytes
-                                            .get_or_read(typeindex.into(), p2pkhaddressindex_to_p2pkhbytes_reader)?
-                                            .map(|v| AddressBytes::from(v.into_owned())),
-                                        OutputType::P2SH => vecs
-                                            .p2shaddressindex_to_p2shbytes
-                                            .get_or_read(typeindex.into(), p2shaddressindex_to_p2shbytes_reader)?
-                                            .map(|v| AddressBytes::from(v.into_owned())),
-                                        OutputType::P2WPKH => vecs
-                                            .p2wpkhaddressindex_to_p2wpkhbytes
-                                            .get_or_read(typeindex.into(), p2wpkhaddressindex_to_p2wpkhbytes_reader)?
-                                            .map(|v| AddressBytes::from(v.into_owned())),
-                                        OutputType::P2WSH => vecs
-                                            .p2wshaddressindex_to_p2wshbytes
-                                            .get_or_read(typeindex.into(), p2wshaddressindex_to_p2wshbytes_reader)?
-                                            .map(|v| AddressBytes::from(v.into_owned())),
-                                        OutputType::P2TR => vecs
-                                            .p2traddressindex_to_p2trbytes
-                                            .get_or_read(typeindex.into(), p2traddressindex_to_p2trbytes_reader)?
-                                            .map(|v| AddressBytes::from(v.into_owned())),
-                                        OutputType::P2A => vecs
-                                            .p2aaddressindex_to_p2abytes
-                                            .get_or_read(typeindex.into(), p2aaddressindex_to_p2abytes_reader)?
-                                            .map(|v| AddressBytes::from(v.into_owned())),
-                                        _ => {
-                                            unreachable!()
-                                        }
-                                    };
-                                    let prev_addressbytes =
-                                        prev_addressbytes_opt.as_ref().ok_or(Error::Str("Expect to have addressbytes"))?;
+                            if let Some(Some(typeindex)) = check_collisions.then_some(typeindex_opt) {
+                                let addressbytes = address_bytes_res.as_ref().unwrap();
 
-                                    if stores.addressbyteshash_to_typeindex.needs(height)
-                                            && prev_addressbytes != addressbytes
-                                    {
-                                        let txid = tx.compute_txid();
-                                        dbg!(
-                                            height,
-                                            txid,
-                                            vout,
-                                            block_txindex,
-                                            outputtype,
-                                            prev_addressbytes,
-                                            addressbytes,
-                                            &idxs,
-                                            typeindex,
-                                            typeindex,
-                                            txout,
-                                            AddressBytesHash::from((addressbytes, outputtype)),
-                                        );
-                                        panic!()
+                                let prev_addressbytes_opt = match outputtype {
+                                    OutputType::P2PK65 => vecs
+                                        .p2pk65addressindex_to_p2pk65bytes
+                                        .get_or_read(typeindex.into(), p2pk65addressindex_to_p2pk65bytes_reader)?
+                                        .map(|v| AddressBytes::from(v.into_owned())),
+                                    OutputType::P2PK33 => vecs
+                                        .p2pk33addressindex_to_p2pk33bytes
+                                        .get_or_read(typeindex.into(), p2pk33addressindex_to_p2pk33bytes_reader)?
+                                        .map(|v| AddressBytes::from(v.into_owned())),
+                                    OutputType::P2PKH => vecs
+                                        .p2pkhaddressindex_to_p2pkhbytes
+                                        .get_or_read(typeindex.into(), p2pkhaddressindex_to_p2pkhbytes_reader)?
+                                        .map(|v| AddressBytes::from(v.into_owned())),
+                                    OutputType::P2SH => vecs
+                                        .p2shaddressindex_to_p2shbytes
+                                        .get_or_read(typeindex.into(), p2shaddressindex_to_p2shbytes_reader)?
+                                        .map(|v| AddressBytes::from(v.into_owned())),
+                                    OutputType::P2WPKH => vecs
+                                        .p2wpkhaddressindex_to_p2wpkhbytes
+                                        .get_or_read(typeindex.into(), p2wpkhaddressindex_to_p2wpkhbytes_reader)?
+                                        .map(|v| AddressBytes::from(v.into_owned())),
+                                    OutputType::P2WSH => vecs
+                                        .p2wshaddressindex_to_p2wshbytes
+                                        .get_or_read(typeindex.into(), p2wshaddressindex_to_p2wshbytes_reader)?
+                                        .map(|v| AddressBytes::from(v.into_owned())),
+                                    OutputType::P2TR => vecs
+                                        .p2traddressindex_to_p2trbytes
+                                        .get_or_read(typeindex.into(), p2traddressindex_to_p2trbytes_reader)?
+                                        .map(|v| AddressBytes::from(v.into_owned())),
+                                    OutputType::P2A => vecs
+                                        .p2aaddressindex_to_p2abytes
+                                        .get_or_read(typeindex.into(), p2aaddressindex_to_p2abytes_reader)?
+                                        .map(|v| AddressBytes::from(v.into_owned())),
+                                    _ => {
+                                        unreachable!()
                                     }
-                                }
+                                };
+                                let prev_addressbytes =
+                                    prev_addressbytes_opt.as_ref().ok_or(Error::Str("Expect to have addressbytes"))?;
 
-                                Ok((
-                                    outputindex,
-                                    (
-                                        txout,
-                                        txindex,
+                                if stores.addressbyteshash_to_typeindex.needs(height)
+                                        && prev_addressbytes != addressbytes
+                                {
+                                    let txid = tx.compute_txid();
+                                    dbg!(
+                                        height,
+                                        txid,
                                         vout,
+                                        block_txindex,
                                         outputtype,
-                                        address_bytes_res,
-                                        typeindex_opt,
-                                        tx,
-                                    ),
-                                ))
-                            },
-                        )
-                        .try_fold(BTreeMap::new, |mut map, tuple| -> Result<_> {
-                            let (key, value) = tuple?;
-                            map.insert(key, value);
-                            Ok(map)
-                        })
-                        .try_reduce(BTreeMap::new, |mut map, mut map2| {
-                            if map.len() > map2.len() {
-                                map.append(&mut map2);
-                                Ok(map)
-                            } else {
-                                map2.append(&mut map);
-                                Ok(map2)
+                                        prev_addressbytes,
+                                        addressbytes,
+                                        &idxs,
+                                        typeindex,
+                                        typeindex,
+                                        txout,
+                                        AddressBytesHash::from((addressbytes, outputtype)),
+                                    );
+                                    panic!()
+                                }
                             }
-                        });
 
-                    (
-                        txid_prefix_to_txid_and_block_txindex_and_prev_txindex_handle.join(),
-                        input_source_vec_handle.join(),
-                        outputindex_to_txout_outputtype_addressbytes_res_addressindex,
+                            Ok((
+                                outputindex,
+                                (
+                                    txout,
+                                    txindex,
+                                    vout,
+                                    outputtype,
+                                    address_bytes_res,
+                                    typeindex_opt,
+                                    tx,
+                                ),
+                            ))
+                        },
                     )
-                });
-
-                let txid_prefix_to_txid_and_block_txindex_and_prev_txindex =
-                    txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle
-                        .map_err(|_|
-                            Error::Str("Expect txid_prefix_to_txid_and_block_txindex_and_prev_txindex_join_handle to join")
-                        )??;
-
-                let input_source_vec = input_source_vec_handle
-                    .map_err(|_|
-                        Error::Str("Export input_source_vec_handle to join")
-                    )??;
-
-                let outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt =
-                    outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle
-                        .map_err(|_|
-                            Error::Str("Expect outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt_handle to join")
-                        )?;
+                    .try_fold(BTreeMap::new, |mut map, tuple| -> Result<_> {
+                        let (key, value) = tuple?;
+                        map.insert(key, value);
+                        Ok(map)
+                    })
+                    .try_reduce(BTreeMap::new, |mut map, mut map2| {
+                        if map.len() > map2.len() {
+                            map.append(&mut map2);
+                            Ok(map)
+                        } else {
+                            map2.append(&mut map);
+                            Ok(map2)
+                        }
+                    })?;
 
                 let outputs_len = outputindex_to_txout_outputtype_addressbytes_res_addressindex_opt.len();
                 let inputs_len = input_source_vec.len();
@@ -747,7 +712,6 @@ impl Indexer {
                 idxs.inputindex += InputIndex::from(inputs_len);
                 idxs.outputindex += OutputIndex::from(outputs_len);
 
-
                 if should_export(height, false) {
                     txindex_to_first_outputindex_reader_opt.take();
                     p2pk65addressindex_to_p2pk65bytes_reader_opt.take();

crates/brk_interface/Cargo.toml

@@ -19,8 +19,9 @@ brk_structs = { workspace = true }
 vecdb = { workspace = true }
 derive_deref = { workspace = true }
 quick_cache = { workspace = true }
-schemars = "1.0.4"
+schemars = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+sonic-rs = { workspace = true }
 serde_with = "3.14.1"
 nucleo-matcher = "0.3.1"

crates/brk_interface/examples/main.rs

@@ -38,12 +38,12 @@ pub fn main() -> Result<()> {
 
     dbg!(interface.search_and_format(Params {
         index: Index::Height,
-        ids: vec!["date"].into(),
+        metrics: vec!["date"].into(),
         rest: ParamsOpt::default().set_from(-1),
     })?);
     dbg!(interface.search_and_format(Params {
         index: Index::Height,
-        ids: vec!["date", "timestamp"].into(),
+        metrics: vec!["date", "timestamp"].into(),
         rest: ParamsOpt::default().set_from(-10).set_count(5),
     })?);
 

crates/brk_interface/src/deser.rs

@@ -1,26 +1,27 @@
 use serde::{Deserialize, Deserializer};
+use serde_json::Value;
 
 pub fn de_unquote_i64<'de, D>(deserializer: D) -> Result<Option<i64>, D::Error>
 where
     D: Deserializer<'de>,
 {
-    let value: Option<serde_json::Value> = Option::deserialize(deserializer)?;
+    let value: Option<Value> = Option::deserialize(deserializer)?;
 
-    match value {
-        None => Ok(None),
-        Some(serde_json::Value::String(mut s)) => {
-            if s.starts_with('"') && s.ends_with('"') && s.len() >= 2 {
-                s = s[1..s.len() - 1].to_string();
-            }
-            s.parse::<i64>().map(Some).map_err(serde::de::Error::custom)
+    if value.is_none() {
+        return Ok(None);
+    }
+
+    let value = value.unwrap();
+
+    if let Some(mut s) = value.as_str().map(|s| s.to_string()) {
+        if s.starts_with('"') && s.ends_with('"') && s.len() >= 2 {
+            s = s[1..s.len() - 1].to_string();
         }
-        Some(serde_json::Value::Number(n)) => {
-            // If it's a number, convert it to i64
-            n.as_i64()
-                .ok_or_else(|| serde::de::Error::custom("number out of range"))
-                .map(Some)
-        }
-        _ => Err(serde::de::Error::custom("expected a string or number")),
+        s.parse::<i64>().map(Some).map_err(serde::de::Error::custom)
+    } else if let Some(n) = value.as_i64() {
+        Ok(Some(n))
+    } else {
+        Err(serde::de::Error::custom("expected a string or number"))
     }
 }
 
@@ -28,28 +29,24 @@ pub fn de_unquote_usize<'de, D>(deserializer: D) -> Result<Option<usize>, D::Err
 where
     D: Deserializer<'de>,
 {
-    let value: Option<serde_json::Value> = Option::deserialize(deserializer)?;
+    let value: Option<Value> = Option::deserialize(deserializer)?;
 
-    match value {
-        None => Ok(None),
-        Some(serde_json::Value::String(mut s)) => {
-            if s.starts_with('"') && s.ends_with('"') && s.len() >= 2 {
-                s = s[1..s.len() - 1].to_string();
-            }
-            s.parse::<usize>()
-                .map(Some)
-                .map_err(serde::de::Error::custom)
-        }
-        Some(serde_json::Value::Number(n)) => {
-            // If it's a number, convert it to usize
-            n.as_u64()
-                .ok_or_else(|| serde::de::Error::custom("number out of range"))
-                .map(|v| v as usize)
-                .map(Some)
-        }
-        _ => {
-            dbg!(value);
-            Err(serde::de::Error::custom("expected a string or number"))
+    if value.is_none() {
+        return Ok(None);
+    }
+
+    let value = value.unwrap();
+
+    if let Some(mut s) = value.as_str().map(|s| s.to_string()) {
+        if s.starts_with('"') && s.ends_with('"') && s.len() >= 2 {
+            s = s[1..s.len() - 1].to_string();
         }
+        s.parse::<usize>()
+            .map(Some)
+            .map_err(serde::de::Error::custom)
+    } else if let Some(n) = value.as_u64() {
+        Ok(Some(n as usize))
+    } else {
+        Err(serde::de::Error::custom("expected a string or number"))
     }
 }

crates/brk_interface/src/ids.rs

@@ -1,82 +0,0 @@
-use std::fmt;
-
-use derive_deref::Deref;
-use schemars::JsonSchema;
-use serde::Deserialize;
-
-#[derive(Debug, Deref, JsonSchema)]
-pub struct MaybeIds(Vec<String>);
-
-const MAX_VECS: usize = 32;
-const MAX_STRING_SIZE: usize = 64 * MAX_VECS;
-
-impl From<String> for MaybeIds {
-    fn from(value: String) -> Self {
-        Self(vec![value])
-    }
-}
-
-impl<'a> From<Vec<&'a str>> for MaybeIds {
-    fn from(value: Vec<&'a str>) -> Self {
-        Self(value.iter().map(|s| s.to_string()).collect::<Vec<_>>())
-    }
-}
-
-impl<'de> Deserialize<'de> for MaybeIds {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        match serde_json::Value::deserialize(deserializer)? {
-            serde_json::Value::String(str) => {
-                if str.len() <= MAX_STRING_SIZE {
-                    Ok(MaybeIds(sanitize_ids(
-                        str.split(",").map(|s| s.to_string()),
-                    )))
-                } else {
-                    Err(serde::de::Error::custom("Given parameter is too long"))
-                }
-            }
-            serde_json::Value::Array(vec) => {
-                if vec.len() <= MAX_VECS {
-                    Ok(MaybeIds(sanitize_ids(
-                        vec.into_iter().map(|s| s.as_str().unwrap().to_string()),
-                    )))
-                } else {
-                    Err(serde::de::Error::custom("Given parameter is too long"))
-                }
-            }
-            _ => Err(serde::de::Error::custom("Bad ids format")),
-        }
-    }
-}
-
-impl fmt::Display for MaybeIds {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let s = self.0.join(",");
-        write!(f, "{s}")
-    }
-}
-
-fn sanitize_ids(raw_ids: impl Iterator<Item = String>) -> Vec<String> {
-    let mut results = Vec::new();
-    raw_ids.for_each(|s| {
-        let mut current = String::new();
-        for c in s.to_lowercase().chars() {
-            match c {
-                ' ' | ',' | '+' => {
-                    if !current.is_empty() {
-                        results.push(std::mem::take(&mut current));
-                    }
-                }
-                '-' => current.push('_'),
-                c if c.is_alphanumeric() || c == '_' => current.push(c),
-                _ => {}
-            }
-        }
-        if !current.is_empty() {
-            results.push(current);
-        }
-    });
-    results
-}

crates/brk_interface/src/lib.rs

@@ -16,8 +16,8 @@ use vecdb::{AnyCollectableVec, AnyStoredVec};
 
 mod deser;
 mod format;
-mod ids;
 mod index;
+mod metrics;
 mod output;
 mod pagination;
 mod params;
@@ -27,10 +27,10 @@ pub use format::Format;
 pub use index::Index;
 pub use output::{Output, Value};
 pub use pagination::{PaginatedIndexParam, PaginationParam};
-pub use params::{IdParam, Params, ParamsOpt};
+pub use params::{Params, ParamsDeprec, ParamsOpt};
 use vecs::Vecs;
 
-use crate::vecs::{IdToVec, IndexToVec};
+use crate::vecs::{IndexToVec, MetricToVec};
 
 pub fn cached_errors() -> &'static Cache<String, String> {
     static CACHE: OnceLock<Cache<String, String>> = OnceLock::new();
@@ -65,37 +65,37 @@ impl<'a> Interface<'a> {
     }
 
     pub fn search(&self, params: &Params) -> Result<Vec<(String, &&dyn AnyCollectableVec)>> {
-        let ids = &params.ids;
+        let metrics = &params.metrics;
         let index = params.index;
 
         let ids_to_vec = self
             .vecs
-            .index_to_id_to_vec
+            .index_to_metric_to_vec
             .get(&index)
             .ok_or(Error::String(format!(
                 "Index \"{}\" isn't a valid index",
                 index
             )))?;
 
-        ids.iter()
-            .map(|id| {
-                let vec = ids_to_vec.get(id.as_str()).ok_or_else(|| {
+        metrics.iter()
+            .map(|metric| {
+                let vec = ids_to_vec.get(metric.as_str()).ok_or_else(|| {
                     let cached_errors = cached_errors();
 
-                    if let Some(message) = cached_errors.get(id) {
+                    if let Some(message) = cached_errors.get(metric) {
                         return Error::String(message)
                     }
 
                     let mut message = format!(
                         "No vec named \"{}\" indexed by \"{}\" found.\n",
-                        id,
+                        metric,
                         index
                     );
 
                     let mut matcher = Matcher::new(Config::DEFAULT);
 
                     let matches = Pattern::new(
-                        id.as_str(),
+                        metric.as_str(),
                         CaseMatching::Ignore,
                         Normalization::Smart,
                         AtomKind::Fuzzy,
@@ -111,33 +111,35 @@ impl<'a> Interface<'a> {
                             &format!("\nMaybe you meant one of the following: {matches:#?} ?\n");
                     }
 
-                    if let Some(index_to_vec) = self.id_to_index_to_vec().get(id.as_str()) {
-                        message += &format!("\nBut there is a vec named {id} which supports the following indexes: {:#?}\n", index_to_vec.keys());
+                    if let Some(index_to_vec) = self.metric_to_index_to_vec().get(metric.as_str()) {
+                        message += &format!("\nBut there is a vec named {metric} which supports the following indexes: {:#?}\n", index_to_vec.keys());
                     }
 
-                    cached_errors.insert(id.clone(), message.clone());
+                    cached_errors.insert(metric.clone(), message.clone());
 
                     Error::String(message)
                 });
-                vec.map(|vec| (id.clone(), vec))
+                vec.map(|vec| (metric.clone(), vec))
             })
             .collect::<Result<Vec<_>>>()
     }
 
     pub fn format(
         &self,
-        vecs: Vec<(String, &&dyn AnyCollectableVec)>,
+        metrics: Vec<(String, &&dyn AnyCollectableVec)>,
         params: &ParamsOpt,
     ) -> Result<Output> {
         let from = params.from().map(|from| {
-            vecs.iter()
+            metrics
+                .iter()
                 .map(|(_, v)| v.i64_to_usize(from))
                 .min()
                 .unwrap_or_default()
         });
 
         let to = params.to().map(|to| {
-            vecs.iter()
+            metrics
+                .iter()
                 .map(|(_, v)| v.i64_to_usize(to))
                 .min()
                 .unwrap_or_default()
@@ -147,8 +149,11 @@ impl<'a> Interface<'a> {
 
         Ok(match format {
             Format::CSV => {
-                let headers = vecs.iter().map(|(id, _)| id.as_str()).collect::<Vec<_>>();
-                let mut values = vecs
+                let headers = metrics
+                    .iter()
+                    .map(|(id, _)| id.as_str())
+                    .collect::<Vec<_>>();
+                let mut values = metrics
                     .iter()
                     .map(|(_, vec)| Ok(vec.collect_range_string(from, to)?))
                     .collect::<Result<Vec<_>>>()?;
@@ -190,7 +195,7 @@ impl<'a> Interface<'a> {
                 Output::CSV(csv)
             }
             Format::JSON => {
-                let mut values = vecs
+                let mut values = metrics
                     .iter()
                     .map(|(_, vec)| -> Result<Vec<u8>> {
                         Ok(vec.collect_range_json_bytes(from, to)?)
@@ -214,44 +219,36 @@ impl<'a> Interface<'a> {
         self.format(self.search(&params)?, &params.rest)
     }
 
-    pub fn id_to_index_to_vec(&self) -> &BTreeMap<&str, IndexToVec<'_>> {
-        &self.vecs.id_to_index_to_vec
+    pub fn metric_to_index_to_vec(&self) -> &BTreeMap<&str, IndexToVec<'_>> {
+        &self.vecs.metric_to_index_to_vec
     }
 
-    pub fn index_to_id_to_vec(&self) -> &BTreeMap<Index, IdToVec<'_>> {
-        &self.vecs.index_to_id_to_vec
+    pub fn index_to_metric_to_vec(&self) -> &BTreeMap<Index, MetricToVec<'_>> {
+        &self.vecs.index_to_metric_to_vec
     }
 
-    pub fn get_vecid_count(&self) -> usize {
-        self.vecs.id_count
+    pub fn distinct_metric_count(&self) -> usize {
+        self.vecs.distinct_metric_count
     }
 
-    pub fn get_index_count(&self) -> usize {
-        self.vecs.index_count
+    pub fn total_metric_count(&self) -> usize {
+        self.vecs.total_metric_count
     }
 
-    pub fn get_vec_count(&self) -> usize {
-        self.vecs.vec_count
-    }
-
-    pub fn get_indexes(&self) -> &[&'static str] {
+    pub fn get_indexes(&self) -> &BTreeMap<&'static str, &'static [&'static str]> {
         &self.vecs.indexes
     }
 
-    pub fn get_accepted_indexes(&self) -> &BTreeMap<&'static str, &'static [&'static str]> {
-        &self.vecs.accepted_indexes
-    }
-
-    pub fn get_vecids(&self, pagination: PaginationParam) -> &[&str] {
-        self.vecs.ids(pagination)
+    pub fn get_metrics(&self, pagination: PaginationParam) -> &[&str] {
+        self.vecs.metrics(pagination)
     }
 
     pub fn get_index_to_vecids(&self, paginated_index: PaginatedIndexParam) -> Vec<&str> {
         self.vecs.index_to_ids(paginated_index)
     }
 
-    pub fn get_vecid_to_indexes(&self, id: String) -> Option<&Vec<&'static str>> {
-        self.vecs.id_to_indexes(id)
+    pub fn metric_to_indexes(&self, metric: String) -> Option<&Vec<&'static str>> {
+        self.vecs.metric_to_indexes(metric)
     }
 
     pub fn parser(&self) -> &Parser {

crates/brk_interface/src/metrics.rs

@@ -0,0 +1,88 @@
+use std::fmt;
+
+use derive_deref::Deref;
+use schemars::JsonSchema;
+use serde::Deserialize;
+use serde_json::Value;
+
+#[derive(Debug, Deref, JsonSchema)]
+pub struct MaybeMetrics(Vec<String>);
+
+const MAX_VECS: usize = 32;
+const MAX_STRING_SIZE: usize = 64 * MAX_VECS;
+
+impl From<String> for MaybeMetrics {
+    fn from(value: String) -> Self {
+        Self(vec![value.replace("-", "_").to_lowercase()])
+    }
+}
+
+impl<'a> From<Vec<&'a str>> for MaybeMetrics {
+    fn from(value: Vec<&'a str>) -> Self {
+        Self(
+            value
+                .iter()
+                .map(|s| s.replace("-", "_").to_lowercase())
+                .collect::<Vec<_>>(),
+        )
+    }
+}
+
+impl<'de> Deserialize<'de> for MaybeMetrics {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let value = Value::deserialize(deserializer)?;
+
+        if let Some(str) = value.as_str() {
+            if str.len() <= MAX_STRING_SIZE {
+                Ok(MaybeMetrics(sanitize_metrics(
+                    str.split(",").map(|s| s.to_string()),
+                )))
+            } else {
+                Err(serde::de::Error::custom("Given parameter is too long"))
+            }
+        } else if let Some(vec) = value.as_array() {
+            if vec.len() <= MAX_VECS {
+                Ok(MaybeMetrics(sanitize_metrics(
+                    vec.iter().map(|s| s.as_str().unwrap().to_string()),
+                )))
+            } else {
+                Err(serde::de::Error::custom("Given parameter is too long"))
+            }
+        } else {
+            Err(serde::de::Error::custom("Bad ids format"))
+        }
+    }
+}
+
+impl fmt::Display for MaybeMetrics {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let s = self.0.join(",");
+        write!(f, "{s}")
+    }
+}
+
+fn sanitize_metrics(raw_ids: impl Iterator<Item = String>) -> Vec<String> {
+    let mut results = Vec::new();
+    raw_ids.for_each(|s| {
+        let mut current = String::new();
+        for c in s.to_lowercase().chars() {
+            match c {
+                ' ' | ',' | '+' => {
+                    if !current.is_empty() {
+                        results.push(std::mem::take(&mut current));
+                    }
+                }
+                '-' => current.push('_'),
+                c if c.is_alphanumeric() || c == '_' => current.push(c),
+                _ => {}
+            }
+        }
+        if !current.is_empty() {
+            results.push(current);
+        }
+    });
+    results
+}

crates/brk_interface/src/params.rs

@@ -6,23 +6,22 @@ use serde::Deserialize;
 use crate::{
     Format, Index,
     deser::{de_unquote_i64, de_unquote_usize},
-    ids::MaybeIds,
+    metrics::MaybeMetrics,
 };
 
 #[derive(Debug, Deserialize, JsonSchema)]
 pub struct Params {
-    #[serde(alias = "i")]
-    #[schemars(description = "Index of requested vecs")]
-    pub index: Index,
+    #[serde(alias = "m")]
+    #[schemars(description = "Requested metrics")]
+    pub metrics: MaybeMetrics,
 
-    #[serde(alias = "v")]
-    #[schemars(description = "Ids of requested vecs")]
-    pub ids: MaybeIds,
+    #[serde(alias = "i")]
+    #[schemars(description = "Requested index")]
+    pub index: Index,
 
     #[serde(flatten)]
     pub rest: ParamsOpt,
 }
-serde_with::flattened_maybe!(deserialize_rest, "rest");
 
 impl Deref for Params {
     type Target = ParamsOpt;
@@ -32,10 +31,10 @@ impl Deref for Params {
 }
 
 impl From<((Index, String), ParamsOpt)> for Params {
-    fn from(((index, id), rest): ((Index, String), ParamsOpt)) -> Self {
+    fn from(((index, metric), rest): ((Index, String), ParamsOpt)) -> Self {
         Self {
             index,
-            ids: MaybeIds::from(id),
+            metrics: MaybeMetrics::from(metric),
             rest,
         }
     }
@@ -107,7 +106,22 @@ impl ParamsOpt {
     }
 }
 
-#[derive(Debug, Deserialize, JsonSchema)]
-pub struct IdParam {
-    pub id: String,
+#[derive(Debug, Deserialize)]
+pub struct ParamsDeprec {
+    #[serde(alias = "i")]
+    pub index: Index,
+    #[serde(alias = "v")]
+    pub ids: MaybeMetrics,
+    #[serde(flatten)]
+    pub rest: ParamsOpt,
+}
+
+impl From<ParamsDeprec> for Params {
+    fn from(value: ParamsDeprec) -> Self {
+        Params {
+            index: value.index,
+            metrics: value.ids,
+            rest: value.rest,
+        }
+    }
 }

crates/brk_interface/src/vecs.rs

@@ -11,16 +11,14 @@ use super::index::Index;
 
 #[derive(Default)]
 pub struct Vecs<'a> {
-    pub id_to_index_to_vec: BTreeMap<&'a str, IndexToVec<'a>>,
-    pub index_to_id_to_vec: BTreeMap<Index, IdToVec<'a>>,
-    pub ids: Vec<&'a str>,
-    pub indexes: Vec<&'static str>,
-    pub accepted_indexes: BTreeMap<&'static str, &'static [&'static str]>,
-    pub index_count: usize,
-    pub id_count: usize,
-    pub vec_count: usize,
-    id_to_indexes: BTreeMap<&'a str, Vec<&'static str>>,
-    indexes_to_ids: BTreeMap<Index, Vec<&'a str>>,
+    pub metric_to_index_to_vec: BTreeMap<&'a str, IndexToVec<'a>>,
+    pub index_to_metric_to_vec: BTreeMap<Index, MetricToVec<'a>>,
+    pub metrics: Vec<&'a str>,
+    pub indexes: BTreeMap<&'static str, &'static [&'static str]>,
+    pub distinct_metric_count: usize,
+    pub total_metric_count: usize,
+    metric_to_indexes: BTreeMap<&'a str, Vec<&'static str>>,
+    index_to_metrics: BTreeMap<Index, Vec<&'a str>>,
 }
 
 impl<'a> Vecs<'a> {
@@ -33,7 +31,11 @@ impl<'a> Vecs<'a> {
             .iter_any_collectable()
             .for_each(|vec| this.insert(vec));
 
-        let mut ids = this.id_to_index_to_vec.keys().cloned().collect::<Vec<_>>();
+        let mut ids = this
+            .metric_to_index_to_vec
+            .keys()
+            .cloned()
+            .collect::<Vec<_>>();
 
         let sort_ids = |ids: &mut Vec<&str>| {
             ids.sort_unstable_by(|a, b| {
@@ -48,26 +50,20 @@ impl<'a> Vecs<'a> {
 
         sort_ids(&mut ids);
 
-        this.ids = ids;
-        this.id_count = this.id_to_index_to_vec.keys().count();
-        this.index_count = this.index_to_id_to_vec.keys().count();
-        this.vec_count = this
-            .index_to_id_to_vec
+        this.metrics = ids;
+        this.distinct_metric_count = this.metric_to_index_to_vec.keys().count();
+        this.total_metric_count = this
+            .index_to_metric_to_vec
             .values()
             .map(|tree| tree.len())
             .sum::<usize>();
         this.indexes = this
-            .index_to_id_to_vec
-            .keys()
-            .map(|i| i.serialize_long())
-            .collect::<Vec<_>>();
-        this.accepted_indexes = this
-            .index_to_id_to_vec
+            .index_to_metric_to_vec
             .keys()
             .map(|i| (i.serialize_long(), i.possible_values()))
             .collect::<BTreeMap<_, _>>();
-        this.id_to_indexes = this
-            .id_to_index_to_vec
+        this.metric_to_indexes = this
+            .metric_to_index_to_vec
             .iter()
             .map(|(id, index_to_vec)| {
                 (
@@ -79,12 +75,12 @@ impl<'a> Vecs<'a> {
                 )
             })
             .collect();
-        this.indexes_to_ids = this
-            .index_to_id_to_vec
+        this.index_to_metrics = this
+            .index_to_metric_to_vec
             .iter()
             .map(|(index, id_to_vec)| (*index, id_to_vec.keys().cloned().collect::<Vec<_>>()))
             .collect();
-        this.indexes_to_ids
+        this.index_to_metrics
             .values_mut()
             .for_each(|ids| sort_ids(ids));
 
@@ -101,7 +97,7 @@ impl<'a> Vecs<'a> {
             })
             .unwrap();
         let prev = self
-            .id_to_index_to_vec
+            .metric_to_index_to_vec
             .entry(name)
             .or_default()
             .insert(index, vec);
@@ -110,7 +106,7 @@ impl<'a> Vecs<'a> {
             panic!()
         }
         let prev = self
-            .index_to_id_to_vec
+            .index_to_metric_to_vec
             .entry(index)
             .or_default()
             .insert(name, vec);
@@ -120,22 +116,23 @@ impl<'a> Vecs<'a> {
         }
     }
 
-    pub fn ids(&self, pagination: PaginationParam) -> &[&'_ str] {
-        let len = self.ids.len();
+    pub fn metrics(&self, pagination: PaginationParam) -> &[&'_ str] {
+        let len = self.metrics.len();
         let start = pagination.start(len);
         let end = pagination.end(len);
-        &self.ids[start..end]
+        &self.metrics[start..end]
     }
 
-    pub fn id_to_indexes(&self, id: String) -> Option<&Vec<&'static str>> {
-        self.id_to_indexes.get(id.as_str())
+    pub fn metric_to_indexes(&self, metric: String) -> Option<&Vec<&'static str>> {
+        self.metric_to_indexes
+            .get(metric.replace("-", "_").as_str())
     }
 
     pub fn index_to_ids(
         &self,
         PaginatedIndexParam { index, pagination }: PaginatedIndexParam,
     ) -> Vec<&'a str> {
-        let vec = self.indexes_to_ids.get(&index).unwrap();
+        let vec = self.index_to_metrics.get(&index).unwrap();
 
         let len = vec.len();
         let start = pagination.start(len);
@@ -149,4 +146,4 @@ impl<'a> Vecs<'a> {
 pub struct IndexToVec<'a>(BTreeMap<Index, &'a dyn AnyCollectableVec>);
 
 #[derive(Default, Deref, DerefMut)]
-pub struct IdToVec<'a>(BTreeMap<&'a str, &'a dyn AnyCollectableVec>);
+pub struct MetricToVec<'a>(BTreeMap<&'a str, &'a dyn AnyCollectableVec>);

crates/brk_logger/Cargo.toml

@@ -13,4 +13,4 @@ build = "build.rs"
 env_logger = "0.11.8"
 jiff = { workspace = true }
 log = { workspace = true }
-owo-colors = "4.2.2"
+owo-colors = "4.2.3"

crates/brk_mcp/Cargo.toml

@@ -12,8 +12,14 @@ build = "build.rs"
 [dependencies]
 axum = { workspace = true }
 brk_interface = { workspace = true }
-log = { workspace = true }
-brk_rmcp = { version = "0.6.0", features = [
+brk_rmcp = { version = "0.7.1", features = [
     "transport-worker",
     "transport-streamable-http-server",
 ] }
+log = { workspace = true }
+schemars = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+
+[package.metadata.cargo-machete]
+ignored = ["serde_json"]

crates/brk_mcp/src/lib.rs

@@ -1,14 +1,16 @@
 #![doc = include_str!("../README.md")]
 
-use brk_interface::{IdParam, Interface, PaginatedIndexParam, PaginationParam, Params};
+use brk_interface::{Interface, PaginatedIndexParam, PaginationParam, Params};
 use brk_rmcp::{
     ErrorData as McpError, RoleServer, ServerHandler,
-    handler::server::{router::tool::ToolRouter, tool::Parameters},
+    handler::server::{router::tool::ToolRouter, wrapper::Parameters},
     model::*,
     service::RequestContext,
     tool, tool_handler, tool_router,
 };
 use log::info;
+use schemars::JsonSchema;
+use serde::Deserialize;
 
 pub mod route;
 
@@ -30,38 +32,27 @@ impl MCP {
     }
 
     #[tool(description = "
-Get the count of all existing indexes.
+Get the count of unique metrics.
 ")]
-    async fn get_index_count(&self) -> Result<CallToolResult, McpError> {
-        info!("mcp: get_index_count");
+    async fn get_metric_count(&self) -> Result<CallToolResult, McpError> {
+        info!("mcp: distinct_metric_count");
         Ok(CallToolResult::success(vec![
-            Content::json(self.interface.get_index_count()).unwrap(),
+            Content::json(self.interface.distinct_metric_count()).unwrap(),
         ]))
     }
 
     #[tool(description = "
-Get the count of all existing vec ids.
-")]
-    async fn get_vecid_count(&self) -> Result<CallToolResult, McpError> {
-        info!("mcp: get_vecid_count");
-        Ok(CallToolResult::success(vec![
-            Content::json(self.interface.get_vecid_count()).unwrap(),
-        ]))
-    }
-
-    #[tool(description = "
-Get the count of all existing vecs.
-Equals to the sum of supported Indexes of each vec id.
+Get the count of all metrics. (distinct metrics multiplied by the number of indexes supported by each one)
 ")]
     async fn get_vec_count(&self) -> Result<CallToolResult, McpError> {
-        info!("mcp: get_vec_count");
+        info!("mcp: total_metric_count");
         Ok(CallToolResult::success(vec![
-            Content::json(self.interface.get_vec_count()).unwrap(),
+            Content::json(self.interface.total_metric_count()).unwrap(),
         ]))
     }
 
     #[tool(description = "
-Get the list of all existing indexes.
+Get the list of all existing indexes and their accepted variants.
 ")]
     async fn get_indexes(&self) -> Result<CallToolResult, McpError> {
         info!("mcp: get_indexes");
@@ -70,16 +61,6 @@ Get the list of all existing indexes.
         ]))
     }
 
-    #[tool(description = "
-Get an object which has all existing indexes as keys and a list of their accepted variants as values.
-")]
-    async fn get_accepted_indexes(&self) -> Result<CallToolResult, McpError> {
-        info!("mcp: get_accepted_indexes");
-        Ok(CallToolResult::success(vec![
-            Content::json(self.interface.get_accepted_indexes()).unwrap(),
-        ]))
-    }
-
     #[tool(description = "
 Get a paginated list of all existing vec ids.
 There are up to 1,000 values per page.
@@ -89,9 +70,9 @@ If the `page` param is omitted, it will default to the first page.
         &self,
         Parameters(pagination): Parameters<PaginationParam>,
     ) -> Result<CallToolResult, McpError> {
-        info!("mcp: get_vecids");
+        info!("mcp: get_metrics");
         Ok(CallToolResult::success(vec![
-            Content::json(self.interface.get_vecids(pagination)).unwrap(),
+            Content::json(self.interface.get_metrics(pagination)).unwrap(),
         ]))
     }
 
@@ -120,7 +101,7 @@ The list will be empty if the vec id isn't correct.
     ) -> Result<CallToolResult, McpError> {
         info!("mcp: get_vecid_to_indexes");
         Ok(CallToolResult::success(vec![
-            Content::json(self.interface.get_vecid_to_indexes(param.id)).unwrap(),
+            Content::json(self.interface.metric_to_indexes(param.id)).unwrap(),
         ]))
     }
 
@@ -186,3 +167,8 @@ An 'Index' (or indexes) is the timeframe of a dataset.
         Ok(self.get_info())
     }
 }
+
+#[derive(Debug, Deserialize, JsonSchema)]
+pub struct IdParam {
+    pub id: String,
+}

crates/brk_parser/examples/main.rs

@@ -1,4 +1,4 @@
-use std::{collections::BTreeMap, path::Path};
+use std::path::Path;
 
 use bitcoincore_rpc::{Auth, Client};
 use brk_error::Result;
@@ -20,17 +20,11 @@ fn main() -> Result<()> {
 
     let parser = Parser::new(bitcoin_dir.join("blocks"), rpc);
 
-    let start = Some(915_138_u32.into());
+    let start = None;
+    // let start = Some(916037_u32.into());
     let end = None;
-    let mut blk_index = 0;
-    let mut diff = BTreeMap::new();
     parser.parse(start, end).iter().for_each(|block| {
         println!("{}: {}", block.height(), block.hash());
-        let new_blk_index = block.metadata().blk_index();
-        if new_blk_index < blk_index {
-            diff.insert(blk_index - new_blk_index, block.height());
-        }
-        blk_index = new_blk_index;
     });
 
     // let v = diff.iter().rev().take(10).collect::<Vec<_>>();

crates/brk_parser/src/lib.rs

@@ -9,6 +9,7 @@ use std::{
     path::PathBuf,
     sync::Arc,
     thread,
+    time::Duration,
 };
 
 use bitcoin::{block::Header, consensus::Decodable};
@@ -16,7 +17,7 @@ use bitcoincore_rpc::RpcApi;
 use blk_index_to_blk_path::*;
 use brk_error::Result;
 use brk_structs::{BlkMetadata, BlkPosition, Block, Height, ParsedBlock};
-use crossbeam::channel::{bounded, Receiver};
+use crossbeam::channel::{Receiver, bounded};
 use parking_lot::{RwLock, RwLockReadGuard};
 use rayon::prelude::*;
 
@@ -29,8 +30,6 @@ use any_block::*;
 pub use xor_bytes::*;
 pub use xor_index::*;
 
-pub const NUMBER_OF_UNSAFE_BLOCKS: usize = 100;
-
 const MAGIC_BYTES: [u8; 4] = [249, 190, 180, 217];
 const BOUND_CAP: usize = 50;
 
@@ -79,7 +78,7 @@ impl Parser {
     pub fn parse(&self, start: Option<Height>, end: Option<Height>) -> Receiver<ParsedBlock> {
         let rpc = self.rpc;
 
-        let (send_bytes, recv_bytes) = bounded(BOUND_CAP);
+        let (send_bytes, recv_bytes) = bounded(BOUND_CAP / 2);
         let (send_block, recv_block) = bounded(BOUND_CAP);
         let (send_ordered, recv_ordered) = bounded(BOUND_CAP);
 
@@ -157,20 +156,27 @@ impl Parser {
 
             let mut bulk = vec![];
 
-            let drain_and_send = |bulk: &mut Vec<(BlkMetadata, AnyBlock, XORIndex)>| {
-                // Using a vec and sending after to not end up with stuck threads in par iter
-                mem::take(bulk)
-                    .into_par_iter()
-                    .try_for_each(|(metdata, any_block, xor_i)| {
-                        if let Ok(AnyBlock::Decoded(block)) =
-                            any_block.decode(metdata, rpc, xor_i, xor_bytes, start, end)
-                            && send_block.send(block).is_err()
-                        {
-                            return ControlFlow::Break(());
-                        }
+            // Private pool to prevent collision with the global pool
+            // Without it there can be hanging
+            let parser_pool = rayon::ThreadPoolBuilder::new()
+                .num_threads(thread::available_parallelism().unwrap().get() / 2)
+                .build()
+                .expect("Failed to create parser thread pool");
 
-                        ControlFlow::Continue(())
-                    })
+            let drain_and_send = |bulk: &mut Vec<(BlkMetadata, AnyBlock, XORIndex)>| {
+                parser_pool.install(|| {
+                    mem::take(bulk)
+                        .into_par_iter()
+                        .try_for_each(|(metdata, any_block, xor_i)| {
+                            if let Ok(AnyBlock::Decoded(block)) =
+                                any_block.decode(metdata, rpc, xor_i, xor_bytes, start, end)
+                                && send_block.send(block).is_err()
+                            {
+                                return ControlFlow::Break(());
+                            }
+                            ControlFlow::Continue(())
+                        })
+                })
             };
 
             recv_bytes.iter().try_for_each(|tuple| {
@@ -180,7 +186,9 @@ impl Parser {
                     return ControlFlow::Continue(());
                 }
 
-                // Sending in bulk to not lock threads in standby
+                while send_block.len() >= bulk.len() {
+                    thread::sleep(Duration::from_micros(100));
+                }
                 drain_and_send(&mut bulk)
             })?;
 

crates/brk_server/Cargo.toml

@@ -27,7 +27,6 @@ jiff = { workspace = true }
 log = { workspace = true }
 quick_cache = { workspace = true }
 serde = { workspace = true }
-serde_json = { workspace = true }
 sonic-rs = { workspace = true }
 tokio = { workspace = true }
 tower-http = { version = "0.6.6", features = ["compression-full", "trace"] }

crates/brk_server/src/api/explorer/mod.rs

@@ -0,0 +1,238 @@
+use std::{
+    fs::File,
+    io::{Cursor, Read, Seek, SeekFrom},
+    str::FromStr,
+};
+
+use axum::{
+    Json, Router,
+    extract::{Path, State},
+    response::{IntoResponse, Response},
+    routing::get,
+};
+use bitcoin::{Address, Network, Transaction, consensus::Decodable};
+use brk_parser::XORIndex;
+use brk_structs::{
+    AddressBytesHash, AnyAddressDataIndexEnum, Bitcoin, OutputType, TxIndex, Txid, TxidPrefix,
+};
+use serde::Serialize;
+use sonic_rs::{Number, Value};
+use vecdb::{AnyIterableVec, VecIterator};
+
+use super::AppState;
+
+pub trait ApiExplorerRoutes {
+    fn add_api_explorer_routes(self) -> Self;
+}
+
+#[derive(Serialize)]
+struct TxResponse {
+    txid: Txid,
+    index: TxIndex,
+    tx: Transaction,
+}
+
+impl ApiExplorerRoutes for Router<AppState> {
+    fn add_api_explorer_routes(self) -> Self {
+        self.route(
+            "/api/address/{address}",
+            get(
+                async |Path(address): Path<String>, state: State<AppState>| -> Response {
+                    let Ok(address) = Address::from_str(&address) else {
+                        return "Invalid address".into_response();
+                    };
+                    if !address.is_valid_for_network(Network::Bitcoin) {
+                        return "Invalid address".into_response();
+                    }
+                    let address = address.assume_checked();
+                    let interface = state.interface;
+                    let indexer = interface.indexer();
+                    let computer = interface.computer();
+                    let stores = &indexer.stores;
+                    let hash = AddressBytesHash::from(&address);
+
+                    let Ok(Some(addri)) = stores
+                        .addressbyteshash_to_typeindex
+                        .get(&hash)
+                        .map(|opt| opt.map(|cow| cow.into_owned()))
+                    else {
+                        return "Unknown address".into_response();
+                    };
+
+                    let output_type = OutputType::from(&address);
+                    let stateful = &computer.stateful;
+                    let price = computer.price.as_ref().map(|v| {
+                        *v.timeindexes_to_price_close
+                            .dateindex
+                            .as_ref()
+                            .unwrap()
+                            .iter()
+                            .last()
+                            .unwrap()
+                            .1
+                            .into_owned()
+                    });
+
+                    let anyaddri = match output_type {
+                        OutputType::P2PK33 => stateful
+                            .p2pk33addressindex_to_anyaddressindex
+                            .iter()
+                            .unwrap_get_inner(addri.into()),
+                        OutputType::P2PK65 => stateful
+                            .p2pk65addressindex_to_anyaddressindex
+                            .iter()
+                            .unwrap_get_inner(addri.into()),
+                        OutputType::P2PKH => stateful
+                            .p2pkhaddressindex_to_anyaddressindex
+                            .iter()
+                            .unwrap_get_inner(addri.into()),
+                        OutputType::P2SH => stateful
+                            .p2shaddressindex_to_anyaddressindex
+                            .iter()
+                            .unwrap_get_inner(addri.into()),
+                        OutputType::P2TR => stateful
+                            .p2traddressindex_to_anyaddressindex
+                            .iter()
+                            .unwrap_get_inner(addri.into()),
+                        OutputType::P2WPKH => stateful
+                            .p2wpkhaddressindex_to_anyaddressindex
+                            .iter()
+                            .unwrap_get_inner(addri.into()),
+                        OutputType::P2WSH => stateful
+                            .p2wshaddressindex_to_anyaddressindex
+                            .iter()
+                            .unwrap_get_inner(addri.into()),
+                        OutputType::P2A => stateful
+                            .p2aaddressindex_to_anyaddressindex
+                            .iter()
+                            .unwrap_get_inner(addri.into()),
+
+                        _ => unreachable!(),
+                    };
+
+                    let addr_data = match anyaddri.to_enum() {
+                        AnyAddressDataIndexEnum::Loaded(loadedi) => stateful
+                            .loadedaddressindex_to_loadedaddressdata
+                            .iter()
+                            .unwrap_get_inner(loadedi),
+                        AnyAddressDataIndexEnum::Empty(emptyi) => stateful
+                            .emptyaddressindex_to_emptyaddressdata
+                            .iter()
+                            .unwrap_get_inner(emptyi)
+                            .into(),
+                    };
+
+                    let amount = addr_data.amount();
+                    Json(sonic_rs::json!({
+                        "address": address,
+                        "type": output_type,
+                        "index": addri,
+                        "chain_stats": {
+                            "funded_txo_count":	null,
+                            "funded_txo_sum": addr_data.received,
+                            "spent_txo_count": null,
+                            "spent_txo_sum": addr_data.sent,
+                            "utxo_count": addr_data.utxos,
+                            "balance": amount,
+                            "balance_usd": price.map_or(Value::new(), |p| {
+                                Value::from(Number::from_f64(*(p * Bitcoin::from(amount))).unwrap())
+                            }),
+                            "realized_value": addr_data.realized_cap,
+                            "tx_count":	null,
+                            "avg_cost_basis": addr_data.realized_price()
+                        },
+                        "mempool_stats": null
+                    }))
+                    .into_response()
+                },
+            ),
+        )
+        .route(
+            "/api/tx/{txid}",
+            get(
+                async |Path(txid): Path<String>, state: State<AppState>| -> Response {
+                    let Ok(txid) = bitcoin::Txid::from_str(&txid) else {
+                        return "Invalid txid".into_response();
+                    };
+
+                    let txid = Txid::from(txid);
+                    let prefix = TxidPrefix::from(&txid);
+                    let interface = state.interface;
+                    let indexer = interface.indexer();
+                    let Ok(Some(txindex)) = indexer
+                        .stores
+                        .txidprefix_to_txindex
+                        .get(&prefix)
+                        .map(|opt| opt.map(|cow| cow.into_owned()))
+                    else {
+                        return "Unknown transaction".into_response();
+                    };
+
+                    let txid = indexer
+                        .vecs
+                        .txindex_to_txid
+                        .iter()
+                        .unwrap_get_inner(txindex);
+
+                    let parser = interface.parser();
+                    let computer = interface.computer();
+
+                    let position = computer
+                        .blks
+                        .txindex_to_position
+                        .iter()
+                        .unwrap_get_inner(txindex);
+                    let len = indexer
+                        .vecs
+                        .txindex_to_total_size
+                        .iter()
+                        .unwrap_get_inner(txindex);
+
+                    let blk_index_to_blk_path = parser.blk_index_to_blk_path();
+
+                    let Some(blk_path) = blk_index_to_blk_path.get(&position.blk_index()) else {
+                        return "Unknown blk index".into_response();
+                    };
+
+                    let mut xori = XORIndex::default();
+                    xori.add_assign(position.offset() as usize);
+
+                    let Ok(mut file) = File::open(blk_path) else {
+                        return "Error opening blk file".into_response();
+                    };
+
+                    if file
+                        .seek(SeekFrom::Start(position.offset() as u64))
+                        .is_err()
+                    {
+                        return "Error seeking position in blk file".into_response();
+                    }
+
+                    let mut buffer = vec![0u8; *len as usize];
+                    if file.read_exact(&mut buffer).is_err() {
+                        return "File fail read exact".into_response();
+                    }
+                    xori.bytes(&mut buffer, parser.xor_bytes());
+
+                    let mut reader = Cursor::new(buffer);
+                    let Ok(tx) = Transaction::consensus_decode(&mut reader) else {
+                        return "Error decoding transaction".into_response();
+                    };
+
+                    let response = TxResponse {
+                        txid,
+                        index: txindex,
+                        tx,
+                    };
+
+                    let bytes = sonic_rs::to_vec(&response).unwrap();
+
+                    Response::builder()
+                        .header("content-type", "application/json")
+                        .body(bytes.into())
+                        .unwrap()
+                },
+            ),
+        )
+    }
+}

crates/brk_server/src/api/metrics/mod.rs

@@ -0,0 +1,131 @@
+use axum::{
+    Json, Router,
+    extract::{Path, Query, State},
+    http::{HeaderMap, Uri},
+    response::{IntoResponse, Response},
+    routing::get,
+};
+use brk_interface::{Index, PaginatedIndexParam, PaginationParam, Params, ParamsDeprec, ParamsOpt};
+
+use super::AppState;
+
+mod data;
+
+pub trait ApiMetricsRoutes {
+    fn add_api_metrics_routes(self) -> Self;
+}
+
+const TO_SEPARATOR: &str = "_to_";
+
+impl ApiMetricsRoutes for Router<AppState> {
+    fn add_api_metrics_routes(self) -> Self {
+        self.route(
+            "/api/metrics/count",
+            get(async |State(app_state): State<AppState>| -> Response {
+                Json(sonic_rs::json!({
+                    "distinct": app_state.interface.distinct_metric_count(),
+                    "total": app_state.interface.total_metric_count(),
+                }))
+                .into_response()
+            }),
+        )
+        .route(
+            "/api/metrics/indexes",
+            get(async |State(app_state): State<AppState>| -> Response {
+                Json(app_state.interface.get_indexes()).into_response()
+            }),
+        )
+        // .route(
+        //     "/api/vecs/metrics",
+        //     get(
+        //         async |State(app_state): State<AppState>,
+        //                Query(pagination): Query<PaginationParam>|
+        //                -> Response {
+        //             Json(app_state.interface.get_metrics(pagination)).into_response()
+        //         },
+        //     ),
+        // )
+        // .route(
+        //     "/api/vecs/index-to-metrics",
+        //     get(
+        //         async |State(app_state): State<AppState>,
+        //                Query(paginated_index): Query<PaginatedIndexParam>|
+        //                -> Response {
+        //             Json(app_state.interface.get_index_to_vecids(paginated_index)).into_response()
+        //         },
+        //     ),
+        // )
+        .route(
+            "/api/metrics/{metric}",
+            get(
+                async |State(app_state): State<AppState>, Path(metric): Path<String>| -> Response {
+                    // If not found do fuzzy search but here or in interface ?
+                    Json(app_state.interface.metric_to_indexes(metric)).into_response()
+                },
+            ),
+        )
+        .route("/api/metrics/bulk", get(data::handler))
+        .route(
+            "/api/metrics/{metric}/{index}",
+            get(
+                async |uri: Uri,
+                       headers: HeaderMap,
+                       state: State<AppState>,
+                       Path((metric, index)): Path<(String, Index)>,
+                       Query(params_opt): Query<ParamsOpt>|
+                       -> Response {
+                    data::handler(
+                        uri,
+                        headers,
+                        Query(Params::from(((index, metric), params_opt))),
+                        state,
+                    )
+                    .await
+                },
+            ),
+        )
+        // !!!
+        // DEPRECATED
+        // !!!
+        .route(
+            "/api/vecs/query",
+            get(
+                async |uri: Uri,
+                       headers: HeaderMap,
+                       Query(params): Query<ParamsDeprec>,
+                       state: State<AppState>|
+                       -> Response {
+                    data::handler(uri, headers, Query(params.into()), state).await
+                },
+            ),
+        )
+        // !!!
+        // DEPRECATED
+        // !!!
+        .route(
+            "/api/vecs/{variant}",
+            get(
+                async |uri: Uri,
+                       headers: HeaderMap,
+                       Path(variant): Path<String>,
+                       Query(params_opt): Query<ParamsOpt>,
+                       state: State<AppState>|
+                       -> Response {
+                    let variant = variant.replace("-", "_");
+                    let mut split = variant.split(TO_SEPARATOR);
+
+                    let ser_index = split.next().unwrap();
+                    let Ok(index) = Index::try_from(ser_index) else {
+                        return format!("Index {ser_index} doesn't exist").into_response();
+                    };
+
+                    let params = Params::from((
+                        (index, split.collect::<Vec<_>>().join(TO_SEPARATOR)),
+                        params_opt,
+                    ));
+                    data::handler(uri, headers, Query(params), state).await
+                },
+            ),
+        )
+    }
+}

crates/brk_server/src/api/mod.rs

@@ -1,321 +1,27 @@
-use std::{
-    fs::File,
-    io::{Cursor, Read, Seek, SeekFrom},
-    str::FromStr,
-};
+use axum::{Router, response::Redirect, routing::get};
 
-use axum::{
-    Json, Router,
-    extract::{Path, Query, State},
-    http::{HeaderMap, Uri},
-    response::{IntoResponse, Redirect, Response},
-    routing::get,
-};
-use bitcoin::{Address, Network, Transaction, consensus::Decodable};
-use bitcoincore_rpc::bitcoin;
-use brk_interface::{IdParam, Index, PaginatedIndexParam, PaginationParam, Params, ParamsOpt};
-use brk_parser::XORIndex;
-use brk_structs::{
-    AddressBytesHash, AnyAddressDataIndexEnum, Bitcoin, OutputType, TxIndex, Txid, TxidPrefix,
-};
-use serde::Serialize;
-use serde_json::Number;
-use vecdb::{AnyIterableVec, VecIterator};
+use crate::api::{explorer::ApiExplorerRoutes, metrics::ApiMetricsRoutes};
 
 use super::AppState;
 
 mod explorer;
-mod vecs;
+mod metrics;
 
 pub trait ApiRoutes {
     fn add_api_routes(self) -> Self;
 }
 
-const TO_SEPARATOR: &str = "_to_";
-
-#[derive(Serialize)]
-struct TxResponse {
-    txid: Txid,
-    index: TxIndex,
-    tx: Transaction,
-}
-
 impl ApiRoutes for Router<AppState> {
     fn add_api_routes(self) -> Self {
-        self.route(
-            "/api/address/{address}",
-            get(
-                async |Path(address): Path<String>, state: State<AppState>| -> Response {
-                    let Ok(address) = Address::from_str(&address) else {
-                        return "Invalid address".into_response();
-                    };
-                    if !address.is_valid_for_network(Network::Bitcoin) {
-                        return "Invalid address".into_response();
-                    }
-                    let address = address.assume_checked();
-                    let interface = state.interface;
-                    let indexer = interface.indexer();
-                    let computer = interface.computer();
-                    let stores = &indexer.stores;
-                    let hash = AddressBytesHash::from(&address);
-
-                    let Ok(Some(addri)) = stores
-                        .addressbyteshash_to_typeindex
-                        .get(&hash)
-                        .map(|opt| opt.map(|cow| cow.into_owned())) else {
-                            return "Unknown address".into_response();
-                        };
-
-                    let output_type = OutputType::from(&address);
-                    let stateful = &computer.stateful;
-                    let price = computer.price.as_ref().map(|v| {
-                        *v.timeindexes_to_price_close
-                            .dateindex
-                            .as_ref()
-                            .unwrap()
-                            .iter()
-                            .last()
-                            .unwrap()
-                            .1
-                            .into_owned()
-                    });
-
-                    let anyaddri = match output_type {
-                        OutputType::P2PK33 => stateful
-                            .p2pk33addressindex_to_anyaddressindex
-                            .iter()
-                            .unwrap_get_inner(addri.into()),
-                        OutputType::P2PK65 => stateful
-                            .p2pk65addressindex_to_anyaddressindex
-                            .iter()
-                            .unwrap_get_inner(addri.into()),
-                        OutputType::P2PKH => stateful
-                            .p2pkhaddressindex_to_anyaddressindex
-                            .iter()
-                            .unwrap_get_inner(addri.into()),
-                        OutputType::P2SH => stateful
-                            .p2shaddressindex_to_anyaddressindex
-                            .iter()
-                            .unwrap_get_inner(addri.into()),
-                        OutputType::P2TR => stateful
-                            .p2traddressindex_to_anyaddressindex
-                            .iter()
-                            .unwrap_get_inner(addri.into()),
-                        OutputType::P2WPKH => stateful
-                            .p2wpkhaddressindex_to_anyaddressindex
-                            .iter()
-                            .unwrap_get_inner(addri.into()),
-                        OutputType::P2WSH => stateful
-                            .p2wshaddressindex_to_anyaddressindex
-                            .iter()
-                            .unwrap_get_inner(addri.into()),
-                        OutputType::P2A => stateful
-                            .p2aaddressindex_to_anyaddressindex
-                            .iter()
-                            .unwrap_get_inner(addri.into()),
-
-                        _ => unreachable!(),
-                    };
-
-                    let addr_data = match anyaddri.to_enum() {
-                        AnyAddressDataIndexEnum::Loaded(loadedi) => stateful
-                            .loadedaddressindex_to_loadedaddressdata
-                            .iter()
-                            .unwrap_get_inner(loadedi),
-                        AnyAddressDataIndexEnum::Empty(emptyi) => stateful
-                            .emptyaddressindex_to_emptyaddressdata
-                            .iter()
-                            .unwrap_get_inner(emptyi)
-                            .into(),
-                    };
-
-                    let amount = addr_data.amount();
-                    Json(serde_json::json!({
-                        "address": address,
-                        "type": output_type,
-                        "index": addri,
-                        "chain_stats": {
-                            "funded_txo_count":	serde_json::Value::Null,
-                            "funded_txo_sum": addr_data.received,
-                            "spent_txo_count": serde_json::Value::Null,
-                            "spent_txo_sum": addr_data.sent,
-                            "utxo_count": addr_data.utxos,
-                            "balance": amount,
-                            "balance_usd": price.map_or(serde_json::Value::Null, |p| serde_json::Value::Number(Number::from_f64( *(p * Bitcoin::from(amount))).unwrap())),
-                            "realized_value": addr_data.realized_cap,
-                            "tx_count":	serde_json::Value::Null,
-                            "avg_cost_basis": addr_data.realized_price()
-                        },
-                        "mempool_stats": serde_json::Value::Null
-                    }))
-                    .into_response()
-                },
-            ),
-        )
-        .route(
-            "/api/tx/{txid}",
-            get(
-                async |Path(txid): Path<String>, state: State<AppState>| -> Response {
-                    let Ok(txid) = bitcoin::Txid::from_str(&txid) else {
-                        return "Invalid txid".into_response()
-                    };
-
-                    let txid = Txid::from(txid);
-                    let prefix = TxidPrefix::from(&txid);
-                    let interface = state.interface;
-                    let indexer = interface.indexer();
-                    let Ok(Some(txindex)) = indexer
-                        .stores
-                        .txidprefix_to_txindex
-                        .get(&prefix)
-                        .map(|opt| opt.map(|cow| cow.into_owned())) else {
-                            return "Unknown transaction".into_response();
-                        };
-
-                    let txid = indexer
-                        .vecs
-                        .txindex_to_txid
-                        .iter()
-                        .unwrap_get_inner(txindex);
-
-                    let parser = interface.parser();
-                    let computer = interface.computer();
-
-                    let position = computer.blks.txindex_to_position.iter().unwrap_get_inner(txindex);
-                    let len = computer.blks.txindex_to_len.iter().unwrap_get_inner(txindex);
-
-                    let blk_index_to_blk_path = parser.blk_index_to_blk_path();
-
-                    let Some(blk_path) = blk_index_to_blk_path.get(&position.blk_index()) else {
-                        return "Unknown blk index".into_response();
-                    };
-
-                    let mut xori = XORIndex::default();
-                    xori.add_assign(position.offset() as usize);
-
-                    let Ok(mut file) = File::open(blk_path) else {
-                        return "Error opening blk file".into_response();
-                    };
-
-                    if file.seek(SeekFrom::Start(position.offset() as u64)).is_err() {
-                         return "Error seeking position in blk file".into_response();
-                    }
-
-                    let mut buffer = vec![0u8; *len as usize];
-                    if file.read_exact(&mut buffer).is_err() {
-                            return "File fail read exact".into_response();
-                    }
-                    xori.bytes(&mut buffer, parser.xor_bytes());
-
-                    let mut reader = Cursor::new(buffer);
-                    let Ok(tx) = Transaction::consensus_decode(&mut reader) else {
-                        return "Error decoding transaction".into_response();
-                    };
-
-                    let response = TxResponse { txid, index: txindex, tx };
-
-                    let bytes = sonic_rs::to_vec(&response).unwrap();
-
-                    Response::builder()
-                            .header("content-type", "application/json")
-                            .body(bytes.into())
-                            .unwrap()
-                },
-            ),
-        )
-        .route(
-            "/api/vecs/index-count",
-            get(async |State(app_state): State<AppState>| -> Response {
-                Json(app_state.interface.get_index_count()).into_response()
-            }),
-        )
-        .route(
-            "/api/vecs/id-count",
-            get(async |State(app_state): State<AppState>| -> Response {
-                Json(app_state.interface.get_vecid_count()).into_response()
-            }),
-        )
-        .route(
-            "/api/vecs/vec-count",
-            get(async |State(app_state): State<AppState>| -> Response {
-                Json(app_state.interface.get_vec_count()).into_response()
-            }),
-        )
-        .route(
-            "/api/vecs/indexes",
-            get(async |State(app_state): State<AppState>| -> Response {
-                Json(app_state.interface.get_indexes()).into_response()
-            }),
-        )
-        .route(
-            "/api/vecs/accepted-indexes",
-            get(async |State(app_state): State<AppState>| -> Response {
-                Json(app_state.interface.get_accepted_indexes()).into_response()
-            }),
-        )
-        .route(
-            "/api/vecs/ids",
-            get(
-                async |State(app_state): State<AppState>,
-                       Query(pagination): Query<PaginationParam>|
-                       -> Response {
-                    Json(app_state.interface.get_vecids(pagination)).into_response()
-                },
-            ),
-        )
-        .route(
-            "/api/vecs/index-to-ids",
-            get(
-                async |State(app_state): State<AppState>,
-                       Query(paginated_index): Query<PaginatedIndexParam>|
-                       -> Response {
-                    Json(app_state.interface.get_index_to_vecids(paginated_index)).into_response()
-                },
-            ),
-        )
-        .route(
-            "/api/vecs/id-to-indexes",
-            get(
-                async |State(app_state): State<AppState>,
-                       Query(param): Query<IdParam>|
-                       -> Response {
-                    Json(app_state.interface.get_vecid_to_indexes(param.id)).into_response()
-                },
-            ),
-        )
-        // .route("/api/vecs/variants", get(variants_handler))
-        .route("/api/vecs/query", get(vecs::handler))
-        .route(
-            "/api/vecs/{variant}",
-            get(
-                async |uri: Uri,
-                       headers: HeaderMap,
-                       Path(variant): Path<String>,
-                       Query(params_opt): Query<ParamsOpt>,
-                       state: State<AppState>|
-                       -> Response {
-                    let variant = variant.replace("-", "_");
-                    let mut split = variant.split(TO_SEPARATOR);
-
-                    if let Ok(index) = Index::try_from(split.next().unwrap()) {
-                        let params = Params::from((
-                            (index, split.collect::<Vec<_>>().join(TO_SEPARATOR)),
-                            params_opt,
-                        ));
-                        vecs::handler(uri, headers, Query(params), state).await
-                    } else {
-                        "Bad variant".into_response()
-                    }
-                },
-            ),
-        )
-        .route(
-            "/api",
-            get(|| async {
-                Redirect::temporary(
-                    "https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#api",
-                )
-            }),
-        )
+        self.add_api_explorer_routes()
+            .add_api_metrics_routes()
+            .route(
+                "/api",
+                get(|| async {
+                    Redirect::temporary(
+                        "https://github.com/bitcoinresearchkit/brk/tree/main/crates/brk_server#api",
+                    )
+                }),
+            )
     }
 }

crates/brk_server/src/lib.rs

@@ -101,7 +101,7 @@ impl Server {
             .route("/version", get(Json(VERSION)))
             .route(
                 "/health",
-                get(Json(serde_json::json!({
+                get(Json(sonic_rs::json!({
                     "status": "healthy",
                     "service": "brk-server",
                     "timestamp": jiff::Timestamp::now().to_string()

docs/TODO.md

@@ -1,21 +1,21 @@
 # TODO
 
 - __CRATES__
+  - _BUNDLER_
   - _CLI_
-    - launch
+    - UX: launch
       - if first, test read/write speed, add warning if too low (<2gb/s)
       - check available disk space
       - pull latest version and notify if out of date
-    - add custom path support for config.toml
-    - maybe add bitcoind download and launch support
-      - via: https://github.com/rust-bitcoin/corepc/blob/master/node
+    - FEAT: add custom path support for config.toml
   - _COMPUTER_
-    - **add rollback of states (in stateful)**
-    - add support for per index computation
-    - fix min fee_rate which is always ZERO due to coinbase transaction
-    - before computing multiple sources check their length, panic if not equal
-    - create usd versions of vecs structs instead of having options everywhere
-    - datasets
+    - BUG: **add rollback of states (in stateful)**
+    - FEAT: add support for per index computation
+    - FEAT: Add percentiles of cost basis weighted by amount invested compared to total invested
+    - BUG: fix min fee_rate which is always ZERO due to coinbase transaction
+    - BUG: before computing multiple sources check their length, panic if not equal
+    - DX: create usd versions of vecs structs instead of having options everywhere
+    - FEAT: datasets
       - `sats` version of all price datasets (average and co)
       - pools
         - highest dominance
@@ -39,11 +39,13 @@
       - https://checkonchain.com
       - https://researchbitcoin.net/exciting-update-coming-to-the-bitcoin-lab/
       - https://mempool.space/research
+  - _ERROR_
+  - _FETCHER_
   - _INDEXER_
-    - parse only the needed block number instead the last 100 blocks
+    - PERF: parse only the needed block number instead the last 100 blocks
       - maybe using https://developer.bitcoin.org/reference/rpc/getblockhash.html
   - _INTERFACE_
-    - Maybe change `json` to:
+    - DX: Maybe change `json` to:
       ```json
       {
         "price_close": {
@@ -56,98 +58,96 @@
         }
       }
       ```
-    - create pagination enum
+    - DX: create pagination enum
       - from to
       - from option<count>
       - to option<count>
       - page + option<per page> default 1000 max 1000
-    - from/to/count params don’t cap all combinations
+    - BUG: from/to/count params don’t cap all combinations
       - example: from -10,000 count 10, won’t work if underlying vec isn’t 10k or more long
   - _LOGGER_
-    - remove colors from file
+    - BUG: remove colors from file
+  - _MCP_
   - _PARSER_
-    - Stateless
-    - if less than X (10 maybe ?) get block using rpc instead of parsing the block files
   - _SERVER_
     - api
-      - copy mempool's rest api
+      - FEAT: copy mempool's rest api
         - https://mempool.space/docs/api/rest
-      - add extensions support (.json .csv …) instead of only format
-      - if format instead of extension then don't download file
-      - ddos protection
+      - FEAT: add extensions support (.json .csv …) instead of only format
+      - FEAT: if format instead of extension then don't download file
+      - BUG: ddos protection
         - against API params varying in range
         - search
         - fuzzy on typo
           - https://github.com/rapidfuzz/strsim-rs or stick with current impl
           - create map of all single words
           - do some kind of score with that ?
-      - discoverability
+      - FEAT: discoverability
         - catalog (tree/groups)
         - search
-      - failover to `/api`
-      - no HTML / redirects ?
-      - change `/api/vecs/{index}-to-{metric}` to `/api/{metric}/index`
-      - change `/api/vecs/query` to `/api/bulk`
-      - support keyed version when fetching dataset: {date: value} / {date: [value]}
-    - add support for https (rustls)
+      - BUG: failover to `/api`
+      - ???: no HTML / redirects ?
+      - FEAT: support keyed version when fetching dataset: {date: value} / {date: [value]}
+    - FEAT: add support for https (rustls)
   - _STORE_
-    - save height and version in one file
+    - FEAT: save height and version in one file
   - _STRUCTS_
-    - remove `checked_sub` trait ? (checked with the `dev` profile)
+  - _GLOBAL_
+    - PERF: https://davidlattimore.github.io/posts/2025/09/02/rustforge-wild-performance-tricks.html
 - __DOCS__
   - _README_
-    - add a comparison table with alternatives
-    - add contribution section where help is needed
-      - documentation/mcp/datasets/different front ends
-    - add faq
+    - FEAT: add a comparison table with alternatives
+    - FEAT: add faq
 - __WEBSITES__
   - _PACKAGES_
-    - move packages from `bitview` to `/packages` or `/websites/packages` or else
-    - move the fetching logic from `bitview` website to an independent `brk` package which could be published to npm
+    - DX: move the fetching logic from `bitview` website to an independent `brk` package which could be published to npm
       - https://www.npmjs.com/package/@mempool/mempool.js
       - auto publish with github actions
   - _BITVIEW_
-    - explorer
-      - blocks (interval as length between)
-      - transactions
-      - addresses
-      - miners
-      - maybe xpubs
-    - charts
-      - selected unit sometimes changes when going back end forth
-      - add support for custom charts
-      - price scale format depends on unit, hide digits for sats for example (if/when possible)
-      - shows certain series as [scatter plots](https://github.com/tradingview/lightweight-charts/issues/1662) with a solid sma/ema
+    - EXPLORER
+      - FEAT: blocks (interval as length between)
+      - FEAT: transactions
+      - FEAT: addresses
+      - FEAT: miners
+      - FEAT: xpubs ?
+    - CHART
+      - FEAT: Make candlesticks a bi-series with a candlestick series and a line when too zoomed out (like the auto mode)
+      - FEAT: Add min/max markers back now that they can be ignored when scaling the chart (to avoids stuttering)
+      - BUG: selected unit sometimes changes when going back end forth
+      - FEAT: add support for custom charts
+      - BUG: price scale format depends on unit, hide digits for sats for example (if/when possible)
+      - FEAT: shows certain series as [scatter plots](https://github.com/tradingview/lightweight-charts/issues/1662) with a solid sma/ema
         - mainly datasets with a big variance like raw `hash_rate`
-      - hide pane if no series on it
-      - fix (and reset) pane size (50/50) when changing charts
-      - units: add short name / long name / title
-      - verify that "compare" folders aren't missing charts/datasets
-      - legend
-        - add link to explanation for each name (to glassnode ?)
-    - table
-      - pagination
-      - exports (.json, .csv,…)
-      - improve dataset selection
-      - display 1k values (instead of 10k) but to avoid caching multiple times the same values apply everywhere
-    - search
-      - improve
-      - datasets add legend, and keywords ?
-      - support height/address/txid
-    - api
-      - add api page with interactivity
-    - glossary ?
-    - nav
-      - move share button to footer ?
-      - when clicking on already selected option, pushes to history, bad !
-    - global
-      - improve behavior when local storage is unavailable
-        - by having a global state
-      - font:
+      - BUG: hide pane if no series on it
+      - BUG: fix (and reset) pane size (50/50) when changing charts
+      - UX: units: add short name / long name / title
+      - BUG: verify that "compare" folders aren't missing charts/datasets
+      - LEGEND
+        - UX: add link to explanation for each name (to glassnode ?)
+    - TABLE
+      - FEAT: pagination
+      - FEAT: exports (.json, .csv,…)
+      - UX: improve dataset selection
+      - UX: display 1k values (instead of 10k) but to avoid caching multiple times the same values apply everywhere
+    - SEARCH
+      - UX: improve
+      - UX:datasets add legend, and keywords ?
+      - FEAT: support height/address/txid
+    - GLOSSARY
+      - FEAT: Add ?
+    - NAV
+      - UX: move share button to footer ?
+      - FEAT: add hide sidebar button
+      - BUG: when clicking on already selected option, pushes to history, bad !
+    - GLOBAL
+      - BUG: improve behavior when local storage is unavailable by having a global state, otherwise the website forgets/don't save user's settings
+      - FEAT: Add manual theme switcher, maybe in a smart way to avoid using real estate ?
+      - UI: font:
         - https://fonts.google.com/specimen/Space+Mono
-
-    - keep as many files as possible [under 14kb](https://endtimes.dev/why-your-website-should-be-under-14kb-in-size/)
-    - No classes: https://news.ycombinator.com/item?id=45287155
+    - PERF: keep as many files as possible [under 14kb](https://endtimes.dev/why-your-website-should-be-under-14kb-in-size/)
+    - DX: [No classes](https://news.ycombinator.com/item?id=45287155)
+    - UX: [Organic animations](https://courses.joshwcomeau.com/playground/magic-wand-final)
 - __GLOBAL__
   - check `TODO`s in codebase
   - rename `output` to `txout` or `vout`, `input` to `txin` or `vin`
+  - https://マリウス.com/thoughts-on-cloudflare/

modules/.gitignore

@@ -1,5 +1,5 @@
 LICENSE
-*.json
+**/*.*.*/*.json
 *webcomponent*
 README*.md
 cli*

modules/brk-client/.gitignore

@@ -0,0 +1 @@
+generated

modules/brk-client/idle.js

@@ -0,0 +1,11 @@
+/**
+ * @param {VoidFunction} callback
+ * @param {number} [timeout = 1]
+ */
+export function runWhenIdle(callback, timeout = 1) {
+  if ("requestIdleCallback" in window) {
+    requestIdleCallback(callback);
+  } else {
+    setTimeout(callback, timeout);
+  }
+}

modules/brk-client/index.js

@@ -0,0 +1,132 @@
+/**
+ * @import { IndexName } from "./generated/metrics"
+ * @import { Metric } from './metrics'
+ *
+ * @typedef {ReturnType<createClient>} BRK
+ */
+
+// client.metrics.catalog.a.b.c() -> string (uncompress inside)
+
+import { runWhenIdle } from "./idle";
+
+import { POOL_ID_TO_POOL_NAME } from "./generated/pools";
+import { INDEXES } from "./generated/metrics";
+import { hasMetric, getIndexesFromMetric } from "./metrics";
+import { VERSION } from "./generated/version";
+
+const CACHE_NAME = "__BRK_CLIENT__";
+
+/**
+ * @param {string} origin
+ */
+export function createClient(origin) {
+  /**
+   * @template T
+   * @param {(value: T) => void} callback
+   * @param {string} url
+   */
+  async function fetchJson(callback, url) {
+    /** @type {T | null} */
+    let cachedJson = null;
+
+    /** @type {Cache | undefined} */
+    let cache;
+    /** @type {Response | undefined} */
+    let cachedResponse;
+    try {
+      cache = await caches.open(CACHE_NAME);
+      cachedResponse = await cache.match(url);
+      if (cachedResponse) {
+        console.debug(`cache: ${url}`);
+        const json = /** @type {T} */ (await cachedResponse.json());
+        cachedJson = json;
+        callback(json);
+      }
+    } catch {}
+
+    try {
+      if (!navigator.onLine) {
+        throw "Offline";
+      }
+
+      console.debug(`fetch: ${url}`);
+
+      const fetchedResponse = await fetch(url, {
+        signal: AbortSignal.timeout(5000),
+      });
+      if (!fetchedResponse.ok) {
+        throw `Bad response: ${fetchedResponse}`;
+      }
+
+      if (
+        cachedResponse?.headers.get("ETag") ===
+        fetchedResponse.headers.get("ETag")
+      ) {
+        return cachedJson;
+      }
+
+      const clonedResponse = fetchedResponse.clone();
+
+      const fetchedJson = /** @type {T} */ (await fetchedResponse.json());
+      if (!fetchedJson) throw `JSON is false`;
+
+      callback(fetchedJson);
+
+      runWhenIdle(async function () {
+        try {
+          await cache?.put(url, clonedResponse);
+        } catch (_) {}
+      });
+
+      return fetchedJson;
+    } catch (e) {
+      console.error(e);
+      return cachedJson;
+    }
+  }
+
+  /**
+   * @param {Metric} metric
+   * @param {IndexName} index
+   * @param {number} [from]
+   * @param {number} [to]
+   */
+  function genMetricURL(metric, index, from, to) {
+    let path = `${origin}api/metrics/${metric.replaceAll("_", "-")}/${index}?`;
+
+    if (from !== undefined) {
+      path += `from=${from}`;
+    }
+    if (to !== undefined) {
+      if (!path.endsWith("?")) {
+        path += `&`;
+      }
+      path += `to=${to}`;
+    }
+    return path;
+  }
+
+  /**
+   * @template T
+   * @param {(v: T[]) => void} callback
+   * @param {IndexName} index
+   * @param {Metric} metric
+   * @param {number} [from]
+   * @param {number} [to]
+   */
+  function fetchMetric(callback, index, metric, from, to) {
+    return fetchJson(callback, genMetricURL(metric, index, from, to));
+  }
+
+  return {
+    VERSION,
+    POOL_ID_TO_POOL_NAME,
+    INDEXES,
+
+    hasMetric,
+    getIndexesFromMetric,
+
+    genMetricURL,
+    fetchMetric,
+  };
+}

modules/brk-client/jsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "checkJs": true,
+    "strict": true,
+    "target": "ESNext",
+    "module": "ESNext",
+    "skipLibCheck": true
+  },
+  "exclude": ["dist"]
+}

modules/brk-client/metrics.js

@@ -0,0 +1,111 @@
+import {
+  INDEX_TO_WORD,
+  COMPRESSED_METRIC_TO_INDEXES,
+} from "./generated/metrics";
+
+/**
+ * @typedef {typeof import("./generated/metrics")["COMPRESSED_METRIC_TO_INDEXES"]} MetricToIndexes
+ * @typedef {string} Metric
+ */
+
+/** @type {Record<string, number>} */
+const WORD_TO_INDEX = {};
+
+INDEX_TO_WORD.forEach((word, index) => {
+  WORD_TO_INDEX[word] = index;
+});
+
+/**
+ * @param {Metric} metric
+ */
+export function getIndexesFromMetric(metric) {
+  return COMPRESSED_METRIC_TO_INDEXES[compressMetric(metric)];
+}
+
+/**
+ * @param {Metric} metric
+ */
+export function hasMetric(metric) {
+  return compressMetric(metric) in COMPRESSED_METRIC_TO_INDEXES;
+}
+
+/**
+ * @param {string} metric
+ */
+function compressMetric(metric) {
+  return metric
+    .split("_")
+    .map((word) => {
+      const index = WORD_TO_INDEX[word];
+      return index !== undefined ? indexToLetters(index) : word;
+    })
+    .join("_");
+}
+
+/**
+ * @param {string} compressedMetric
+ */
+function decompressMetric(compressedMetric) {
+  return compressedMetric
+    .split("_")
+    .map((code) => {
+      const index = lettersToIndex(code);
+      return INDEX_TO_WORD[index] || code; // Fallback to original if not found
+    })
+    .join("_");
+}
+
+/**
+ * @param {string} letters
+ */
+function lettersToIndex(letters) {
+  let result = 0;
+  for (let i = 0; i < letters.length; i++) {
+    const value = charToIndex(letters.charCodeAt(i));
+    result = result * 52 + value + 1;
+  }
+  return result - 1;
+}
+
+/**
+ * @param {number} byte
+ */
+function charToIndex(byte) {
+  if (byte >= 65 && byte <= 90) {
+    // 'A' to 'Z'
+    return byte - 65;
+  } else if (byte >= 97 && byte <= 122) {
+    // 'a' to 'z'
+    return byte - 97 + 26;
+  } else {
+    return 255; // Invalid
+  }
+}
+
+/**
+ * @param {number} index
+ */
+function indexToLetters(index) {
+  if (index < 52) {
+    return indexToChar(index);
+  }
+  let result = [];
+  while (true) {
+    result.push(indexToChar(index % 52));
+    index = Math.floor(index / 52);
+    if (index === 0) break;
+    index -= 1;
+  }
+  return result.reverse().join("");
+}
+
+/**
+ * @param {number} index
+ */
+function indexToChar(index) {
+  if (index <= 25) {
+    return String.fromCharCode(65 + index); // A-Z
+  } else {
+    return String.fromCharCode(97 + index - 26); // a-z
+  }
+}

modules/brk-client/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "allowJs": true,
+    "checkJs": true,
+    "strict": true,
+    "target": "ESNext",
+    "module": "ESNext",
+    "outDir": "/tmp/brk",
+    "lib": ["DOM", "DOM.Iterable", "ESNext", "WebWorker"],
+    "skipLibCheck": true
+  },
+  "exclude": ["dist"]
+}

modules/brk-resources/index.js

@@ -0,0 +1,132 @@
+/**
+ * @import { Signal, Signals } from "../brk-signals/index";
+ * @import { BRK } from '../brk-client/index'
+ * @import { Metric } from '../brk-client/metrics'
+ * @import { IndexName } from '../brk-client/generated/metrics'
+ */
+
+/**
+ * @typedef {ReturnType<typeof createResources>} Resources
+ * @typedef {ReturnType<Resources["metrics"]["getOrCreate"]>} MetricResource
+ */
+
+/**
+ * @param {BRK} brk
+ * @param {Signals} signals
+ */
+export function createResources(brk, signals) {
+  const owner = signals.getOwner();
+
+  const defaultFrom = -10_000;
+  const defaultTo = undefined;
+
+  /**
+   * @param {Object} [args]
+   * @param {number} [args.from]
+   * @param {number} [args.to]
+   */
+  function genKey(args) {
+    return `${args?.from ?? defaultFrom}-${args?.to ?? ""}`;
+  }
+
+  /**
+   * @template T
+   * @param {Metric} metric
+   * @param {IndexName} index
+   */
+  function createMetricResource(metric, index) {
+    if (!brk.hasMetric(metric)) {
+      throw Error(`${metric} is invalid`);
+    }
+
+    return signals.runWithOwner(owner, () => {
+      const fetchedRecord = signals.createSignal(
+        /** @type {Map<string, {loading: boolean, at: Date | null, data: Signal<T[] | null>}>} */ (
+          new Map()
+        ),
+      );
+
+      return {
+        url: brk.genMetricURL(metric, index, defaultFrom),
+        fetched: fetchedRecord,
+        /**
+         * Defaults
+         * - from: -10_000
+         * - to: undefined
+         *
+         * @param {Object} [args]
+         * @param {number} [args.from]
+         * @param {number} [args.to]
+         */
+        async fetch(args) {
+          const from = args?.from ?? defaultFrom;
+          const to = args?.to ?? defaultTo;
+          const fetchedKey = genKey({ from, to });
+          if (!fetchedRecord().has(fetchedKey)) {
+            fetchedRecord.set((map) => {
+              map.set(fetchedKey, {
+                loading: false,
+                at: null,
+                data: signals.createSignal(/** @type {T[] | null} */ (null), {
+                  equals: false,
+                }),
+              });
+              return map;
+            });
+          }
+          const fetched = fetchedRecord().get(fetchedKey);
+          if (!fetched) throw Error("Unreachable");
+          if (fetched.loading) return fetched.data();
+          if (fetched.at) {
+            const diff = new Date().getTime() - fetched.at.getTime();
+            const ONE_MINUTE_IN_MS = 60_000;
+            if (diff < ONE_MINUTE_IN_MS) return fetched.data();
+          }
+          fetched.loading = true;
+          const res = /** @type {T[] | null} */ (
+            await brk.fetchMetric(
+              (data) => {
+                if (data.length || !fetched.data()) {
+                  fetched.data.set(data);
+                }
+              },
+              index,
+              metric,
+              from,
+              to,
+            )
+          );
+          fetched.at = new Date();
+          fetched.loading = false;
+          return res;
+        },
+      };
+    });
+  }
+
+  /** @type {Map<string, NonNullable<ReturnType<typeof createMetricResource>>>} */
+  const map = new Map();
+
+  const metrics = {
+    /**
+     * @template T
+     * @param {Metric} metric
+     * @param {IndexName} index
+     */
+    getOrCreate(metric, index) {
+      const key = `${metric}/${index}`;
+      const found = map.get(key);
+      if (found) {
+        return found;
+      }
+
+      const resource = createMetricResource(metric, index);
+      if (!resource) throw Error("metric is undefined");
+      map.set(key, /** @type {any} */ (resource));
+      return resource;
+    },
+    genKey,
+  };
+
+  return { metrics };
+}

modules/brk-resources/jsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "checkJs": true,
+    "strict": true,
+    "target": "ESNext",
+    "module": "ESNext",
+    "skipLibCheck": true
+  },
+  "exclude": ["dist"]
+}

modules/brk-resources/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "allowJs": true,
+    "checkJs": true,
+    "strict": true,
+    "target": "ESNext",
+    "module": "ESNext",
+    "outDir": "/tmp/brk",
+    "lib": ["DOM", "DOM.Iterable", "ESNext", "WebWorker"],
+    "skipLibCheck": true
+  },
+  "exclude": ["dist"]
+}

modules/brk-signals/index.js

@@ -1,11 +1,12 @@
-// @ts-check
-
 /**
- * @import { SignalOptions } from "./0.4.1/dist/types/core/core"
- * @import { getOwner as GetOwner, onCleanup as OnCleanup } from "./0.4.1/dist/types/core/owner"
- * @import { createSignal as CreateSignal, createEffect as CreateEffect, createMemo as CreateMemo, createRoot as CreateRoot, runWithOwner as RunWithOwner, Setter } from "./0.4.1/dist/types/signals";
+ * @import { SignalOptions } from "../solidjs-signals/0.6.3/dist/types/core/core"
+ * @import { getOwner as GetOwner, onCleanup as OnCleanup } from "../solidjs-signals/0.6.3/dist/types/core/owner"
+ * @import { createSignal as CreateSignal, createEffect as CreateEffect, createMemo as CreateMemo, createRoot as CreateRoot, runWithOwner as RunWithOwner, Setter } from "../solidjs-signals/0.6.3/dist/types/signals";
  */
 
+// test
+// wkopwfk
+
 /**
  * @template T
  * @typedef {() => T} Accessor
@@ -24,7 +25,7 @@ import {
   createRoot,
   runWithOwner,
   onCleanup,
-} from "./0.4.1/dist/prod.js";
+} from "../solidjs-signals/0.6.3/dist/prod.js";
 
 let effectCount = 0;
 

modules/brk-signals/jsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "checkJs": true,
+    "strict": true,
+    "target": "ESNext",
+    "module": "ESNext",
+    "skipLibCheck": true
+  },
+  "exclude": ["dist"]
+}

modules/brk-signals/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "allowJs": true,
+    "checkJs": true,
+    "strict": true,
+    "target": "ESNext",
+    "module": "ESNext",
+    "outDir": "/tmp/brk",
+    "lib": ["DOM", "DOM.Iterable", "ESNext", "WebWorker"],
+    "skipLibCheck": true
+  },
+  "exclude": ["dist"]
+}

modules/lean-qr/2.6.0/index.d.ts

@@ -40,10 +40,19 @@ declare module 'lean-qr' {
     /** the text to use for linefeeds between rows */
     lf?: string;
 
-    /** the padding to apply on the left and right of the output (populated with 'off' modules) */
+    /** the padding to apply around the output (populated with 'off' modules) */
+    pad?: number;
+
+    /**
+     * the padding to apply on the left and right of the output (populated with 'off' modules)
+     * @deprecated use `pad` instead
+     */
     padX?: number;
 
-    /** the padding to apply on the top and bottom of the output (populated with 'off' modules) */
+    /**
+     * the padding to apply on the top and bottom of the output (populated with 'off' modules)
+     * @deprecated use `pad` instead
+     */
     padY?: number;
   }
 
@@ -54,10 +63,19 @@ declare module 'lean-qr' {
     /** the colour to use for modules which are 'off' (typically white) */
     off?: RGBA;
 
-    /** the padding to apply on the left and right of the output (filled with 'off') */
+    /** the padding to apply around the output (filled with 'off') */
+    pad?: number;
+
+    /**
+     * the padding to apply on the left and right of the output (filled with 'off')
+     * @deprecated use `pad` instead
+     */
     padX?: number;
 
-    /** the padding to apply on the top and bottom of the output (filled with 'off') */
+    /**
+     * the padding to apply on the top and bottom of the output (filled with 'off')
+     * @deprecated use `pad` instead
+     */
     padY?: number;
   }
 
@@ -167,7 +185,10 @@ declare module 'lean-qr' {
 
   export type Correction = number & { readonly _: unique symbol };
   export const correction: Readonly<{
-    /** minimum possible correction level (same as L) */
+    /**
+     * minimum possible correction level (same as L)
+     * @deprecated use correction.L
+     */
     min: Correction;
     /** ~7.5% error tolerance, ~25% data overhead */
     L: Correction;
@@ -177,7 +198,10 @@ declare module 'lean-qr' {
     Q: Correction;
     /** ~30% error tolerance, ~190% data overhead */
     H: Correction;
-    /** maximum possible correction level (same as H) */
+    /**
+     * maximum possible correction level (same as H)
+     * @deprecated use correction.H
+     */
     max: Correction;
   }>;
 
@@ -215,6 +239,8 @@ declare module 'lean-qr' {
      * @param modes the modes to add.
      * @returns a `generate` function which will additionally consider the
      * given modes when using auto encoding.
+     *
+     * @deprecated this will be removed in version 3. Prefer passing an explicit list of modes when calling `generate`.
      */
     with(...modes: ReadonlyArray<ModeFactory>): GenerateFn;
   }
@@ -263,9 +289,17 @@ declare module 'lean-qr/extras/svg' {
     on?: string;
     /** the colour to use for modules which are 'off' (typically white) */
     off?: string;
-    /** the padding to apply on the left and right of the output (filled with 'off') */
+    /** the padding to apply around the output (filled with 'off') */
+    pad?: number;
+    /**
+     * the padding to apply on the left and right of the output (filled with 'off')
+     * @deprecated use `pad` instead
+     */
     padX?: number;
-    /** the padding to apply on the top and bottom of the output (filled with 'off') */
+    /**
+     * the padding to apply on the top and bottom of the output (filled with 'off')
+     * @deprecated use `pad` instead
+     */
     padY?: number;
     /** a width to apply to the resulting image (overrides `scale`) */
     width?: number | null;
@@ -337,9 +371,17 @@ declare module 'lean-qr/extras/node_export' {
     on?: RGBA;
     /** the colour to use for modules which are 'off' (typically white) */
     off?: RGBA;
-    /** the padding to apply on the left and right of the output (filled with 'off') */
+    /** the padding to apply around the output (filled with 'off') */
+    pad?: number;
+    /**
+     * the padding to apply on the left and right of the output (filled with 'off')
+     * @deprecated use `pad` instead
+     */
     padX?: number;
-    /** the padding to apply on the top and bottom of the output (filled with 'off') */
+    /**
+     * the padding to apply on the top and bottom of the output (filled with 'off')
+     * @deprecated use `pad` instead
+     */
     padY?: number;
     /** a scale to apply to the resulting image (`scale` pixels = 1 module) */
     scale?: number;
@@ -413,6 +455,13 @@ declare module 'lean-qr/extras/react' {
    * Generate an asynchronous QR component (rendering to a `canvas`).
    * You should call this just once, in the global scope.
    *
+   * ```js
+   * import * as React from 'react';
+   * import { generate } from 'lean-qr';
+   * import { makeAsyncComponent } from 'lean-qr/extras/react';
+   * const QR = makeAsyncComponent(React, generate);
+   * ```
+   *
    * This is not suitable for server-side rendering (use `makeSyncComponent`
    * instead).
    *
@@ -455,6 +504,14 @@ declare module 'lean-qr/extras/react' {
    * Generate a synchronous QR component (rendering to an SVG).
    * You should call this just once, in the global scope.
    *
+   * ```js
+   * import * as React from 'react';
+   * import { generate } from 'lean-qr';
+   * import { toSvgDataURL } from 'lean-qr/extras/svg';
+   * import { makeSyncComponent } from 'lean-qr/extras/react';
+   * const QR = makeSyncComponent(React, generate, toSvgDataURL);
+   * ```
+   *
    * This is best suited for server-side rendering (prefer
    * `makeAsyncComponent` if you only need client-side rendering).
    *
@@ -478,6 +535,114 @@ declare module 'lean-qr/extras/react' {
   ): SyncQRComponent<T>;
 }
 
+declare module 'lean-qr/extras/vue' {
+  import type {
+    Bitmap2D as FullBitmap2D,
+    GenerateOptions,
+    ImageDataOptions,
+  } from 'lean-qr';
+  import type {
+    SVGOptions,
+    toSvgDataURL as toSvgDataURLFn,
+  } from 'lean-qr/extras/svg';
+
+  export interface Framework<T> {
+    h:
+      | ((type: 'canvas', props: { ref: string; style: string }) => T)
+      | ((type: 'img', props: { src: string; style: string }) => T);
+  }
+
+  interface QRComponentProps {
+    content: string;
+  }
+
+  export interface VueCanvasComponentProps
+    extends ImageDataOptions,
+      GenerateOptions,
+      QRComponentProps {}
+
+  type VueComponentDefinition<Props, Node> = {
+    props: {
+      [k in keyof Props]: {
+        type: {
+          (): Props[k];
+          required: undefined extends Props[k] ? false : true;
+        };
+      };
+    };
+    render: () => Node;
+  } & ThisType<unknown>;
+
+  /**
+   * Generate a QR component which renders to a `canvas`.
+   * You should call this just once, in the global scope.
+   *
+   * ```js
+   * import { h, defineComponent } from 'vue';
+   * import { generate } from 'lean-qr';
+   * import { makeVueCanvasComponent } from 'lean-qr/extras/vue';
+   * export const QR = defineComponent(makeVueCanvasComponent({ h }, generate));
+   * ```
+   *
+   * This is not suitable for server-side rendering (use `makeSyncComponent`
+   * instead).
+   *
+   * @param framework the framework to use (e.g. `{ h }`).
+   * @param generate the `generate` function to use
+   * (from `lean-qr` or `lean-qr/nano`).
+   * @param defaultProps optional default properties to apply when the
+   * component is used (overridden by properties set on use).
+   * @returns a component which can be rendered elsewhere.
+   */
+  export function makeVueCanvasComponent<T>(
+    framework: Readonly<Framework<T>>,
+    generate: (
+      data: string,
+      options?: Readonly<GenerateOptions>,
+    ) => Pick<FullBitmap2D, 'toCanvas'>,
+    defaultProps?: Readonly<Partial<VueCanvasComponentProps>>,
+  ): VueComponentDefinition<Partial<VueCanvasComponentProps>, T>;
+
+  export interface VueSVGComponentProps
+    extends SVGOptions,
+      GenerateOptions,
+      QRComponentProps {}
+
+  /**
+   * Generate a QR component which renders to an SVG.
+   * You should call this just once, in the global scope:
+   *
+   * ```js
+   * import { h, defineComponent } from 'vue';
+   * import { generate } from 'lean-qr';
+   * import { toSvgDataURL } from 'lean-qr/extras/svg';
+   * import { makeVueSvgComponent } from 'lean-qr/extras/vue';
+   * export const QR = defineComponent(makeVueSvgComponent({ h }, generate, toSvgDataURL));
+   * ```
+   *
+   * This is best suited for server-side rendering (prefer
+   * `makeAsyncComponent` if you only need client-side rendering).
+   *
+   * @param framework the framework to use (e.g. `{ h }`).
+   * @param generate the `generate` function to use
+   * (from `lean-qr` or `lean-qr/nano`).
+   * @param toSvgDataURL the `toSvgDataURL` function to use
+   * (from `lean-qr/extras/svg`).
+   * @param defaultProps optional default properties to apply when the
+   * component is used (overridden by properties set on use).
+   * @returns a component which can be rendered elsewhere.
+   */
+  export function makeVueSvgComponent<T>(
+    framework: Readonly<Framework<T>>,
+    generate: (
+      data: string,
+      options?: Readonly<GenerateOptions>,
+    ) => Pick<FullBitmap2D, 'size' | 'get'>,
+    toSvgDataURL: typeof toSvgDataURLFn,
+    defaultProps?: Readonly<Partial<VueSVGComponentProps>>,
+  ): VueComponentDefinition<Partial<VueSVGComponentProps>, T>;
+}
+
 declare module 'lean-qr/extras/errors' {
   /**
    * Convert an error into a human-readable message. This is intended for use

modules/solidjs-signals/0.6.3/dist/types/boundaries.d.ts

@@ -6,7 +6,8 @@ export declare class CollectionQueue extends Queue {
     _disabled: Computation<boolean>;
     constructor(type: number);
     run(type: number): void;
-    notify(node: Effect, type: number, flags: number): boolean;
+    notify(node: Effect, type: number, flags: number): any;
+    merge(queue: CollectionQueue): void;
 }
 export declare enum BoundaryMode {
     VISIBLE = "visible",

modules/solidjs-signals/0.6.3/dist/types/core/core.d.ts

@@ -28,6 +28,7 @@
  */
 import { type Flags } from "./flags.js";
 import { Owner } from "./owner.js";
+import { type Transition } from "./scheduler.js";
 export interface SignalOptions<T> {
     id?: string;
     name?: string;
@@ -35,19 +36,22 @@ export interface SignalOptions<T> {
     pureWrite?: boolean;
     unobserved?: () => void;
 }
-interface SourceType {
+export interface SourceType {
     _observers: ObserverType[] | null;
     _unobserved?: () => void;
     _updateIfNecessary: () => void;
     _stateFlags: Flags;
     _time: number;
+    _transition?: Transition;
+    _cloned?: Computation;
 }
-interface ObserverType {
+export interface ObserverType {
     _sources: SourceType[] | null;
     _notify: (state: number, skipQueue?: boolean) => void;
     _handlerMask: Flags;
     _notifyFlags: (mask: Flags, newFlags: Flags) => void;
     _time: number;
+    _cloned?: Computation;
 }
 /**
  * Returns the current observer.
@@ -71,6 +75,8 @@ export declare class Computation<T = any> extends Owner implements SourceType, O
     _handlerMask: number;
     _time: number;
     _forceNotify: boolean;
+    _transition?: Transition | undefined;
+    _cloned?: Computation;
     constructor(initialValue: T | undefined, compute: null | ((p?: T) => T), options?: SignalOptions<T>);
     _read(): T;
     /**
@@ -154,4 +160,3 @@ export declare function runWithObserver<T>(observer: Computation, run: () => T):
  */
 export declare function compute<T>(owner: Owner | null, fn: (val: T) => T, observer: Computation<T>): T;
 export declare function compute<T>(owner: Owner | null, fn: (val: undefined) => T, observer: null): T;
-export {};

modules/solidjs-signals/0.6.3/dist/types/core/effect.d.ts

@@ -1,5 +1,6 @@
 import { EFFECT_RENDER, EFFECT_USER } from "./constants.js";
 import { Computation, type SignalOptions } from "./core.js";
+import { type Flags } from "./flags.js";
 /**
  * Effects are the leaf nodes of our reactive graph. When their sources change, they are
  * automatically added to the queue of effects to re-execute, which will cause them to fetch their
@@ -18,6 +19,7 @@ export declare class Effect<T = any> extends Computation<T> {
     });
     write(value: T, flags?: number): T;
     _notify(state: number, skipQueue?: boolean): void;
+    _notifyFlags(mask: Flags, newFlags: Flags): void;
     _setError(error: unknown): void;
     _disposeNode(): void;
     _run(type: number): void;

modules/solidjs-signals/0.6.3/dist/types/core/index.d.ts

@@ -2,6 +2,6 @@ export { ContextNotFoundError, NoOwnerError, NotReadyError } from "./error.js";
 export { Owner, createContext, getContext, setContext, hasContext, getOwner, onCleanup, type Context, type ContextRecord, type Disposable } from "./owner.js";
 export { Computation, getObserver, isEqual, untrack, hasUpdated, isPending, latest, UNCHANGED, compute, runWithObserver, type SignalOptions } from "./core.js";
 export { Effect, EagerComputation } from "./effect.js";
-export { flush, Queue, incrementClock, getClock, type IQueue } from "./scheduler.js";
+export { flush, Queue, incrementClock, transition, ActiveTransition, type IQueue } from "./scheduler.js";
 export * from "./constants.js";
 export * from "./flags.js";

modules/solidjs-signals/0.6.3/dist/types/core/scheduler.d.ts

@@ -0,0 +1,86 @@
+import type { Computation, ObserverType, SourceType } from "./core.js";
+import type { Effect } from "./effect.js";
+export declare let clock: number;
+export declare function incrementClock(): void;
+export declare let ActiveTransition: Transition | null;
+export declare let Unobserved: SourceType[];
+export type QueueCallback = (type: number) => void;
+export interface IQueue {
+    enqueue(type: number, fn: QueueCallback): void;
+    run(type: number): boolean | void;
+    flush(): void;
+    addChild(child: IQueue): void;
+    removeChild(child: IQueue): void;
+    created: number;
+    notify(...args: any[]): boolean;
+    merge(queue: IQueue): void;
+    _parent: IQueue | null;
+    _cloned?: IQueue | undefined;
+}
+export declare class Queue implements IQueue {
+    _parent: IQueue | null;
+    _running: boolean;
+    _queues: [QueueCallback[], QueueCallback[]];
+    _children: IQueue[];
+    created: number;
+    enqueue(type: number, fn: QueueCallback): void;
+    run(type: number): void;
+    flush(): void;
+    addChild(child: IQueue): any;
+    removeChild(child: IQueue): any;
+    notify(...args: any[]): boolean;
+    merge(queue: Queue): void;
+}
+export declare const globalQueue: Queue;
+/**
+ * By default, changes are batched on the microtask queue which is an async process. You can flush
+ * the queue synchronously to get the latest updates by calling `flush()`.
+ */
+export declare function flush(): void;
+export declare function removeSourceObservers(node: ObserverType, index: number): void;
+export declare class Transition implements IQueue {
+    _sources: Map<Computation, Computation>;
+    _pendingNodes: Set<Effect>;
+    _promises: Set<Promise<any>>;
+    _optimistic: Set<(() => void) & {
+        _transition?: Transition;
+    }>;
+    _done: Transition | boolean;
+    _queues: [QueueCallback[], QueueCallback[]];
+    _clonedQueues: Map<Queue, Queue>;
+    _pureQueue: QueueCallback[];
+    _children: IQueue[];
+    _parent: IQueue | null;
+    _running: boolean;
+    _scheduled: boolean;
+    _cloned: Queue;
+    created: number;
+    constructor();
+    enqueue(type: number, fn: QueueCallback): void;
+    run(type: number): void;
+    flush(): void;
+    addChild(child: IQueue): void;
+    removeChild(child: IQueue): void;
+    notify(node: Effect, type: number, flags: number): boolean;
+    merge(queue: Transition): void;
+    schedule(): void;
+    runTransition(fn: () => any | Promise<any>, force?: boolean): void;
+    addOptimistic(fn: (() => void) & {
+        _transition?: Transition;
+    }): void;
+}
+/**
+ * Runs the given function in a transition scope, allowing for batch updates and optimizations.
+ * This is useful for grouping multiple state updates together to avoid unnecessary re-renders.
+ *
+ * @param fn A function that receives a resume function to continue the transition.
+ * The resume function can be called with another function to continue the transition.
+ *
+ * @description https://docs.solidjs.com/reference/advanced-reactivity/transition
+ */
+export declare function transition(fn: (resume: (fn: () => any | Promise<any>) => void) => any | Promise<any> | Iterable<any>): void;
+export declare function cloneGraph(node: Computation): Computation;
+export declare function getOGSource<T extends Computation>(input: T): T;
+export declare function getTransitionSource<T extends Computation>(input: T): T;
+export declare function getQueue(node: Computation): IQueue;
+export declare function initialDispose(node: any): void;