Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-24 06:39:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: convert brk_vecs to its own crates and repo (seqdb/vecdb) + changes

Browse Source
This commit is contained in:
nym21
2025-08-10 12:49:41 +02:00
parent c85592eefe
commit 5e8c7da4df
168 changed files with 1779 additions and 9538 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
Cargo.lock generated
View File

@@ -494,7 +494,7 @@ dependencies = [
"brk_server",
"brk_store",
"brk_structs",
"brk_vecs",
"vecdb",
]
[[package]]
@@ -521,7 +521,6 @@ dependencies = [
"brk_logger",
"brk_parser",
"brk_server",
"brk_vecs",
"clap",
"clap_derive",
"color-eyre",
@@ -530,6 +529,7 @@ dependencies = [
"serde",
"tokio",
"toml",
"vecdb",
"zip",
]
@@ -545,11 +545,11 @@ dependencies = [
"brk_logger",
"brk_parser",
"brk_structs",
"brk_vecs",
"derive_deref",
"log",
"rayon",
"serde",
"vecdb",
"zerocopy",
"zerocopy-derive",
]
@@ -559,11 +559,11 @@ name = "brk_error"
version = "0.0.85"
dependencies = [
"bitcoincore-rpc",
"brk_vecs",
"fjall",
"jiff",
"minreq",
"serde_json",
"vecdb",
"zerocopy",
]
@@ -590,10 +590,10 @@ dependencies = [
"brk_parser",
"brk_store",
"brk_structs",
"brk_vecs",
"fjall",
"log",
"rayon",
"vecdb",
]
[[package]]
@@ -604,13 +604,13 @@ dependencies = [
"brk_error",
"brk_indexer",
"brk_structs",
"brk_vecs",
"derive_deref",
"schemars 1.0.4",
"serde",
"serde_json",
"serde_with",
"tabled",
"vecdb",
]
[[package]]
@@ -650,9 +650,9 @@ dependencies = [
[[package]]
name = "brk_rmcp"
version = "0.4.1"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0faa9da00a4fc50c5a4f331850c37bf69529e7a88890e08962c1e9a32ca11dbd"
checksum = "34e99c51043db05e5d77c381124c1705c9a360f9a88bef0af44397134929d730"
dependencies = [
"base64 0.22.1",
"brk_rmcp-macros",
@@ -680,9 +680,9 @@ dependencies = [
[[package]]
name = "brk_rmcp-macros"
version = "0.4.1"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6078c726658d7abc2ec1b55196791fcfb58f20f1dd9f78401c749c5138a79562"
checksum = "c19a9e21ea5789ad190ce5c572bdbba50e589f9dd01acd19600b86442b56f02d"
dependencies = [
"darling 0.21.1",
"proc-macro2",
@@ -1025,7 +1025,6 @@ dependencies = [
"brk_logger",
"brk_mcp",
"brk_parser",
"brk_vecs",
"jiff",
"log",
"quick_cache",
@@ -1033,6 +1032,7 @@ dependencies = [
"tokio",
"tower-http",
"tracing",
"vecdb",
]
[[package]]
@@ -1065,44 +1065,17 @@ dependencies = [
"bitcoin",
"bitcoincore-rpc",
"brk_error",
"brk_vecs",
"byteview",
"derive_deref",
"jiff",
"rapidhash",
"serde",
"serde_bytes",
"vecdb",
"zerocopy",
"zerocopy-derive",
]
[[package]]
name = "brk_vecs"
version = "0.0.85"
dependencies = [
"brk_vecs_macros",
"ctrlc",
"libc",
"log",
"memmap2",
"parking_lot",
"pco",
"rayon",
"serde",
"serde_derive",
"serde_json",
"zerocopy",
"zerocopy-derive",
]
[[package]]
name = "brk_vecs_macros"
version = "0.0.85"
dependencies = [
"quote",
"syn 2.0.104",
]
[[package]]
name = "brotli"
version = "8.0.1"
@@ -1168,9 +1141,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.31"
version = "1.2.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3a42d84bb6b69d3a8b3eaacf0d88f179e1929695e1ad012b6cf64d9caaa5fd2"
checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e"
dependencies = [
"jobserver",
"libc",
@@ -1990,9 +1963,9 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "hashbrown"
version = "0.15.4"
version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"allocator-api2",
"equivalent",
@@ -2268,7 +2241,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661"
dependencies = [
"equivalent",
"hashbrown 0.15.4",
"hashbrown 0.15.5",
"serde",
]
@@ -2812,7 +2785,7 @@ checksum = "5c67f2b817263a72b6cc5b46e32467bc4077496f88315ce08c1796647fed84d4"
dependencies = [
"allocator-api2",
"bumpalo",
"hashbrown 0.15.4",
"hashbrown 0.15.5",
"oxc_data_structures",
"oxc_estree",
"rustc-hash",
@@ -3314,7 +3287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca"
dependencies = [
"fixedbitset",
"hashbrown 0.15.4",
"hashbrown 0.15.5",
"indexmap 2.10.0",
"serde",
]
@@ -3498,7 +3471,7 @@ checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786"
dependencies = [
"ahash",
"equivalent",
"hashbrown 0.15.4",
"hashbrown 0.15.5",
"parking_lot",
]
@@ -3597,9 +3570,9 @@ dependencies = [
[[package]]
name = "rapidhash"
version = "3.0.0"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ef3d82b018f786967b1a5d34a08ebc3c7a9ab35b5bcbe3e2e057a0a453f26c8"
checksum = "efee4b7317469c6c6e7fdeee3d094313af846a97678d6ed971d83a852d730083"
[[package]]
name = "rayon"
@@ -3685,7 +3658,7 @@ version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "145bb27393fe455dd64d6cbc8d059adfa392590a45eadf079c01b11857e7b010"
dependencies = [
"hashbrown 0.15.4",
"hashbrown 0.15.5",
"memchr",
]
@@ -3772,9 +3745,9 @@ dependencies = [
[[package]]
name = "rustversion"
version = "1.0.21"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "ryu"
@@ -3888,6 +3861,19 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
[[package]]
name = "seqdb"
version = "0.0.2"
dependencies = [
"libc",
"log",
"memmap2",
"parking_lot",
"rayon",
"zerocopy",
"zerocopy-derive",
]
[[package]]
name = "serde"
version = "1.0.219"
@@ -4050,9 +4036,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
[[package]]
name = "slab"
version = "0.4.10"
version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d"
checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
[[package]]
name = "smallvec"
@@ -4700,6 +4686,32 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]]
name = "vecdb"
version = "0.0.2"
dependencies = [
"ctrlc",
"log",
"parking_lot",
"pco",
"rayon",
"seqdb",
"serde",
"serde_derive",
"serde_json",
"vecdb_derive",
"zerocopy",
"zerocopy-derive",
]
[[package]]
name = "vecdb_derive"
version = "0.0.2"
dependencies = [
"quote",
"syn 2.0.104",
]
[[package]]
name = "version_check"
version = "0.9.5"

4
Cargo.toml
View File

@@ -38,8 +38,8 @@ brk_mcp = { version = "0.0.85", path = "crates/brk_mcp" }
brk_parser = { version = "0.0.85", path = "crates/brk_parser" }
brk_server = { version = "0.0.85", path = "crates/brk_server" }
brk_store = { version = "0.0.85", path = "crates/brk_store" }
brk_vecs = { version = "0.0.85", path = "crates/brk_vecs" }
brk_vecs_macros = { version = "0.0.85", path = "crates/brk_vecs_macros" }
# vecdb = { version = "0.0.2", features = ["derive"]}
vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]}
byteview = "=0.6.1"
derive_deref = "1.1.1"
fjall = "2.11.2"

4
README.md
View File

@@ -68,8 +68,8 @@ In contrast, existing alternatives tend to be either [very costly](https://studi
- [`brk_server`](https://crates.io/crates/brk_server): A server with an API for anything from BRK
- [`brk_store`](https://crates.io/crates/brk_store): A thin wrapper around [`fjall`](https://crates.io/crates/fjall)
- [`brk_structs`](https://crates.io/crates/brk_structs): Structs used throughout BRK
- [`brk_vecs`](https://crates.io/crates/brk_vecs): A KISS index/value store
- [`brk_vecs_macros`](https://crates.io/crates/brk_vecs_macros): Macros for [`brk_vecs`](https://crates.io/crates/brk_vecs)
- [`vecdb`](https://crates.io/crates/vecdb): A KISS index-value storage engine
- [`vecdb_macros`](https://crates.io/crates/vecdb_macros): Macros for [`vecdb`](https://crates.io/crates/vecdb)
## Hosting as a service

5
TODO.md
View File

@@ -60,11 +60,6 @@
- add extensions support (.json .csv …)
- if format instead of extension then don't download file
- add support for https (rustls)
- lru cache
- _vec_
- add native lock file support (once it's available in stable rust)
- improve compressed mode (slow reads)
- add ema support
- __docs__
- _README_
- add a comparison table with alternatives

4
crates/brk/Cargo.toml
View File

@@ -37,7 +37,7 @@ parser = ["brk_parser"]
server = ["brk_server"]
store = ["brk_store"]
structs = ["brk_structs"]
vecs = ["brk_vecs"]
vecs = ["vecdb"]
[dependencies]
brk_bundler = { workspace = true, optional = true }
@@ -53,7 +53,7 @@ brk_parser = { workspace = true, optional = true }
brk_server = { workspace = true, optional = true }
brk_store = { workspace = true, optional = true }
brk_structs = { workspace = true, optional = true }
brk_vecs = { workspace = true, optional = true }
vecdb = { workspace = true, optional = true }
[package.metadata.docs.rs]
all-features = true

2
crates/brk/src/lib.rs
View File

@@ -53,4 +53,4 @@ pub use brk_store as store;
#[cfg(feature = "vecs")]
#[doc(inline)]
pub use brk_vecs as vecs;
pub use vecdb as vecs;

2
crates/brk_cli/Cargo.toml
View File

@@ -18,7 +18,7 @@ brk_interface = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_server = { workspace = true }
brk_vecs = { workspace = true }
vecdb = { workspace = true }
clap = { version = "4.5.43", features = ["string"] }
clap_derive = "4.5.41"
color-eyre = "0.6.5"

21
crates/brk_cli/src/config.rs
View File

@@ -32,11 +32,16 @@ pub struct Config {
#[arg(long, value_name = "PATH")]
brkdir: Option<String>,
/// Activate fetching prices from exchanges APIs and the computation of all related datasets, default: true, saved
/// Activate fetching prices from BRK's API and the computation of all price related datasets, default: true, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short = 'F', long, value_name = "BOOL")]
fetch: Option<bool>,
/// Activate fetching prices from exchanges APIs if `fetch` is also set to `true`, default: true, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(long, value_name = "BOOL")]
exchanges: Option<bool>,
/// Website served by the server (if active), default: default, saved
#[serde(default, deserialize_with = "default_on_error")]
#[arg(short, long)]
@@ -117,6 +122,10 @@ impl Config {
config_saved.fetch = Some(fetch);
}
if let Some(exchanges) = config_args.exchanges.take() {
config_saved.exchanges = Some(exchanges);
}
if let Some(website) = config_args.website.take() {
config_saved.website = Some(website);
}
@@ -196,7 +205,9 @@ impl Config {
if self.rpc_auth().is_err() {
println!(
"No way found to authenticate the RPC client, please either set --rpccookiefile or --rpcuser and --rpcpassword.\nRun the program with '-h' for help."
"Unsuccessful authentication with the RPC client.
First make sure that `bitcoind` is running. If it is then please either set --rpccookiefile or --rpcuser and --rpcpassword as the default values seemed to have failed.
Finally, you can run the program with '-h' for help."
);
std::process::exit(1);
}
@@ -311,9 +322,13 @@ impl Config {
self.fetch.is_none_or(|b| b)
}
pub fn exchanges(&self) -> bool {
self.exchanges.is_none_or(|b| b)
}
pub fn fetcher(&self) -> Option<Fetcher> {
self.fetch()
.then(|| Fetcher::import(Some(self.harsdir().as_path())).unwrap())
.then(|| Fetcher::import(self.exchanges(), Some(self.harsdir().as_path())).unwrap())
}
pub fn check_collisions(&self) -> bool {

6
crates/brk_cli/src/lib.rs
View File

@@ -14,8 +14,8 @@ use brk_computer::Computer;
use brk_indexer::Indexer;
use brk_interface::Interface;
use brk_server::{Server, VERSION};
use brk_vecs::Exit;
use log::info;
use vecdb::Exit;
mod bridge;
mod config;
@@ -136,9 +136,9 @@ pub fn run() -> color_eyre::Result<()> {
info!("{} blocks found.", block_count + 1);
let starting_indexes =
indexer.index(&parser, rpc, &exit, config.check_collisions())?;
indexer.index(&parser, rpc, &exit, config.check_collisions()).unwrap();
computer.compute(&indexer, starting_indexes, &exit)?;
computer.compute(&indexer, starting_indexes, &exit).unwrap();
if let Some(delay) = config.delay() {
sleep(Duration::from_secs(delay))

2
crates/brk_computer/Cargo.toml
View File

@@ -17,7 +17,7 @@ brk_fetcher = { workspace = true }
brk_indexer = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_vecs = { workspace = true }
vecdb = { workspace = true }
derive_deref = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }

4
crates/brk_computer/examples/computer.rs
View File

@@ -9,7 +9,7 @@ use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_vecs::Exit;
use vecdb::Exit;
pub fn main() -> Result<()> {
brk_logger::init(Some(Path::new(".log")));
@@ -38,7 +38,7 @@ pub fn main() -> Result<()> {
let mut indexer = Indexer::forced_import(&outputs_dir)?;
let fetcher = Fetcher::import(None)?;
let fetcher = Fetcher::import(true, None)?;
let mut computer = Computer::forced_import(&outputs_dir, &indexer, Some(fetcher))?;

36
crates/brk_computer/src/blocks.rs
View File

@@ -1,4 +1,4 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
@@ -6,8 +6,8 @@ use brk_structs::{
CheckedSub, DifficultyEpoch, HalvingEpoch, Height, StoredU32, StoredU64, Timestamp, Version,
Weight,
};
use brk_vecs::{
AnyCollectableVec, Computation, EagerVec, Exit, File, Format, PAGE_SIZE, VecIterator,
use vecdb::{
AnyCollectableVec, Computation, Database, EagerVec, Exit, Format, PAGE_SIZE, VecIterator,
};
use crate::grouped::Source;
@@ -22,7 +22,7 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub height_to_interval: EagerVec<Height, Timestamp>,
pub height_to_vbytes: EagerVec<Height, StoredU64>,
@@ -44,18 +44,18 @@ impl Vecs {
format: Format,
indexes: &indexes::Vecs,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("blocks"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
let db = Database::open(&parent.join("blocks"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
height_to_interval: EagerVec::forced_import(
&file,
&db,
"interval",
version + VERSION + Version::ZERO,
format,
)?,
timeindexes_to_timestamp: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"timestamp",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -65,7 +65,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(),
)?,
indexes_to_block_interval: ComputedVecsFromHeight::forced_import(
&file,
&db,
"block_interval",
Source::None,
version + VERSION + Version::ZERO,
@@ -78,7 +78,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_block_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"block_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -88,7 +88,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_block_weight: ComputedVecsFromHeight::forced_import(
&file,
&db,
"block_weight",
Source::None,
version + VERSION + Version::ZERO,
@@ -98,7 +98,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_block_size: ComputedVecsFromHeight::forced_import(
&file,
&db,
"block_size",
Source::None,
version + VERSION + Version::ZERO,
@@ -108,13 +108,13 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
height_to_vbytes: EagerVec::forced_import(
&file,
&db,
"vbytes",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_block_vbytes: ComputedVecsFromHeight::forced_import(
&file,
&db,
"block_vbytes",
Source::None,
version + VERSION + Version::ZERO,
@@ -124,19 +124,19 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
difficultyepoch_to_timestamp: EagerVec::forced_import(
&file,
&db,
"timestamp",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_timestamp: EagerVec::forced_import(
&file,
&db,
"timestamp",
version + VERSION + Version::ZERO,
format,
)?,
file,
db,
})
}
@@ -148,7 +148,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

60
crates/brk_computer/src/cointime.rs
View File

@@ -1,9 +1,9 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, CheckedSub, Dollars, StoredF64, Version};
use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, PAGE_SIZE, VecIterator};
use vecdb::{AnyCollectableVec, Computation, Database, Exit, Format, PAGE_SIZE, VecIterator};
use super::{
Indexes,
@@ -18,7 +18,7 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub indexes_to_coinblocks_created: ComputedVecsFromHeight<StoredF64>,
pub indexes_to_coinblocks_stored: ComputedVecsFromHeight<StoredF64>,
@@ -55,14 +55,14 @@ impl Vecs {
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("cointime"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
let db = Database::open(&parent.join("cointime"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
let compute_dollars = price.is_some();
Ok(Self {
indexes_to_coinblocks_created: ComputedVecsFromHeight::forced_import(
&file,
&db,
"coinblocks_created",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -72,7 +72,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_coinblocks_stored: ComputedVecsFromHeight::forced_import(
&file,
&db,
"coinblocks_stored",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -82,7 +82,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_liveliness: ComputedVecsFromHeight::forced_import(
&file,
&db,
"liveliness",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -92,7 +92,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaultedness: ComputedVecsFromHeight::forced_import(
&file,
&db,
"vaultedness",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -102,7 +102,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_activity_to_vaultedness_ratio: ComputedVecsFromHeight::forced_import(
&file,
&db,
"activity_to_vaultedness_ratio",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -112,7 +112,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_supply: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"vaulted_supply",
Source::Compute,
version + VERSION + Version::ONE,
@@ -123,7 +123,7 @@ impl Vecs {
indexes,
)?,
indexes_to_active_supply: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"active_supply",
Source::Compute,
version + VERSION + Version::ONE,
@@ -134,7 +134,7 @@ impl Vecs {
indexes,
)?,
indexes_to_thermo_cap: ComputedVecsFromHeight::forced_import(
&file,
&db,
"thermo_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -144,7 +144,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_investor_cap: ComputedVecsFromHeight::forced_import(
&file,
&db,
"investor_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -154,7 +154,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_cap: ComputedVecsFromHeight::forced_import(
&file,
&db,
"vaulted_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -164,7 +164,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_active_cap: ComputedVecsFromHeight::forced_import(
&file,
&db,
"active_cap",
Source::Compute,
version + VERSION + Version::ONE,
@@ -174,7 +174,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_price: ComputedVecsFromHeight::forced_import(
&file,
&db,
"vaulted_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -184,7 +184,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_vaulted_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
&file,
&db,
"vaulted_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -194,7 +194,7 @@ impl Vecs {
true,
)?,
indexes_to_active_price: ComputedVecsFromHeight::forced_import(
&file,
&db,
"active_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -204,7 +204,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_active_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
&file,
&db,
"active_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -214,7 +214,7 @@ impl Vecs {
true,
)?,
indexes_to_true_market_mean: ComputedVecsFromHeight::forced_import(
&file,
&db,
"true_market_mean",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -224,7 +224,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_true_market_mean_ratio: ComputedRatioVecsFromDateIndex::forced_import(
&file,
&db,
"true_market_mean",
Source::None,
version + VERSION + Version::ZERO,
@@ -234,7 +234,7 @@ impl Vecs {
true,
)?,
indexes_to_cointime_value_destroyed: ComputedVecsFromHeight::forced_import(
&file,
&db,
"cointime_value_destroyed",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -244,7 +244,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_value_created: ComputedVecsFromHeight::forced_import(
&file,
&db,
"cointime_value_created",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -254,7 +254,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_value_stored: ComputedVecsFromHeight::forced_import(
&file,
&db,
"cointime_value_stored",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -264,7 +264,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_cointime_price: ComputedVecsFromHeight::forced_import(
&file,
&db,
"cointime_price",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -274,7 +274,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_cointime_cap: ComputedVecsFromHeight::forced_import(
&file,
&db,
"cointime_cap",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -284,7 +284,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_cointime_price_ratio: ComputedRatioVecsFromDateIndex::forced_import(
&file,
&db,
"cointime_price",
Source::None,
version + VERSION + Version::ZERO,
@@ -294,7 +294,7 @@ impl Vecs {
true,
)?,
file,
db,
})
}
@@ -318,7 +318,7 @@ impl Vecs {
stateful,
exit,
)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

20
crates/brk_computer/src/constants.rs
View File

@@ -1,9 +1,9 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{StoredU16, Version};
use brk_vecs::{AnyCollectableVec, AnyVec, Computation, Exit, File, Format};
use vecdb::{AnyCollectableVec, AnyVec, Computation, Database, Exit, Format};
use crate::grouped::Source;
@@ -17,7 +17,7 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub constant_0: ComputedVecsFromHeight<StoredU16>,
pub constant_1: ComputedVecsFromHeight<StoredU16>,
@@ -33,11 +33,11 @@ impl Vecs {
format: Format,
indexes: &indexes::Vecs,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("constants"))?);
let db = Database::open(&parent.join("constants"))?;
Ok(Self {
constant_0: ComputedVecsFromHeight::forced_import(
&file,
&db,
"constant_0",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -47,7 +47,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_1: ComputedVecsFromHeight::forced_import(
&file,
&db,
"constant_1",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -57,7 +57,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_50: ComputedVecsFromHeight::forced_import(
&file,
&db,
"constant_50",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -67,7 +67,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
constant_100: ComputedVecsFromHeight::forced_import(
&file,
&db,
"constant_100",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -77,7 +77,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
file,
db,
})
}
@@ -89,7 +89,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

20
crates/brk_computer/src/fetched.rs
View File

@@ -1,19 +1,19 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_structs::{DateIndex, Height, OHLCCents, Version};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Exit, File, GenericStoredVec, RawVec,
StoredIndex, VecIterator,
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, Exit, GenericStoredVec,
RawVec, StoredIndex, VecIterator,
};
use super::{Indexes, indexes};
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
fetcher: Fetcher,
pub dateindex_to_ohlc_in_cents: RawVec<DateIndex, OHLCCents>,
@@ -22,23 +22,23 @@ pub struct Vecs {
impl Vecs {
pub fn forced_import(parent: &Path, fetcher: Fetcher, version: Version) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("fetched"))?);
let db = Database::open(&parent.join("fetched"))?;
Ok(Self {
fetcher,
dateindex_to_ohlc_in_cents: RawVec::forced_import(
&file,
&db,
"ohlc_in_cents",
version + Version::ZERO,
)?,
height_to_ohlc_in_cents: RawVec::forced_import(
&file,
&db,
"ohlc_in_cents",
version + Version::ZERO,
)?,
file,
db,
})
}
@@ -50,7 +50,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

24
crates/brk_computer/src/grouped/builder_computed.rs
View File

@@ -1,11 +1,9 @@
use std::sync::Arc;
use brk_error::Result;
use brk_structs::Version;
use brk_vecs::{
use vecdb::{
AnyBoxedIterableVec, AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation,
ComputedVec, ComputedVecFrom2, Exit, File, Format, FromCoarserIndex, StoredIndex,
ComputedVec, ComputedVecFrom2, Database, Exit, Format, FromCoarserIndex, StoredIndex,
};
use crate::grouped::{EagerVecBuilder, VecBuilderOptions};
@@ -40,7 +38,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
version: Version,
format: Format,
@@ -67,7 +65,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
&maybe_suffix("first"),
version + VERSION + Version::ZERO,
format,
@@ -92,7 +90,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -101,7 +99,7 @@ where
source
.as_ref()
.unwrap_or_else(|| {
dbg!(file, name, I::to_string());
dbg!(db, name, I::to_string());
panic!()
})
.clone()
@@ -125,7 +123,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
&maybe_suffix("min"),
version + VERSION + Version::ZERO,
format,
@@ -150,7 +148,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
&maybe_suffix("max"),
version + VERSION + Version::ZERO,
format,
@@ -175,7 +173,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
&maybe_suffix("average"),
version + VERSION + Version::ZERO,
format,
@@ -207,7 +205,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
&(if !options.last && !options.average && !options.min && !options.max {
name.to_string()
} else {
@@ -242,7 +240,7 @@ where
Box::new(
ComputedVec::forced_import_or_init_from_2(
computation,
file,
db,
&suffix("cumulative"),
version + VERSION + Version::ZERO,
format,

102
crates/brk_computer/src/grouped/builder_eager.rs
View File

@@ -1,9 +1,7 @@
use std::sync::Arc;
use brk_error::{Error, Result};
use brk_structs::{CheckedSub, StoredU64, Version};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, EagerVec, Exit, File, Format,
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Database, EagerVec, Exit, Format,
GenericStoredVec, StoredIndex, StoredRaw,
};
@@ -39,7 +37,7 @@ where
T: ComputedType,
{
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
version: Version,
format: Format,
@@ -61,7 +59,7 @@ where
first: options.first.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("first"),
version + VERSION + Version::ZERO,
format,
@@ -71,13 +69,13 @@ where
}),
last: options.last.then(|| {
Box::new(
EagerVec::forced_import(file, name, version + Version::ZERO, format).unwrap(),
EagerVec::forced_import(db, name, version + Version::ZERO, format).unwrap(),
)
}),
min: options.min.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("min"),
version + VERSION + Version::ZERO,
format,
@@ -88,7 +86,7 @@ where
max: options.max.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("max"),
version + VERSION + Version::ZERO,
format,
@@ -99,7 +97,7 @@ where
median: options.median.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("median"),
version + VERSION + Version::ZERO,
format,
@@ -110,7 +108,7 @@ where
average: options.average.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("average"),
version + VERSION + Version::ZERO,
format,
@@ -121,7 +119,7 @@ where
sum: options.sum.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&(if !options.last && !options.average && !options.min && !options.max {
name.to_string()
} else {
@@ -136,7 +134,7 @@ where
cumulative: options.cumulative.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&suffix("cumulative"),
version + VERSION + Version::ZERO,
format,
@@ -147,7 +145,7 @@ where
_90p: options._90p.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("90p"),
version + VERSION + Version::ZERO,
format,
@@ -158,7 +156,7 @@ where
_75p: options._75p.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("75p"),
version + VERSION + Version::ZERO,
format,
@@ -169,7 +167,7 @@ where
_25p: options._25p.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("25p"),
version + VERSION + Version::ZERO,
format,
@@ -180,7 +178,7 @@ where
_10p: options._10p.then(|| {
Box::new(
EagerVec::forced_import(
file,
db,
&maybe_suffix("10p"),
version + VERSION + Version::ZERO,
format,
@@ -203,7 +201,7 @@ where
return Ok(());
};
self.validate_computed_version_or_reset_file(source.version())?;
self.validate_computed_version_or_reset(source.version())?;
let index = self.starting_index(max_from);
@@ -234,7 +232,7 @@ where
where
I2: StoredIndex + StoredRaw + CheckedSub<I2>,
{
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
source.version() + first_indexes.version() + count_indexes.version(),
)?;
@@ -253,10 +251,10 @@ where
first_indexes
.iter_at(index)
.try_for_each(|(i, first_index)| -> Result<()> {
.try_for_each(|(index, first_index)| -> Result<()> {
let first_index = first_index.into_owned();
let count_index = count_indexes_iter.unwrap_get_inner(i);
let count_index = count_indexes_iter.unwrap_get_inner(index);
if let Some(first) = self.first.as_mut() {
let f = source_iter
@@ -305,7 +303,7 @@ where
if let Some(max) = self.max.as_mut() {
max.forced_push_at(
i,
index,
*values
.last()
.ok_or(Error::Str("expect some"))
@@ -327,27 +325,27 @@ where
}
if let Some(_90p) = self._90p.as_mut() {
_90p.forced_push_at(i, get_percentile(&values, 0.90), exit)?;
_90p.forced_push_at(index, get_percentile(&values, 0.90), exit)?;
}
if let Some(_75p) = self._75p.as_mut() {
_75p.forced_push_at(i, get_percentile(&values, 0.75), exit)?;
_75p.forced_push_at(index, get_percentile(&values, 0.75), exit)?;
}
if let Some(median) = self.median.as_mut() {
median.forced_push_at(i, get_percentile(&values, 0.50), exit)?;
median.forced_push_at(index, get_percentile(&values, 0.50), exit)?;
}
if let Some(_25p) = self._25p.as_mut() {
_25p.forced_push_at(i, get_percentile(&values, 0.25), exit)?;
_25p.forced_push_at(index, get_percentile(&values, 0.25), exit)?;
}
if let Some(_10p) = self._10p.as_mut() {
_10p.forced_push_at(i, get_percentile(&values, 0.10), exit)?;
_10p.forced_push_at(index, get_percentile(&values, 0.10), exit)?;
}
if let Some(min) = self.min.as_mut() {
min.forced_push_at(i, *values.first().unwrap(), exit)?;
min.forced_push_at(index, *values.first().unwrap(), exit)?;
}
}
@@ -357,18 +355,18 @@ where
if let Some(average) = self.average.as_mut() {
let avg = sum / len;
average.forced_push_at(i, avg, exit)?;
average.forced_push_at(index, avg, exit)?;
}
if needs_sum_or_cumulative {
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum, exit)?;
sum_vec.forced_push_at(index, sum, exit)?;
}
if let Some(cumulative_vec) = self.cumulative.as_mut() {
let t = cumulative.unwrap() + sum;
cumulative.replace(t);
cumulative_vec.forced_push_at(i, t, exit)?;
cumulative_vec.forced_push_at(index, t, exit)?;
}
}
}
@@ -403,7 +401,7 @@ where
panic!("unsupported");
}
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
VERSION + first_indexes.version() + count_indexes.version(),
)?;
@@ -426,10 +424,10 @@ where
first_indexes
.iter_at(index)
.try_for_each(|(i, first_index, ..)| -> Result<()> {
.try_for_each(|(index, first_index, ..)| -> Result<()> {
let first_index = first_index.into_owned();
let count_index = count_indexes_iter.unwrap_get_inner(i);
let count_index = count_indexes_iter.unwrap_get_inner(index);
if let Some(first) = self.first.as_mut() {
let v = source_first_iter
@@ -468,7 +466,7 @@ where
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
values.sort_unstable();
max.forced_push_at(i, *values.last().unwrap(), exit)?;
max.forced_push_at(index, *values.last().unwrap(), exit)?;
}
if let Some(min) = self.min.as_mut() {
@@ -479,7 +477,7 @@ where
.map(|(_, v)| v.into_owned())
.collect::<Vec<_>>();
values.sort_unstable();
min.forced_push_at(i, *values.first().unwrap(), exit)?;
min.forced_push_at(index, *values.first().unwrap(), exit)?;
}
}
@@ -497,7 +495,7 @@ where
// TODO: Multiply by count then divide by cumulative
// Right now it's not 100% accurate as there could be more or less elements in the lower timeframe (28 days vs 31 days in a month for example)
let avg = cumulative / len;
average.forced_push_at(i, avg, exit)?;
average.forced_push_at(index, avg, exit)?;
}
if needs_sum_or_cumulative {
@@ -511,13 +509,13 @@ where
let sum = values.into_iter().fold(T::from(0), |a, b| a + b);
if let Some(sum_vec) = self.sum.as_mut() {
sum_vec.forced_push_at(i, sum, exit)?;
sum_vec.forced_push_at(index, sum, exit)?;
}
if let Some(cumulative_vec) = self.cumulative.as_mut() {
let t = cumulative.unwrap() + sum;
cumulative.replace(t);
cumulative_vec.forced_push_at(i, t, exit)?;
cumulative_vec.forced_push_at(index, t, exit)?;
}
}
}
@@ -665,42 +663,42 @@ where
Ok(())
}
pub fn validate_computed_version_or_reset_file(&mut self, version: Version) -> Result<()> {
pub fn validate_computed_version_or_reset(&mut self, version: Version) -> Result<()> {
if let Some(first) = self.first.as_mut() {
first.validate_computed_version_or_reset_file(Version::ZERO + version)?;
first.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(last) = self.last.as_mut() {
last.validate_computed_version_or_reset_file(Version::ZERO + version)?;
last.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(min) = self.min.as_mut() {
min.validate_computed_version_or_reset_file(Version::ZERO + version)?;
min.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(max) = self.max.as_mut() {
max.validate_computed_version_or_reset_file(Version::ZERO + version)?;
max.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(median) = self.median.as_mut() {
median.validate_computed_version_or_reset_file(Version::ZERO + version)?;
median.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(average) = self.average.as_mut() {
average.validate_computed_version_or_reset_file(Version::ZERO + version)?;
average.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(sum) = self.sum.as_mut() {
sum.validate_computed_version_or_reset_file(Version::ZERO + version)?;
sum.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(cumulative) = self.cumulative.as_mut() {
cumulative.validate_computed_version_or_reset_file(Version::ZERO + version)?;
cumulative.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(_90p) = self._90p.as_mut() {
_90p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
_90p.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(_75p) = self._75p.as_mut() {
_75p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
_75p.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(_25p) = self._25p.as_mut() {
_25p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
_25p.validate_computed_version_or_reset(Version::ZERO + version)?;
}
if let Some(_10p) = self._10p.as_mut() {
_10p.validate_computed_version_or_reset_file(Version::ZERO + version)?;
_10p.validate_computed_version_or_reset(Version::ZERO + version)?;
}
Ok(())

2
crates/brk_computer/src/grouped/computed.rs
View File

@@ -1,6 +1,6 @@
use std::ops::{Add, AddAssign, Div};
use brk_vecs::StoredCompressed;
use vecdb::StoredCompressed;
pub trait ComputedType
where

26
crates/brk_computer/src/grouped/from_dateindex.rs
View File

@@ -1,14 +1,12 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
DateIndex, DecadeIndex, MonthIndex, QuarterIndex, SemesterIndex, Version, WeekIndex, YearIndex,
};
use brk_vecs::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Exit, File,
Format,
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, Database, EagerVec,
Exit, Format,
};
use crate::{Indexes, grouped::ComputedVecBuilder, indexes};
@@ -38,7 +36,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<DateIndex, T>,
version: Version,
@@ -48,11 +46,11 @@ where
options: VecBuilderOptions,
) -> Result<Self> {
let dateindex = source.is_compute().then(|| {
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap()
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format).unwrap()
});
let dateindex_extra = EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -65,7 +63,7 @@ where
Ok(Self {
weekindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -76,7 +74,7 @@ where
options.into(),
)?,
monthindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -87,7 +85,7 @@ where
options.into(),
)?,
quarterindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -98,7 +96,7 @@ where
options.into(),
)?,
semesterindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -109,7 +107,7 @@ where
options.into(),
)?,
yearindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -120,7 +118,7 @@ where
options.into(),
)?,
decadeindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,

32
crates/brk_computer/src/grouped/from_height.rs
View File

@@ -1,5 +1,3 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
@@ -7,9 +5,9 @@ use brk_structs::{
DateIndex, DecadeIndex, DifficultyEpoch, Height, MonthIndex, QuarterIndex, SemesterIndex,
Version, WeekIndex, YearIndex,
};
use brk_vecs::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, EagerVec, Exit, File,
Format,
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, Database, EagerVec,
Exit, Format,
};
use crate::{
@@ -47,7 +45,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<Height, T>,
version: Version,
@@ -57,11 +55,11 @@ where
options: VecBuilderOptions,
) -> Result<Self> {
let height = source.is_compute().then(|| {
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format).unwrap()
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format).unwrap()
});
let height_extra = EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -69,7 +67,7 @@ where
)?;
let dateindex = EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -80,7 +78,7 @@ where
Ok(Self {
weekindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -91,7 +89,7 @@ where
options.into(),
)?,
monthindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -102,7 +100,7 @@ where
options.into(),
)?,
quarterindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -113,7 +111,7 @@ where
options.into(),
)?,
semesterindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -124,7 +122,7 @@ where
options.into(),
)?,
yearindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -135,7 +133,7 @@ where
options.into(),
)?,
decadeindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -145,12 +143,12 @@ where
indexes.decadeindex_to_decadeindex.boxed_clone(),
options.into(),
)?,
// halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
height,
height_extra,
dateindex,
difficultyepoch: EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,

15
crates/brk_computer/src/grouped/from_height_strict.rs
View File

@@ -1,10 +1,8 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{DifficultyEpoch, Height, Version};
use brk_vecs::{AnyCollectableVec, EagerVec, Exit, File, Format};
use vecdb::{AnyCollectableVec, Database, EagerVec, Exit, Format};
use crate::{Indexes, indexes};
@@ -29,17 +27,16 @@ where
f64: From<T>,
{
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
version: Version,
format: Format,
options: VecBuilderOptions,
) -> Result<Self> {
let height =
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)?;
let height = EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)?;
let height_extra = EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -52,13 +49,13 @@ where
height,
height_extra,
difficultyepoch: EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
})
}

36
crates/brk_computer/src/grouped/from_txindex.rs
View File

@@ -1,14 +1,12 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{
Bitcoin, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, MonthIndex, QuarterIndex,
Sats, SemesterIndex, TxIndex, Version, WeekIndex, YearIndex,
};
use brk_vecs::{
AnyCloneableIterableVec, AnyCollectableVec, AnyVec, CollectableVec, Computation, EagerVec,
Exit, File, Format, GenericStoredVec, StoredIndex, VecIterator,
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyVec, CollectableVec, Computation, Database,
EagerVec, Exit, Format, GenericStoredVec, StoredIndex, VecIterator,
};
use crate::{
@@ -46,7 +44,7 @@ where
{
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<TxIndex, T>,
version: Version,
@@ -57,13 +55,13 @@ where
) -> Result<Self> {
let txindex = source.is_compute().then(|| {
Box::new(
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)
.unwrap(),
)
});
let height = EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -73,7 +71,7 @@ where
let options = options.remove_percentiles();
let dateindex = EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -82,7 +80,7 @@ where
Ok(Self {
weekindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -93,7 +91,7 @@ where
options.into(),
)?,
monthindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -104,7 +102,7 @@ where
options.into(),
)?,
quarterindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -115,7 +113,7 @@ where
options.into(),
)?,
semesterindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -126,7 +124,7 @@ where
options.into(),
)?,
yearindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -137,7 +135,7 @@ where
options.into(),
)?,
decadeindex: ComputedVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
@@ -152,13 +150,13 @@ where
height,
dateindex,
difficultyepoch: EagerVecBuilder::forced_import(
file,
db,
name,
version + VERSION + Version::ZERO,
format,
options,
)?,
// halvingepoch: StorableVecGeneator::forced_import(file, name, version + VERSION + Version::ZERO, format, options)?,
// halvingepoch: StorableVecGeneator::forced_import(db, name, version + VERSION + Version::ZERO, format, options)?,
})
}
@@ -325,7 +323,7 @@ impl ComputedVecsFromTxindex<Bitcoin> {
};
self.height
.validate_computed_version_or_reset_file(txindex_version)?;
.validate_computed_version_or_reset(txindex_version)?;
let starting_index = self.height.starting_index(starting_indexes.height);
@@ -504,7 +502,7 @@ impl ComputedVecsFromTxindex<Dollars> {
};
self.height
.validate_computed_version_or_reset_file(txindex_version)?;
.validate_computed_version_or_reset(txindex_version)?;
let starting_index = self.height.starting_index(starting_indexes.height);

88
crates/brk_computer/src/grouped/ratio_from_dateindex.rs
View File

@@ -1,11 +1,9 @@
use std::{f32, sync::Arc};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Date, DateIndex, Dollars, StoredF32, Version};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, EagerVec,
Exit, File, Format, GenericStoredVec, StoredIndex, VecIterator,
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, Database,
EagerVec, Exit, Format, GenericStoredVec, StoredIndex, VecIterator,
};
use crate::{Indexes, grouped::source::Source, indexes, price, utils::get_percentile};
@@ -60,7 +58,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedRatioVecsFromDateIndex {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<DateIndex, Dollars>,
version: Version,
@@ -74,7 +72,7 @@ impl ComputedRatioVecsFromDateIndex {
Ok(Self {
price: source.is_compute().then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
name,
Source::Compute,
version + VERSION,
@@ -86,7 +84,7 @@ impl ComputedRatioVecsFromDateIndex {
.unwrap()
}),
ratio: ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -97,7 +95,7 @@ impl ComputedRatioVecsFromDateIndex {
)?,
ratio_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -110,7 +108,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1w_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_1w_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -123,7 +121,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1m_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_1m_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -136,7 +134,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_1y_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -149,7 +147,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_4y_sma: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_4y_sma"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -162,7 +160,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_sma_momentum_oscillator: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_1y_sma_momentum_oscillator"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -175,7 +173,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -188,7 +186,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_4y_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_4y_sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -201,7 +199,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_1y_sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -214,7 +212,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_9: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p99_9"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -227,7 +225,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_5: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p99_5"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -240,7 +238,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p99"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -253,7 +251,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p1"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -266,7 +264,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_5: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p0_5"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -279,7 +277,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_1: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p0_1"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -292,7 +290,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p1sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -305,7 +303,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p2sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p2sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -318,7 +316,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p3sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p3sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -331,7 +329,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m1sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_m1sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -344,7 +342,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m2sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_m2sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -357,7 +355,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m3sd: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_m3sd"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -370,7 +368,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_9_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p99_9_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -383,7 +381,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_5_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p99_5_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -396,7 +394,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p99_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p99_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -409,7 +407,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p1_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -422,7 +420,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_5_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p0_5_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -435,7 +433,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p0_1_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p0_1_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -448,7 +446,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p1sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p1sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -461,7 +459,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p2sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p2sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -474,7 +472,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_p3sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_p3sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -487,7 +485,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m1sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_m1sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -500,7 +498,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m2sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_m2sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -513,7 +511,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_m3sd_as_price: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_m3sd_as_price"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -526,7 +524,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_zscore"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -539,7 +537,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_4y_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_4y_zscore"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -552,7 +550,7 @@ impl ComputedRatioVecsFromDateIndex {
}),
ratio_1y_zscore: extended.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_ratio_1y_zscore"),
Source::Compute,
version + VERSION + Version::ZERO,
@@ -771,7 +769,7 @@ impl ComputedRatioVecsFromDateIndex {
self.mut_ratio_vecs()
.iter_mut()
.try_for_each(|v| -> Result<()> {
v.validate_computed_version_or_reset_file(
v.validate_computed_version_or_reset(
Version::ZERO + v.inner_version() + ratio_version,
)?;
Ok(())

2
crates/brk_computer/src/grouped/source.rs
View File

@@ -1,4 +1,4 @@
use brk_vecs::AnyBoxedIterableVec;
use vecdb::AnyBoxedIterableVec;
#[derive(Clone)]
pub enum Source<I, T> {

14
crates/brk_computer/src/grouped/value_from_dateindex.rs
View File

@@ -1,10 +1,8 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, DateIndex, Dollars, Sats, Version};
use brk_vecs::{
AnyCollectableVec, CollectableVec, Computation, EagerVec, Exit, File, Format, StoredVec,
use vecdb::{
AnyCollectableVec, CollectableVec, Computation, Database, EagerVec, Exit, Format, StoredVec,
};
use crate::{
@@ -28,7 +26,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromDateIndex {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<DateIndex, Sats>,
version: Version,
@@ -40,7 +38,7 @@ impl ComputedValueVecsFromDateIndex {
) -> Result<Self> {
Ok(Self {
sats: ComputedVecsFromDateIndex::forced_import(
file,
db,
name,
source,
version + VERSION,
@@ -50,7 +48,7 @@ impl ComputedValueVecsFromDateIndex {
options,
)?,
bitcoin: ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_in_btc"),
Source::Compute,
version + VERSION,
@@ -61,7 +59,7 @@ impl ComputedValueVecsFromDateIndex {
)?,
dollars: compute_dollars.then(|| {
ComputedVecsFromDateIndex::forced_import(
file,
db,
&format!("{name}_in_usd"),
Source::Compute,
version + VERSION,

14
crates/brk_computer/src/grouped/value_from_height.rs
View File

@@ -1,10 +1,8 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, Dollars, Height, Sats, Version};
use brk_vecs::{
AnyCollectableVec, CollectableVec, Computation, EagerVec, Exit, File, Format, StoredVec,
use vecdb::{
AnyCollectableVec, CollectableVec, Computation, Database, EagerVec, Exit, Format, StoredVec,
};
use crate::{
@@ -28,7 +26,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromHeight {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<Height, Sats>,
version: Version,
@@ -40,7 +38,7 @@ impl ComputedValueVecsFromHeight {
) -> Result<Self> {
Ok(Self {
sats: ComputedVecsFromHeight::forced_import(
file,
db,
name,
source,
version + VERSION,
@@ -50,7 +48,7 @@ impl ComputedValueVecsFromHeight {
options,
)?,
bitcoin: ComputedVecsFromHeight::forced_import(
file,
db,
&format!("{name}_in_btc"),
Source::Compute,
version + VERSION,
@@ -61,7 +59,7 @@ impl ComputedValueVecsFromHeight {
)?,
dollars: compute_dollars.then(|| {
ComputedVecsFromHeight::forced_import(
file,
db,
&format!("{name}_in_usd"),
Source::Compute,
version + VERSION,

16
crates/brk_computer/src/grouped/value_from_txindex.rs
View File

@@ -1,11 +1,9 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, Close, Dollars, Height, Sats, TxIndex, Version};
use brk_vecs::{
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, CollectableVec, Computation, ComputedVecFrom3,
Exit, File, Format, LazyVecFrom1, StoredIndex, StoredVec,
Database, Exit, Format, LazyVecFrom1, StoredIndex, StoredVec,
};
use crate::{Indexes, grouped::Source, indexes, price};
@@ -38,7 +36,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedValueVecsFromTxindex {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
indexes: &indexes::Vecs,
source: Source<TxIndex, Sats>,
@@ -54,7 +52,7 @@ impl ComputedValueVecsFromTxindex {
let name_in_usd = format!("{name}_in_usd");
let sats = ComputedVecsFromTxindex::forced_import(
file,
db,
name,
source.clone(),
version + VERSION,
@@ -79,7 +77,7 @@ impl ComputedValueVecsFromTxindex {
);
let bitcoin = ComputedVecsFromTxindex::forced_import(
file,
db,
&name_in_btc,
Source::None,
version + VERSION,
@@ -92,7 +90,7 @@ impl ComputedValueVecsFromTxindex {
let dollars_txindex = price.map(|price| {
ComputedVecFrom3::forced_import_or_init_from_3(
computation,
file,
db,
&name_in_usd,
version + VERSION,
format,
@@ -127,7 +125,7 @@ impl ComputedValueVecsFromTxindex {
dollars_txindex,
dollars: compute_dollars.then(|| {
ComputedVecsFromTxindex::forced_import(
file,
db,
&name_in_usd,
Source::None,
version + VERSION,

12
crates/brk_computer/src/grouped/value_height.rs
View File

@@ -1,9 +1,7 @@
use std::sync::Arc;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, Dollars, Height, Sats, Version};
use brk_vecs::{AnyCollectableVec, CollectableVec, EagerVec, Exit, File, Format, StoredVec};
use vecdb::{AnyCollectableVec, CollectableVec, Database, EagerVec, Exit, Format, StoredVec};
use crate::{
Indexes,
@@ -23,7 +21,7 @@ const VERSION: Version = Version::ZERO;
impl ComputedHeightValueVecs {
pub fn forced_import(
file: &Arc<File>,
db: &Database,
name: &str,
source: Source<Height, Sats>,
version: Version,
@@ -32,18 +30,18 @@ impl ComputedHeightValueVecs {
) -> Result<Self> {
Ok(Self {
sats: source.is_compute().then(|| {
EagerVec::forced_import(file, name, version + VERSION + Version::ZERO, format)
EagerVec::forced_import(db, name, version + VERSION + Version::ZERO, format)
.unwrap()
}),
bitcoin: EagerVec::forced_import(
file,
db,
&format!("{name}_in_btc"),
version + VERSION + Version::ZERO,
format,
)?,
dollars: compute_dollars.then(|| {
EagerVec::forced_import(
file,
db,
&format!("{name}_in_usd"),
version + VERSION + Version::ZERO,
format,

136
crates/brk_computer/src/indexes.rs
View File

@@ -1,4 +1,4 @@
use std::{ops::Deref, path::Path, sync::Arc};
use std::{ops::Deref, path::Path};
use brk_error::Result;
use brk_indexer::Indexer;
@@ -10,16 +10,16 @@ use brk_structs::{
P2WPKHBytes, P2WSHAddressIndex, P2WSHBytes, QuarterIndex, Sats, SemesterIndex, StoredU64,
Timestamp, TxIndex, Txid, UnknownOutputIndex, Version, WeekIndex, YearIndex,
};
use brk_vecs::{
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, Computation, ComputedVec, ComputedVecFrom1,
ComputedVecFrom2, EagerVec, Exit, File, Format, PAGE_SIZE, StoredIndex, VecIterator,
ComputedVecFrom2, Database, EagerVec, Exit, Format, PAGE_SIZE, StoredIndex, VecIterator,
};
const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub dateindex_to_date: EagerVec<DateIndex, Date>,
pub dateindex_to_dateindex: EagerVec<DateIndex, DateIndex>,
@@ -105,12 +105,12 @@ impl Vecs {
computation: Computation,
format: Format,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("indexes"))?);
file.set_min_len(PAGE_SIZE * 10_000_000)?;
let db = Database::open(&parent.join("indexes"))?;
db.set_min_len(PAGE_SIZE * 10_000_000)?;
let outputindex_to_outputindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"outputindex",
version + VERSION + Version::ZERO,
format,
@@ -120,7 +120,7 @@ impl Vecs {
let inputindex_to_inputindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"inputindex",
version + VERSION + Version::ZERO,
format,
@@ -130,7 +130,7 @@ impl Vecs {
let txindex_to_txindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"txindex",
version + VERSION + Version::ZERO,
format,
@@ -140,7 +140,7 @@ impl Vecs {
let txindex_to_input_count = ComputedVec::forced_import_or_init_from_2(
computation,
&file,
&db,
"input_count",
version + VERSION + Version::ZERO,
format,
@@ -163,7 +163,7 @@ impl Vecs {
let txindex_to_output_count = ComputedVec::forced_import_or_init_from_2(
computation,
&file,
&db,
"output_count",
version + VERSION + Version::ZERO,
format,
@@ -186,7 +186,7 @@ impl Vecs {
let p2pk33addressindex_to_p2pk33addressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2pk33addressindex",
version + VERSION + Version::ZERO,
format,
@@ -195,7 +195,7 @@ impl Vecs {
)?;
let p2pk65addressindex_to_p2pk65addressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2pk65addressindex",
version + VERSION + Version::ZERO,
format,
@@ -204,7 +204,7 @@ impl Vecs {
)?;
let p2pkhaddressindex_to_p2pkhaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2pkhaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -213,7 +213,7 @@ impl Vecs {
)?;
let p2shaddressindex_to_p2shaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2shaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -222,7 +222,7 @@ impl Vecs {
)?;
let p2traddressindex_to_p2traddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2traddressindex",
version + VERSION + Version::ZERO,
format,
@@ -231,7 +231,7 @@ impl Vecs {
)?;
let p2wpkhaddressindex_to_p2wpkhaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2wpkhaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -240,7 +240,7 @@ impl Vecs {
)?;
let p2wshaddressindex_to_p2wshaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2wshaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -249,7 +249,7 @@ impl Vecs {
)?;
let p2aaddressindex_to_p2aaddressindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2aaddressindex",
version + VERSION + Version::ZERO,
format,
@@ -258,7 +258,7 @@ impl Vecs {
)?;
let p2msoutputindex_to_p2msoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"p2msoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -267,7 +267,7 @@ impl Vecs {
)?;
let emptyoutputindex_to_emptyoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"emptyoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -276,7 +276,7 @@ impl Vecs {
)?;
let unknownoutputindex_to_unknownoutputindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"unknownoutputindex",
version + VERSION + Version::ZERO,
format,
@@ -285,7 +285,7 @@ impl Vecs {
)?;
let opreturnindex_to_opreturnindex = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"opreturnindex",
version + VERSION + Version::ZERO,
format,
@@ -313,265 +313,265 @@ impl Vecs {
unknownoutputindex_to_unknownoutputindex,
dateindex_to_date: EagerVec::forced_import(
&file,
&db,
"date",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_dateindex: EagerVec::forced_import(
&file,
&db,
"dateindex",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_first_height: EagerVec::forced_import(
&file,
&db,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_monthindex: EagerVec::forced_import(
&file,
&db,
"monthindex",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_weekindex: EagerVec::forced_import(
&file,
&db,
"weekindex",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_decadeindex: EagerVec::forced_import(
&file,
&db,
"decadeindex",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_first_yearindex: EagerVec::forced_import(
&file,
&db,
"first_yearindex",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_difficultyepoch: EagerVec::forced_import(
&file,
&db,
"difficultyepoch",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_first_height: EagerVec::forced_import(
&file,
&db,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_first_height: EagerVec::forced_import(
&file,
&db,
"first_height",
version + VERSION + Version::ZERO,
format,
)?,
halvingepoch_to_halvingepoch: EagerVec::forced_import(
&file,
&db,
"halvingepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_date: EagerVec::forced_import(
&file,
&db,
"date",
version + VERSION + Version::ZERO,
format,
)?,
height_to_difficultyepoch: EagerVec::forced_import(
&file,
&db,
"difficultyepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_halvingepoch: EagerVec::forced_import(
&file,
&db,
"halvingepoch",
version + VERSION + Version::ZERO,
format,
)?,
height_to_height: EagerVec::forced_import(
&file,
&db,
"height",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_first_dateindex: EagerVec::forced_import(
&file,
&db,
"first_dateindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_monthindex: EagerVec::forced_import(
&file,
&db,
"monthindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_quarterindex: EagerVec::forced_import(
&file,
&db,
"quarterindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_semesterindex: EagerVec::forced_import(
&file,
&db,
"semesterindex",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_yearindex: EagerVec::forced_import(
&file,
&db,
"yearindex",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_first_monthindex: EagerVec::forced_import(
&file,
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_first_monthindex: EagerVec::forced_import(
&file,
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_first_dateindex: EagerVec::forced_import(
&file,
&db,
"first_dateindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_first_monthindex: EagerVec::forced_import(
&file,
&db,
"first_monthindex",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_quarterindex: EagerVec::forced_import(
&file,
&db,
"quarterindex",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_semesterindex: EagerVec::forced_import(
&file,
&db,
"semesterindex",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_weekindex: EagerVec::forced_import(
&file,
&db,
"weekindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_decadeindex: EagerVec::forced_import(
&file,
&db,
"decadeindex",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_yearindex: EagerVec::forced_import(
&file,
&db,
"yearindex",
version + VERSION + Version::ZERO,
format,
)?,
height_to_date_fixed: EagerVec::forced_import(
&file,
&db,
"date_fixed",
version + VERSION + Version::ZERO,
format,
)?,
height_to_dateindex: EagerVec::forced_import(
&file,
&db,
"dateindex",
version + VERSION + Version::ZERO,
format,
)?,
txindex_to_height: EagerVec::forced_import(
&file,
&db,
"height",
version + VERSION + Version::ZERO,
format,
)?,
height_to_timestamp_fixed: EagerVec::forced_import(
&file,
&db,
"timestamp_fixed",
version + VERSION + Version::ZERO,
format,
)?,
height_to_txindex_count: EagerVec::forced_import(
&file,
&db,
"txindex_count",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_height_count: EagerVec::forced_import(
&file,
&db,
"height_count",
version + VERSION + Version::ZERO,
format,
)?,
weekindex_to_dateindex_count: EagerVec::forced_import(
&file,
&db,
"dateindex_count",
version + VERSION + Version::ZERO,
format,
)?,
difficultyepoch_to_height_count: EagerVec::forced_import(
&file,
&db,
"height_count",
version + VERSION + Version::ZERO,
format,
)?,
monthindex_to_dateindex_count: EagerVec::forced_import(
&file,
&db,
"dateindex_count",
version + VERSION + Version::ZERO,
format,
)?,
quarterindex_to_monthindex_count: EagerVec::forced_import(
&file,
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
semesterindex_to_monthindex_count: EagerVec::forced_import(
&file,
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
yearindex_to_monthindex_count: EagerVec::forced_import(
&file,
&db,
"monthindex_count",
version + VERSION + Version::ZERO,
format,
)?,
decadeindex_to_yearindex_count: EagerVec::forced_import(
&file,
&db,
"yearindex_count",
version + VERSION + Version::ZERO,
format,
)?,
outputindex_to_txindex: EagerVec::forced_import(
&file,
&db,
"txindex",
version + VERSION + Version::ZERO,
format,
)?,
file,
db,
})
}
@@ -582,7 +582,7 @@ impl Vecs {
exit: &Exit,
) -> Result<Indexes> {
let idxs = self.compute_(indexer, starting_indexes, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(idxs)
}

2
crates/brk_computer/src/lib.rs
View File

@@ -9,8 +9,8 @@ use brk_error::Result;
use brk_fetcher::Fetcher;
use brk_indexer::Indexer;
use brk_structs::Version;
use brk_vecs::{AnyCollectableVec, Computation, Exit, Format};
use log::info;
use vecdb::{AnyCollectableVec, Computation, Exit, Format};
mod blocks;
mod cointime;

284
crates/brk_computer/src/market.rs
View File

File diff suppressed because it is too large Load Diff

20
crates/brk_computer/src/mining.rs
View File

@@ -1,9 +1,9 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{DifficultyEpoch, HalvingEpoch, StoredF64, Version};
use brk_vecs::{AnyCollectableVec, Computation, Exit, File, Format, PAGE_SIZE, VecIterator};
use vecdb::{AnyCollectableVec, Computation, Database, Exit, Format, PAGE_SIZE, VecIterator};
use crate::grouped::Source;
@@ -17,7 +17,7 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub indexes_to_difficulty: ComputedVecsFromHeight<StoredF64>,
pub indexes_to_difficultyepoch: ComputedVecsFromDateIndex<DifficultyEpoch>,
@@ -32,12 +32,12 @@ impl Vecs {
format: Format,
indexes: &indexes::Vecs,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("mining"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
let db = Database::open(&parent.join("mining"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
indexes_to_difficulty: ComputedVecsFromHeight::forced_import(
&file,
&db,
"difficulty",
Source::None,
version + VERSION + Version::ZERO,
@@ -47,7 +47,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_difficultyepoch: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"difficultyepoch",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -57,7 +57,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_halvingepoch: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"halvingepoch",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -67,7 +67,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
file,
db,
})
}
@@ -79,7 +79,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

106
crates/brk_computer/src/price.rs
View File

@@ -1,4 +1,4 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
@@ -6,8 +6,8 @@ use brk_structs::{
Cents, Close, DateIndex, DecadeIndex, DifficultyEpoch, Dollars, Height, High, Low, MonthIndex,
OHLCDollars, OHLCSats, Open, QuarterIndex, Sats, SemesterIndex, Version, WeekIndex, YearIndex,
};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, EagerVec, Exit, File,
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, Database, EagerVec, Exit,
Format, GenericStoredVec, PAGE_SIZE, RawVec,
};
@@ -21,7 +21,7 @@ use super::{
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub dateindex_to_close_in_cents: EagerVec<DateIndex, Close<Cents>>,
pub dateindex_to_high_in_cents: EagerVec<DateIndex, High<Cents>>,
@@ -80,80 +80,76 @@ impl Vecs {
format: Format,
indexes: &indexes::Vecs,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("price"))?);
file.set_min_len(PAGE_SIZE * 1_000_000)?;
let db = Database::open(&parent.join("price"))?;
db.set_min_len(PAGE_SIZE * 1_000_000)?;
Ok(Self {
dateindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
dateindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
dateindex_to_close_in_cents: EagerVec::forced_import(
&file,
&db,
"close_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_high_in_cents: EagerVec::forced_import(
&file,
&db,
"high_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_low_in_cents: EagerVec::forced_import(
&file,
&db,
"low_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
dateindex_to_open_in_cents: EagerVec::forced_import(
&file,
&db,
"open_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_ohlc: RawVec::forced_import(
&file,
"ohlc",
version + VERSION + Version::ZERO,
)?,
height_to_ohlc: RawVec::forced_import(&db, "ohlc", version + VERSION + Version::ZERO)?,
height_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
height_to_close_in_cents: EagerVec::forced_import(
&file,
&db,
"close_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_high_in_cents: EagerVec::forced_import(
&file,
&db,
"high_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_low_in_cents: EagerVec::forced_import(
&file,
&db,
"low_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
height_to_open_in_cents: EagerVec::forced_import(
&file,
&db,
"open_in_cents",
version + VERSION + Version::ZERO,
format,
)?,
timeindexes_to_open: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"open",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -163,7 +159,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(),
)?,
timeindexes_to_high: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"high",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -173,7 +169,7 @@ impl Vecs {
VecBuilderOptions::default().add_max(),
)?,
timeindexes_to_low: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"low",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -183,7 +179,7 @@ impl Vecs {
VecBuilderOptions::default().add_min(),
)?,
timeindexes_to_close: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"close",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -193,7 +189,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
timeindexes_to_open_in_sats: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"open_in_sats",
Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -203,7 +199,7 @@ impl Vecs {
VecBuilderOptions::default().add_first(),
)?,
timeindexes_to_high_in_sats: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"high_in_sats",
Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -213,7 +209,7 @@ impl Vecs {
VecBuilderOptions::default().add_max(),
)?,
timeindexes_to_low_in_sats: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"low_in_sats",
Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -223,7 +219,7 @@ impl Vecs {
VecBuilderOptions::default().add_min(),
)?,
timeindexes_to_close_in_sats: ComputedVecsFromDateIndex::forced_import(
&file,
&db,
"close_in_sats",
Source::Compute,
version + VERSION + VERSION_IN_SATS + Version::ZERO,
@@ -233,135 +229,135 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
chainindexes_to_open: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"open",
version + VERSION + Version::ZERO,
format,
VecBuilderOptions::default().add_first(),
)?,
chainindexes_to_high: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"high",
version + VERSION + Version::ZERO,
format,
VecBuilderOptions::default().add_max(),
)?,
chainindexes_to_low: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"low",
version + VERSION + Version::ZERO,
format,
VecBuilderOptions::default().add_min(),
)?,
chainindexes_to_close: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"close",
version + VERSION + Version::ZERO,
format,
VecBuilderOptions::default().add_last(),
)?,
chainindexes_to_open_in_sats: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"open_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
VecBuilderOptions::default().add_first(),
)?,
chainindexes_to_high_in_sats: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"high_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
VecBuilderOptions::default().add_max(),
)?,
chainindexes_to_low_in_sats: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"low_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
VecBuilderOptions::default().add_min(),
)?,
chainindexes_to_close_in_sats: ComputedVecsFromHeightStrict::forced_import(
&file,
&db,
"close_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
format,
VecBuilderOptions::default().add_last(),
)?,
weekindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
weekindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
difficultyepoch_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
difficultyepoch_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
monthindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
monthindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
quarterindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
quarterindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
semesterindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
semesterindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
yearindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
yearindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
// halvingepoch_to_ohlc: StorableVec::forced_import(file,
// halvingepoch_to_ohlc: StorableVec::forced_import(db,
// "halvingepoch_to_ohlc"), version + VERSION + Version::ZERO, format)?,
decadeindex_to_ohlc: RawVec::forced_import(
&file,
&db,
"ohlc",
version + VERSION + Version::ZERO,
)?,
decadeindex_to_ohlc_in_sats: RawVec::forced_import(
&file,
&db,
"ohlc_in_sats",
version + VERSION + VERSION_IN_SATS + Version::ZERO,
)?,
file,
db,
})
}
@@ -374,7 +370,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, fetched, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

55
crates/brk_computer/src/stateful/address_cohort.rs
View File

@@ -1,10 +1,10 @@
use std::{ops::Deref, path::Path, sync::Arc};
use std::{ops::Deref, path::Path};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, StoredU64, Version};
use brk_vecs::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, EagerVec, Exit, File,
use vecdb::{
AnyCollectableVec, AnyIterableVec, AnyStoredVec, AnyVec, Computation, Database, EagerVec, Exit,
Format, GenericStoredVec, VecIterator,
};
@@ -25,7 +25,7 @@ const VERSION: Version = Version::ZERO;
pub struct Vecs {
starting_height: Height,
pub state: AddressCohortState,
pub state: Option<AddressCohortState>,
pub inner: common::Vecs,
@@ -36,14 +36,14 @@ pub struct Vecs {
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
cohort_name: Option<&str>,
computation: Computation,
format: Format,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
states_path: Option<&Path>,
compute_relative_to_all: bool,
) -> Result<Self> {
let compute_dollars = price.is_some();
@@ -52,19 +52,22 @@ impl Vecs {
Ok(Self {
starting_height: Height::ZERO,
state: AddressCohortState::default_and_import(
states_path,
cohort_name.unwrap_or_default(),
compute_dollars,
)?,
state: states_path.map(|states_path| {
AddressCohortState::default_and_import(
states_path,
cohort_name.unwrap_or_default(),
compute_dollars,
)
.unwrap()
}),
height_to_address_count: EagerVec::forced_import(
file,
db,
&suffix("address_count"),
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_address_count: ComputedVecsFromHeight::forced_import(
file,
db,
&suffix("address_count"),
Source::None,
version + VERSION + Version::ZERO,
@@ -74,7 +77,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
inner: common::Vecs::forced_import(
file,
db,
cohort_name,
computation,
format,
@@ -83,6 +86,7 @@ impl Vecs {
price,
compute_relative_to_all,
false,
false,
)?,
})
}
@@ -91,7 +95,9 @@ impl Vecs {
impl DynCohortVecs for Vecs {
fn starting_height(&self) -> Height {
[
self.state.height().map_or(Height::MAX, |h| h.incremented()),
self.state.as_ref().map_or(Height::MAX, |state| {
state.height().map_or(Height::MAX, |h| h.incremented())
}),
self.height_to_address_count.len().into(),
self.inner.starting_height(),
]
@@ -108,19 +114,21 @@ impl DynCohortVecs for Vecs {
self.starting_height = starting_height;
if let Some(prev_height) = starting_height.decremented() {
self.state.address_count = *self
self.state.as_mut().unwrap().address_count = *self
.height_to_address_count
.into_iter()
.unwrap_get_inner(prev_height);
}
self.inner
.init(&mut self.starting_height, &mut self.state.inner);
self.inner.init(
&mut self.starting_height,
&mut self.state.as_mut().unwrap().inner,
);
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
self.height_to_address_count
.validate_computed_version_or_reset_file(
.validate_computed_version_or_reset(
base_version + self.height_to_address_count.inner_version(),
)?;
@@ -134,11 +142,12 @@ impl DynCohortVecs for Vecs {
self.height_to_address_count.forced_push_at(
height,
self.state.address_count.into(),
self.state.as_ref().unwrap().address_count.into(),
exit,
)?;
self.inner.forced_pushed_at(height, exit, &self.state.inner)
self.inner
.forced_pushed_at(height, exit, &self.state.as_ref().unwrap().inner)
}
fn compute_then_force_push_unrealized_states(
@@ -155,7 +164,7 @@ impl DynCohortVecs for Vecs {
dateindex,
date_price,
exit,
&self.state.inner,
&self.state.as_ref().unwrap().inner,
)
}
@@ -163,7 +172,7 @@ impl DynCohortVecs for Vecs {
self.height_to_address_count.safe_flush(exit)?;
self.inner
.safe_flush_stateful_vecs(height, exit, &mut self.state.inner)
.safe_flush_stateful_vecs(height, exit, &mut self.state.as_mut().unwrap().inner)
}
#[allow(clippy::too_many_arguments)]

170
crates/brk_computer/src/stateful/address_cohorts.rs
View File

@@ -1,4 +1,4 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
@@ -6,8 +6,8 @@ use brk_structs::{
AddressGroups, Bitcoin, ByAmountRange, ByGreatEqualAmount, ByLowerThanAmount, DateIndex,
Dollars, GroupFilter, Height, Version,
};
use brk_vecs::{AnyIterableVec, Computation, Exit, File, Format};
use derive_deref::{Deref, DerefMut};
use vecdb::{AnyIterableVec, Computation, Database, Exit, Format};
use crate::{
Indexes, indexes, market, price,
@@ -24,7 +24,7 @@ pub struct Vecs(AddressGroups<(GroupFilter, address_cohort::Vecs)>);
impl Vecs {
pub fn forced_import(
file: &Arc<File>,
db: &Database,
version: Version,
_computation: Computation,
format: Format,
@@ -36,458 +36,458 @@ impl Vecs {
AddressGroups {
amount_range: ByAmountRange {
_0sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_with_0sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_1sat_to_10sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1sat_under_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_10sats_to_100sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10sats_under_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_100sats_to_1k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100sats_under_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_1k_sats_to_10k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1k_sats_under_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_10k_sats_to_100k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10k_sats_under_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_100k_sats_to_1m_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100k_sats_under_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_1m_sats_to_10m_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1m_sats_under_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_10m_sats_to_1btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10m_sats_under_1btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_1btc_to_10btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1btc_under_10btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_10btc_to_100btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10btc_under_100btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_100btc_to_1k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100btc_under_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_1k_btc_to_10k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1k_btc_under_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_10k_btc_to_100k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10k_btc_under_100k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
_100k_btc_or_more: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
Some(states_path),
true,
)?,
},
lt_amount: ByLowerThanAmount {
_10sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1m_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10m_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_1btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_10btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_100btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_under_100k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
},
ge_amount: ByGreatEqualAmount {
_1sat: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1sat"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100k_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100k_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1m_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10m_sats: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10m_sats"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_100btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_100btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_1k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_1k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
_10k_btc: address_cohort::Vecs::forced_import(
file,
db,
Some("addrs_above_10k_btc"),
_computation,
format,
version + VERSION + Version::ZERO,
indexes,
price,
states_path,
None,
true,
)?,
},

2
crates/brk_computer/src/stateful/addresstype_to_addresscount.rs
View File

@@ -1,6 +1,6 @@
use brk_structs::{ByAddressType, Height};
use brk_vecs::VecIterator;
use derive_deref::{Deref, DerefMut};
use vecdb::VecIterator;
use crate::stateful::addresstype_to_height_to_addresscount::AddressTypeToHeightToAddressCount;

2
crates/brk_computer/src/stateful/addresstype_to_height_to_addresscount.rs
View File

@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_structs::{ByAddressType, Height, StoredU64};
use brk_vecs::{EagerVec, Exit, GenericStoredVec};
use vecdb::{EagerVec, Exit, GenericStoredVec};
use derive_deref::{Deref, DerefMut};
use crate::stateful::addresstype_to_addresscount::AddressTypeToAddressCount;

2
crates/brk_computer/src/stateful/addresstype_to_indexes_to_addresscount.rs
View File

@@ -1,7 +1,7 @@
use brk_error::Result;
use brk_structs::{ByAddressType, StoredU64};
use brk_vecs::{AnyCollectableVec, Exit};
use derive_deref::{Deref, DerefMut};
use vecdb::{AnyCollectableVec, Exit};
use crate::{
Indexes, grouped::ComputedVecsFromHeight, indexes,

450
crates/brk_computer/src/stateful/common.rs
View File

File diff suppressed because it is too large Load Diff

185
crates/brk_computer/src/stateful/mod.rs
View File

@@ -1,4 +1,4 @@
use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, sync::Arc, thread};
use std::{cmp::Ordering, collections::BTreeMap, mem, path::Path, thread};
use brk_error::Result;
use brk_indexer::Indexer;
@@ -9,12 +9,12 @@ use brk_structs::{
P2PK65AddressIndex, P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex,
P2WSHAddressIndex, Sats, StoredU64, Timestamp, TypeIndex, Version,
};
use brk_vecs::{
AnyCollectableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, EagerVec, Exit, File,
Format, GenericStoredVec, PAGE_SIZE, RawVec, Reader, Stamp, StoredIndex, VecIterator,
};
use log::info;
use rayon::prelude::*;
use vecdb::{
AnyCollectableVec, AnyStoredVec, AnyVec, CollectableVec, Computation, Database, EagerVec, Exit,
Format, GenericStoredVec, PAGE_SIZE, RawVec, Reader, Stamp, StoredIndex, VecIterator,
};
use crate::{
BlockState, Indexes, SupplyState, Transacted,
@@ -52,7 +52,7 @@ const VERSION: Version = Version::new(21);
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
pub chain_state: RawVec<Height, SupplyState>,
@@ -92,29 +92,29 @@ impl Vecs {
price: Option<&price::Vecs>,
states_path: &Path,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("stateful"))?);
file.set_min_len(PAGE_SIZE * 20_000_000)?;
file.set_min_regions(50_000)?;
let db = Database::open(&parent.join("stateful"))?;
db.set_min_len(PAGE_SIZE * 20_000_000)?;
db.set_min_regions(50_000)?;
let compute_dollars = price.is_some();
let chain_file = Arc::new(File::open(&parent.join("chain"))?);
let chain_db = Database::open(&parent.join("chain"))?;
Ok(Self {
chain_state: RawVec::forced_import(
&chain_file,
&chain_db,
"chain",
version + VERSION + Version::ZERO,
)?,
height_to_unspendable_supply: EagerVec::forced_import(
&file,
&db,
"unspendable_supply",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_unspendable_supply: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"unspendable_supply",
Source::None,
version + VERSION + Version::ZERO,
@@ -125,13 +125,13 @@ impl Vecs {
indexes,
)?,
height_to_opreturn_supply: EagerVec::forced_import(
&file,
&db,
"opreturn_supply",
version + VERSION + Version::ZERO,
format,
)?,
indexes_to_opreturn_supply: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"opreturn_supply",
Source::None,
version + VERSION + Version::ZERO,
@@ -142,7 +142,7 @@ impl Vecs {
indexes,
)?,
indexes_to_address_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"address_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -152,7 +152,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
indexes_to_empty_address_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"empty_address_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -164,49 +164,49 @@ impl Vecs {
addresstype_to_height_to_address_count: AddressTypeToHeightToAddressCount::from(
ByAddressType {
p2pk65: EagerVec::forced_import(
&file,
&db,
"p2pk65_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
&file,
&db,
"p2pk33_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
&file,
&db,
"p2pkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
&file,
&db,
"p2sh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
&file,
&db,
"p2wpkh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
&file,
&db,
"p2wsh_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
&file,
&db,
"p2tr_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
&file,
&db,
"p2a_address_count",
version + VERSION + Version::ZERO,
format,
@@ -216,49 +216,49 @@ impl Vecs {
addresstype_to_height_to_empty_address_count: AddressTypeToHeightToAddressCount::from(
ByAddressType {
p2pk65: EagerVec::forced_import(
&file,
&db,
"p2pk65_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pk33: EagerVec::forced_import(
&file,
&db,
"p2pk33_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2pkh: EagerVec::forced_import(
&file,
&db,
"p2pkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2sh: EagerVec::forced_import(
&file,
&db,
"p2sh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wpkh: EagerVec::forced_import(
&file,
&db,
"p2wpkh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2wsh: EagerVec::forced_import(
&file,
&db,
"p2wsh_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2tr: EagerVec::forced_import(
&file,
&db,
"p2tr_empty_address_count",
version + VERSION + Version::ZERO,
format,
)?,
p2a: EagerVec::forced_import(
&file,
&db,
"p2a_empty_address_count",
version + VERSION + Version::ZERO,
format,
@@ -268,7 +268,7 @@ impl Vecs {
addresstype_to_indexes_to_address_count: AddressTypeToIndexesToAddressCount::from(
ByAddressType {
p2pk65: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pk65_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -278,7 +278,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pk33: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pk33_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -288,7 +288,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pkh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pkh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -298,7 +298,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2sh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2sh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -308,7 +308,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wpkh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2wpkh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -318,7 +318,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wsh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2wsh_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -328,7 +328,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2tr: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2tr_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -338,7 +338,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2a: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2a_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -352,7 +352,7 @@ impl Vecs {
addresstype_to_indexes_to_empty_address_count: AddressTypeToIndexesToAddressCount::from(
ByAddressType {
p2pk65: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pk65_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -362,7 +362,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pk33: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pk33_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -372,7 +372,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2pkh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pkh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -382,7 +382,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2sh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2sh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -392,7 +392,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wpkh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2wpkh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -402,7 +402,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2wsh: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2wsh_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -412,7 +412,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2tr: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2tr_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -422,7 +422,7 @@ impl Vecs {
VecBuilderOptions::default().add_last(),
)?,
p2a: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2a_empty_address_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -434,7 +434,7 @@ impl Vecs {
},
),
utxo_cohorts: utxo_cohorts::Vecs::forced_import(
&file,
&db,
version,
computation,
format,
@@ -443,7 +443,7 @@ impl Vecs {
states_path,
)?,
address_cohorts: address_cohorts::Vecs::forced_import(
&file,
&db,
version,
computation,
format,
@@ -453,58 +453,58 @@ impl Vecs {
)?,
p2aaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2pk33addressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2pk65addressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2pkhaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2shaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2traddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2wpkhaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
p2wshaddressindex_to_anyaddressindex: RawVec::forced_import(
&file,
&db,
"anyaddressindex",
version + VERSION + Version::ZERO,
)?,
loadedaddressindex_to_loadedaddressdata: RawVec::forced_import(
&file,
&db,
"loadedaddressdata",
version + VERSION + Version::ZERO,
)?,
emptyaddressindex_to_emptyaddressdata: RawVec::forced_import(
&file,
&db,
"emptyaddressdata",
version + VERSION + Version::ZERO,
)?,
file,
db,
})
}
@@ -529,7 +529,7 @@ impl Vecs {
starting_indexes,
exit,
)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}
@@ -624,11 +624,11 @@ impl Vecs {
.par_iter_mut()
.try_for_each(|(_, v)| v.validate_computed_versions(base_version))?;
self.height_to_unspendable_supply
.validate_computed_version_or_reset_file(
.validate_computed_version_or_reset(
base_version + self.height_to_unspendable_supply.inner_version(),
)?;
self.height_to_opreturn_supply
.validate_computed_version_or_reset_file(
.validate_computed_version_or_reset(
base_version + self.height_to_opreturn_supply.inner_version(),
)?;
@@ -712,13 +712,14 @@ impl Vecs {
separate_utxo_vecs
.par_iter_mut()
.try_for_each(|(_, v)| v.state.reset_price_to_amount())?;
.flat_map(|(_, v)| v.state.as_mut())
.try_for_each(|state| state.reset_price_to_amount())?;
info!("Resetting address price maps...");
separate_address_vecs
.par_iter_mut()
.try_for_each(|(_, v)| v.state.reset_price_to_amount())?;
.try_for_each(|(_, v)| v.state.as_mut().unwrap().reset_price_to_amount())?;
};
let last_height = Height::from(indexer.vecs.height_to_blockhash.stamp());
@@ -820,12 +821,12 @@ impl Vecs {
self.utxo_cohorts
.as_mut_separate_vecs()
.iter_mut()
.for_each(|(_, v)| v.state.reset_single_iteration_values());
.for_each(|(_, v)| v.state.as_mut().unwrap().reset_single_iteration_values());
self.address_cohorts
.as_mut_separate_vecs()
.iter_mut()
.for_each(|(_, v)| v.state.reset_single_iteration_values());
.for_each(|(_, v)| v.state.as_mut().unwrap().reset_single_iteration_values());
let timestamp = height_to_timestamp_fixed_iter.unwrap_get_inner(height);
let price = height_to_close_iter
@@ -1874,17 +1875,27 @@ impl AddressTypeToVec<(TypeIndex, Sats)> {
.get_mut(prev_amount)
.1
.state
.as_mut()
.unwrap()
.subtract(addressdata);
}
addressdata.receive(value, price);
vecs.amount_range.get_mut(amount).1.state.add(addressdata);
vecs.amount_range
.get_mut(amount)
.1
.state
.as_mut()
.unwrap()
.add(addressdata);
} else {
vecs.amount_range
.get_mut(amount)
.1
.state
.as_mut()
.unwrap()
.receive(addressdata, value, price);
}
});
@@ -1965,6 +1976,8 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
.get_mut(prev_amount)
.1
.state
.as_mut()
.unwrap()
.subtract(addressdata);
addressdata.send(value, prev_price)?;
@@ -1985,18 +1998,30 @@ impl HeightToAddressTypeToVec<(TypeIndex, Sats)> {
.unwrap()
.insert(type_index, addressdata.into());
} else {
vecs.amount_range.get_mut(amount).1.state.add(addressdata);
vecs.amount_range
.get_mut(amount)
.1
.state
.as_mut()
.unwrap()
.add(addressdata);
}
} else {
vecs.amount_range.get_mut(amount).1.state.send(
addressdata,
value,
price,
prev_price,
blocks_old,
days_old,
older_than_hour,
)?;
vecs.amount_range
.get_mut(amount)
.1
.state
.as_mut()
.unwrap()
.send(
addressdata,
value,
price,
prev_price,
blocks_old,
days_old,
older_than_hour,
)?;
}
Ok(())

2
crates/brk_computer/src/stateful/range_map.rs
View File

@@ -1,6 +1,6 @@
use std::collections::BTreeMap;
use brk_vecs::{CompressedVec, RawVec, StoredCompressed, StoredIndex, StoredRaw};
use vecdb::{CompressedVec, RawVec, StoredCompressed, StoredIndex, StoredRaw};
#[derive(Debug)]
pub struct RangeMap<I, T>(BTreeMap<I, T>);

2
crates/brk_computer/src/stateful/trait.rs
View File

@@ -1,7 +1,7 @@
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version};
use brk_vecs::{AnyCollectableVec, AnyIterableVec, Exit};
use vecdb::{AnyCollectableVec, AnyIterableVec, Exit};
use crate::{Indexes, indexes, market, price};

41
crates/brk_computer/src/stateful/utxo_cohort.rs
View File

@@ -1,9 +1,9 @@
use std::{ops::Deref, path::Path, sync::Arc};
use std::{ops::Deref, path::Path};
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::{Bitcoin, DateIndex, Dollars, Height, Version};
use brk_vecs::{AnyCollectableVec, AnyIterableVec, Computation, Exit, File, Format};
use vecdb::{AnyCollectableVec, AnyIterableVec, Computation, Database, Exit, Format};
use crate::{
Indexes, UTXOCohortState, indexes, market, price,
@@ -17,7 +17,7 @@ use crate::{
pub struct Vecs {
starting_height: Height,
pub state: UTXOCohortState,
pub state: Option<UTXOCohortState>,
inner: common::Vecs,
}
@@ -25,30 +25,34 @@ pub struct Vecs {
impl Vecs {
#[allow(clippy::too_many_arguments)]
pub fn forced_import(
file: &Arc<File>,
db: &Database,
cohort_name: Option<&str>,
computation: Computation,
format: Format,
version: Version,
indexes: &indexes::Vecs,
price: Option<&price::Vecs>,
states_path: &Path,
states_path: Option<&Path>,
compute_relative_to_all: bool,
ratio_extended: bool,
compute_adjusted: bool,
) -> Result<Self> {
let compute_dollars = price.is_some();
Ok(Self {
starting_height: Height::ZERO,
state: UTXOCohortState::default_and_import(
states_path,
cohort_name.unwrap_or_default(),
compute_dollars,
)?,
state: states_path.map(|states_path| {
UTXOCohortState::default_and_import(
states_path,
cohort_name.unwrap_or_default(),
compute_dollars,
)
.unwrap()
}),
inner: common::Vecs::forced_import(
file,
db,
cohort_name,
computation,
format,
@@ -57,6 +61,7 @@ impl Vecs {
price,
compute_relative_to_all,
ratio_extended,
compute_adjusted,
)?,
})
}
@@ -65,7 +70,9 @@ impl Vecs {
impl DynCohortVecs for Vecs {
fn starting_height(&self) -> Height {
[
self.state.height().map_or(Height::MAX, |h| h.incremented()),
self.state.as_ref().map_or(Height::MAX, |state| {
state.height().map_or(Height::MAX, |h| h.incremented())
}),
self.inner.starting_height(),
]
.into_iter()
@@ -80,7 +87,8 @@ impl DynCohortVecs for Vecs {
self.starting_height = starting_height;
self.inner.init(&mut self.starting_height, &mut self.state);
self.inner
.init(&mut self.starting_height, self.state.as_mut().unwrap());
}
fn validate_computed_versions(&mut self, base_version: Version) -> Result<()> {
@@ -92,7 +100,8 @@ impl DynCohortVecs for Vecs {
return Ok(());
}
self.inner.forced_pushed_at(height, exit, &self.state)
self.inner
.forced_pushed_at(height, exit, self.state.as_ref().unwrap())
}
fn compute_then_force_push_unrealized_states(
@@ -109,13 +118,13 @@ impl DynCohortVecs for Vecs {
dateindex,
date_price,
exit,
&self.state,
self.state.as_mut().unwrap(),
)
}
fn safe_flush_stateful_vecs(&mut self, height: Height, exit: &Exit) -> Result<()> {
self.inner
.safe_flush_stateful_vecs(height, exit, &mut self.state)
.safe_flush_stateful_vecs(height, exit, self.state.as_mut().unwrap())
}
#[allow(clippy::too_many_arguments)]

653
crates/brk_computer/src/stateful/utxo_cohorts.rs
View File

File diff suppressed because it is too large Load Diff

16
crates/brk_computer/src/traits.rs
View File

@@ -1,6 +1,6 @@
use brk_error::Result;
use brk_structs::{Bitcoin, CheckedSub, Close, Date, DateIndex, Dollars, Sats, StoredF32};
use brk_vecs::{
use vecdb::{
AnyIterableVec, AnyStoredVec, AnyVec, EagerVec, Exit, GenericStoredVec, StoredIndex,
VecIterator, Version,
};
@@ -32,7 +32,7 @@ impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
len: usize,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + closes.version(),
)?;
@@ -83,7 +83,7 @@ impl ComputeDCAStackViaLen for EagerVec<DateIndex, Sats> {
from: DateIndex,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + closes.version(),
)?;
@@ -143,7 +143,7 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
len: usize,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ONE + self.inner_version() + stacks.version(),
)?;
@@ -176,7 +176,7 @@ impl ComputeDCAAveragePriceViaLen for EagerVec<DateIndex, Dollars> {
from: DateIndex,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + stacks.version(),
)?;
@@ -218,7 +218,7 @@ where
sats: &impl AnyIterableVec<I, Sats>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + sats.version(),
)?;
@@ -254,7 +254,7 @@ where
price: &impl AnyIterableVec<I, Close<Dollars>>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + bitcoin.version(),
)?;
@@ -292,7 +292,7 @@ where
ath: &impl AnyIterableVec<I, Dollars>,
exit: &Exit,
) -> Result<()> {
self.validate_computed_version_or_reset_file(
self.validate_computed_version_or_reset(
Version::ZERO + self.inner_version() + ath.version() + close.version(),
)?;

88
crates/brk_computer/src/transactions.rs
View File

@@ -1,4 +1,4 @@
use std::{path::Path, sync::Arc};
use std::path::Path;
use brk_error::Result;
use brk_indexer::Indexer;
@@ -6,9 +6,9 @@ use brk_structs::{
CheckedSub, Feerate, HalvingEpoch, Height, InputIndex, OutputIndex, Sats, StoredBool,
StoredU32, StoredU64, TxIndex, TxVersion, Version, Weight,
};
use brk_vecs::{
use vecdb::{
AnyCloneableIterableVec, AnyCollectableVec, AnyIterableVec, Computation, ComputedVec,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Exit, File, Format, PAGE_SIZE,
ComputedVecFrom1, ComputedVecFrom2, ComputedVecFrom3, Database, Exit, Format, PAGE_SIZE,
StoredIndex, VecIterator,
};
@@ -23,7 +23,7 @@ const VERSION: Version = Version::ZERO;
#[derive(Clone)]
pub struct Vecs {
file: Arc<File>,
db: Database,
// pub txindex_to_is_v1: LazyVec<Txindex, bool>,
// pub txindex_to_is_v2: LazyVec<Txindex, bool>,
@@ -90,14 +90,14 @@ impl Vecs {
format: Format,
price: Option<&price::Vecs>,
) -> Result<Self> {
let file = Arc::new(File::open(&parent.join("transactions"))?);
file.set_min_len(PAGE_SIZE * 10_000_000)?;
let db = Database::open(&parent.join("transactions"))?;
db.set_min_len(PAGE_SIZE * 10_000_000)?;
let compute_dollars = price.is_some();
let inputindex_to_value = ComputedVec::forced_import_or_init_from_2(
computation,
&file,
&db,
"value",
version + VERSION + Version::ZERO,
format,
@@ -124,7 +124,7 @@ impl Vecs {
let txindex_to_weight = ComputedVec::forced_import_or_init_from_2(
computation,
&file,
&db,
"weight",
version + VERSION + Version::ZERO,
format,
@@ -152,7 +152,7 @@ impl Vecs {
let txindex_to_vsize = ComputedVec::forced_import_or_init_from_1(
computation,
&file,
&db,
"vsize",
version + VERSION + Version::ZERO,
format,
@@ -169,7 +169,7 @@ impl Vecs {
let txindex_to_is_coinbase = ComputedVec::forced_import_or_init_from_2(
computation,
&file,
&db,
"is_coinbase",
version + VERSION + Version::ZERO,
format,
@@ -192,7 +192,7 @@ impl Vecs {
let txindex_to_input_value = ComputedVec::forced_import_or_init_from_3(
computation,
&file,
&db,
"input_value",
version + VERSION + Version::ZERO,
format,
@@ -228,7 +228,7 @@ impl Vecs {
// let indexes_to_input_value: ComputedVecsFromTxindex<Sats> =
// ComputedVecsFromTxindex::forced_import(
// file,
// db,
// "input_value",
// true,
// version + VERSION + Version::ZERO,
@@ -242,7 +242,7 @@ impl Vecs {
let txindex_to_output_value = ComputedVec::forced_import_or_init_from_3(
computation,
&file,
&db,
"output_value",
version + VERSION + Version::ZERO,
format,
@@ -278,7 +278,7 @@ impl Vecs {
// let indexes_to_output_value: ComputedVecsFromTxindex<Sats> =
// ComputedVecsFromTxindex::forced_import(
// file,
// db,
// "output_value",
// true,
// version + VERSION + Version::ZERO,
@@ -292,7 +292,7 @@ impl Vecs {
let txindex_to_fee = ComputedVecFrom2::forced_import_or_init_from_2(
Computation::Eager,
&file,
&db,
"fee",
version + VERSION + Version::ZERO,
format,
@@ -315,7 +315,7 @@ impl Vecs {
let txindex_to_feerate = ComputedVecFrom2::forced_import_or_init_from_2(
Computation::Eager,
&file,
&db,
"feerate",
version + VERSION + Version::ZERO,
format,
@@ -335,7 +335,7 @@ impl Vecs {
Ok(Self {
indexes_to_tx_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"tx_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -350,7 +350,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_input_count: ComputedVecsFromTxindex::forced_import(
&file,
&db,
"input_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -365,7 +365,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_output_count: ComputedVecsFromTxindex::forced_import(
&file,
&db,
"output_count",
Source::None,
version + VERSION + Version::ZERO,
@@ -380,7 +380,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_tx_v1: ComputedVecsFromHeight::forced_import(
&file,
&db,
"tx_v1",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -390,7 +390,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_tx_v2: ComputedVecsFromHeight::forced_import(
&file,
&db,
"tx_v2",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -400,7 +400,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_tx_v3: ComputedVecsFromHeight::forced_import(
&file,
&db,
"tx_v3",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -410,7 +410,7 @@ impl Vecs {
VecBuilderOptions::default().add_sum().add_cumulative(),
)?,
indexes_to_fee: ComputedValueVecsFromTxindex::forced_import(
&file,
&db,
"fee",
indexes,
Source::Vec(txindex_to_fee.boxed_clone()),
@@ -426,7 +426,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_feerate: ComputedVecsFromTxindex::forced_import(
&file,
&db,
"feerate",
Source::None,
version + VERSION + Version::ZERO,
@@ -439,7 +439,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_tx_vsize: ComputedVecsFromTxindex::forced_import(
&file,
&db,
"tx_vsize",
Source::None,
version + VERSION + Version::ZERO,
@@ -452,7 +452,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_tx_weight: ComputedVecsFromTxindex::forced_import(
&file,
&db,
"tx_weight",
Source::None,
version + VERSION + Version::ZERO,
@@ -465,7 +465,7 @@ impl Vecs {
.add_average(),
)?,
indexes_to_subsidy: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"subsidy",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -481,7 +481,7 @@ impl Vecs {
indexes,
)?,
indexes_to_coinbase: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"coinbase",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -497,7 +497,7 @@ impl Vecs {
indexes,
)?,
indexes_to_unclaimed_rewards: ComputedValueVecsFromHeight::forced_import(
&file,
&db,
"unclaimed_rewards",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -508,7 +508,7 @@ impl Vecs {
indexes,
)?,
indexes_to_p2a_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2a_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -523,7 +523,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2ms_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2ms_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -538,7 +538,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pk33_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pk33_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -553,7 +553,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pk65_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pk65_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -568,7 +568,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2pkh_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2pkh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -583,7 +583,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2sh_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2sh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -598,7 +598,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2tr_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2tr_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -613,7 +613,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2wpkh_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2wpkh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -628,7 +628,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_p2wsh_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"p2wsh_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -643,7 +643,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_opreturn_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"opreturn_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -658,7 +658,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_unknownoutput_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"unknownoutput_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -673,7 +673,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_emptyoutput_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"emptyoutput_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -688,7 +688,7 @@ impl Vecs {
.add_cumulative(),
)?,
indexes_to_exact_utxo_count: ComputedVecsFromHeight::forced_import(
&file,
&db,
"exact_utxo_count",
Source::Compute,
version + VERSION + Version::ZERO,
@@ -708,7 +708,7 @@ impl Vecs {
txindex_to_vsize,
txindex_to_weight,
file,
db,
})
}
@@ -721,7 +721,7 @@ impl Vecs {
exit: &Exit,
) -> Result<()> {
self.compute_(indexer, indexes, starting_indexes, price, exit)?;
self.file.flush_then_punch()?;
self.db.flush_then_punch()?;
Ok(())
}

2
crates/brk_error/Cargo.toml
View File

@@ -9,7 +9,7 @@ repository.workspace = true
build = "build.rs"
[dependencies]
brk_vecs = { workspace = true }
vecdb = { workspace = true }
bitcoincore-rpc = { workspace = true }
fjall = { workspace = true }
jiff = { workspace = true }

24
crates/brk_error/src/lib.rs
View File

@@ -11,11 +11,13 @@ pub enum Error {
BitcoinRPC(bitcoincore_rpc::Error),
Jiff(jiff::Error),
Fjall(fjall::Error),
VecDB(vecdb::Error),
SeqDB(vecdb::SeqDBError),
Minreq(minreq::Error),
SystemTimeError(time::SystemTimeError),
SerdeJson(serde_json::Error),
ZeroCopyError,
Vecs(brk_vecs::Error),
Vecs(vecdb::Error),
WrongLength,
WrongAddressType,
@@ -43,6 +45,18 @@ impl From<io::Error> for Error {
}
}
impl From<vecdb::Error> for Error {
fn from(value: vecdb::Error) -> Self {
Self::VecDB(value)
}
}
impl From<vecdb::SeqDBError> for Error {
fn from(value: vecdb::SeqDBError) -> Self {
Self::SeqDB(value)
}
}
impl From<bitcoincore_rpc::Error> for Error {
fn from(value: bitcoincore_rpc::Error) -> Self {
Self::BitcoinRPC(value)
@@ -55,12 +69,6 @@ impl From<minreq::Error> for Error {
}
}
impl From<brk_vecs::Error> for Error {
fn from(value: brk_vecs::Error) -> Self {
Self::Vecs(value)
}
}
impl From<jiff::Error> for Error {
fn from(value: jiff::Error) -> Self {
Self::Jiff(value)
@@ -91,6 +99,8 @@ impl fmt::Display for Error {
Error::IO(error) => Display::fmt(&error, f),
Error::Minreq(error) => Display::fmt(&error, f),
Error::SerdeJson(error) => Display::fmt(&error, f),
Error::VecDB(error) => Display::fmt(&error, f),
Error::SeqDB(error) => Display::fmt(&error, f),
Error::Vecs(error) => Display::fmt(&error, f),
Error::BitcoinRPC(error) => Display::fmt(&error, f),
Error::SystemTimeError(error) => Display::fmt(&error, f),

2
crates/brk_fetcher/examples/main.rs
View File

@@ -9,7 +9,7 @@ fn main() -> Result<()> {
dbg!(brk.get_from_height(Height::new(900_000))?);
dbg!(brk.get_from_date(Date::new(2025, 6, 7))?);
let mut fetcher = Fetcher::import(None)?;
let mut fetcher = Fetcher::import(true, None)?;
Binance::fetch_1d().map(|b| {
dbg!(b.last_key_value());

28
crates/brk_fetcher/src/binance.rs
View File

@@ -11,7 +11,7 @@ use brk_structs::{Cents, OHLCCents, Timestamp};
use log::info;
use serde_json::Value;
use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, retry};
use crate::{Close, Date, Dollars, Fetcher, High, Low, Open, default_retry};
#[derive(Clone)]
pub struct Binance {
@@ -68,17 +68,13 @@ impl Binance {
pub fn fetch_1mn() -> Result<BTreeMap<Timestamp, OHLCCents>> {
info!("Fetching 1mn prices from Binance...");
retry(
|_| {
Self::json_to_timestamp_to_ohlc(
&minreq::get(Self::url("interval=1m&limit=1000"))
.send()?
.json()?,
)
},
30,
10,
)
default_retry(|_| {
Self::json_to_timestamp_to_ohlc(
&minreq::get(Self::url("interval=1m&limit=1000"))
.send()?
.json()?,
)
})
}
pub fn get_from_1d(&mut self, date: &Date) -> Result<OHLCCents> {
@@ -97,11 +93,9 @@ impl Binance {
pub fn fetch_1d() -> Result<BTreeMap<Date, OHLCCents>> {
info!("Fetching daily prices from Binance...");
retry(
|_| Self::json_to_date_to_ohlc(&minreq::get(Self::url("interval=1d")).send()?.json()?),
30,
10,
)
default_retry(|_| {
Self::json_to_date_to_ohlc(&minreq::get(Self::url("interval=1d")).send()?.json()?)
})
}
fn read_har(&self) -> Result<BTreeMap<Timestamp, OHLCCents>> {

63
crates/brk_fetcher/src/brk.rs
View File

@@ -5,7 +5,7 @@ use brk_structs::{Cents, CheckedSub, Date, DateIndex, Height, OHLCCents};
use log::info;
use serde_json::Value;
use crate::{Close, Dollars, High, Low, Open, retry};
use crate::{Close, Dollars, High, Low, Open, default_retry};
#[derive(Default, Clone)]
#[allow(clippy::upper_case_acronyms)]
@@ -15,7 +15,6 @@ pub struct BRK {
}
const API_URL: &str = "https://bitcoinresearchkit.org/api/vecs";
const RETRIES: usize = 10;
const CHUNK_SIZE: usize = 10_000;
impl BRK {
@@ -45,25 +44,21 @@ impl BRK {
fn fetch_height_prices(height: Height) -> Result<Vec<OHLCCents>> {
info!("Fetching BRK height {height} prices...");
retry(
|_| {
let url = format!(
"{API_URL}/height-to-ohlc?from={}&to={}",
height,
height + CHUNK_SIZE
);
default_retry(|_| {
let url = format!(
"{API_URL}/height-to-ohlc?from={}&to={}",
height,
height + CHUNK_SIZE
);
let body: Value = minreq::get(url).send()?.json()?;
let body: Value = minreq::get(url).send()?.json()?;
body.as_array()
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.map(Self::value_to_ohlc)
.collect::<Result<Vec<_>, _>>()
},
30,
RETRIES,
)
body.as_array()
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.map(Self::value_to_ohlc)
.collect::<Result<Vec<_>, _>>()
})
}
pub fn get_from_date(&mut self, date: Date) -> Result<OHLCCents> {
@@ -94,25 +89,21 @@ impl BRK {
fn fetch_date_prices(dateindex: DateIndex) -> Result<Vec<OHLCCents>> {
info!("Fetching BRK dateindex {dateindex} prices...");
retry(
|_| {
let url = format!(
"{API_URL}/dateindex-to-ohlc?from={}&to={}",
dateindex,
dateindex + CHUNK_SIZE
);
default_retry(|_| {
let url = format!(
"{API_URL}/dateindex-to-ohlc?from={}&to={}",
dateindex,
dateindex + CHUNK_SIZE
);
let body: Value = minreq::get(url).send()?.json()?;
let body: Value = minreq::get(url).send()?.json()?;
body.as_array()
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.map(Self::value_to_ohlc)
.collect::<Result<Vec<_>, _>>()
},
30,
RETRIES,
)
body.as_array()
.ok_or(Error::Str("Expect to be an array"))?
.iter()
.map(Self::value_to_ohlc)
.collect::<Result<Vec<_>, _>>()
})
}
fn value_to_ohlc(value: &Value) -> Result<OHLCCents> {

16
crates/brk_fetcher/src/kraken.rs
View File

@@ -5,7 +5,7 @@ use brk_structs::{Cents, Close, Date, Dollars, High, Low, OHLCCents, Open, Times
use log::info;
use serde_json::Value;
use crate::{Fetcher, retry};
use crate::{Fetcher, default_retry};
#[derive(Default, Clone)]
pub struct Kraken {
@@ -35,11 +35,9 @@ impl Kraken {
pub fn fetch_1mn() -> Result<BTreeMap<Timestamp, OHLCCents>> {
info!("Fetching 1mn prices from Kraken...");
retry(
|_| Self::json_to_timestamp_to_ohlc(&minreq::get(Self::url(1)).send()?.json()?),
30,
10,
)
default_retry(|_| {
Self::json_to_timestamp_to_ohlc(&minreq::get(Self::url(1)).send()?.json()?)
})
}
pub fn get_from_1d(&mut self, date: &Date) -> Result<OHLCCents> {
@@ -57,11 +55,7 @@ impl Kraken {
pub fn fetch_1d() -> Result<BTreeMap<Date, OHLCCents>> {
info!("Fetching daily prices from Kraken...");
retry(
|_| Self::json_to_date_to_ohlc(&minreq::get(Self::url(1440)).send()?.json()?),
30,
10,
)
default_retry(|_| Self::json_to_date_to_ohlc(&minreq::get(Self::url(1440)).send()?.json()?))
}
fn json_to_timestamp_to_ohlc(json: &Value) -> Result<BTreeMap<Timestamp, OHLCCents>> {

39
crates/brk_fetcher/src/lib.rs
View File

@@ -23,16 +23,16 @@ const TRIES: usize = 12 * 60;
#[derive(Clone)]
pub struct Fetcher {
binance: Binance,
kraken: Kraken,
binance: Option<Binance>,
kraken: Option<Kraken>,
brk: BRK,
}
impl Fetcher {
pub fn import(hars_path: Option<&Path>) -> Result<Self> {
pub fn import(exchanges: bool, hars_path: Option<&Path>) -> Result<Self> {
Ok(Self {
binance: Binance::init(hars_path),
kraken: Kraken::default(),
binance: exchanges.then(|| Binance::init(hars_path)),
kraken: exchanges.then(Kraken::default),
brk: BRK::default(),
})
}
@@ -43,10 +43,17 @@ impl Fetcher {
fn get_date_(&mut self, date: Date, tries: usize) -> Result<OHLCCents> {
self.kraken
.get_from_1d(&date)
.as_mut()
.map_or(Err(Error::Str("Kraken off")), |kraken| {
kraken.get_from_1d(&date)
})
.or_else(|_| {
// eprintln!("{e}");
self.binance.get_from_1d(&date)
self.binance
.as_mut()
.map_or(Err(Error::Str("Binance off")), |binance| {
binance.get_from_1d(&date)
})
})
.or_else(|_| {
// eprintln!("{e}");
@@ -93,11 +100,17 @@ impl Fetcher {
let ohlc = self
.kraken
.get_from_1mn(timestamp, previous_timestamp)
.as_mut()
.map_or(Err(Error::Str("Kraken off")), |kraken| {
kraken.get_from_1mn(timestamp, previous_timestamp)
})
.unwrap_or_else(|_report| {
// eprintln!("{_report}");
self.binance
.get_from_1mn(timestamp, previous_timestamp)
.as_mut()
.map_or(Err(Error::Str("Binance off")), |binance| {
binance.get_from_1mn(timestamp, previous_timestamp)
})
.unwrap_or_else(|_report| {
// // eprintln!("{_report}");
self.brk.get_from_height(height).unwrap_or_else(|_report| {
@@ -187,8 +200,12 @@ How to fix this:
}
pub fn clear(&mut self) {
self.kraken.clear();
self.binance.clear();
if let Some(kraken) = self.kraken.as_mut() {
kraken.clear()
}
if let Some(binance) = self.binance.as_mut() {
binance.clear()
}
self.brk.clear();
}
}

9
crates/brk_fetcher/src/retry.rs
View File

@@ -3,7 +3,14 @@ use std::{fmt::Debug, thread::sleep, time::Duration};
use brk_error::Result;
use log::info;
pub fn retry<T>(function: impl Fn(usize) -> Result<T>, sleep_in_s: u64, retries: usize) -> Result<T>
pub fn default_retry<T>(function: impl Fn(usize) -> Result<T>) -> Result<T>
where
T: Debug,
{
retry(function, 5, 6)
}
fn retry<T>(function: impl Fn(usize) -> Result<T>, sleep_in_s: u64, retries: usize) -> Result<T>
where
T: Debug,
{

2
crates/brk_indexer/Cargo.toml
View File

@@ -16,7 +16,7 @@ brk_error = { workspace = true }
brk_logger = { workspace = true }
brk_parser = { workspace = true }
brk_store = { workspace = true }
brk_vecs = { workspace = true }
vecdb = { workspace = true }
fjall = { workspace = true }
log = { workspace = true }
rayon = { workspace = true }

2
crates/brk_indexer/examples/indexer.rs
View File

@@ -8,7 +8,7 @@ use std::{
use brk_error::Result;
use brk_indexer::Indexer;
use brk_parser::Parser;
use brk_vecs::Exit;
use vecdb::Exit;
fn main() -> Result<()> {
brk_logger::init(Some(Path::new(".log")));

2
crates/brk_indexer/src/indexes.rs
View File

@@ -7,7 +7,7 @@ use brk_structs::{
P2PKHAddressIndex, P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex,
TxIndex, TypeIndex, UnknownOutputIndex,
};
use brk_vecs::{
use vecdb::{
AnyIterableVec, AnyStoredIterableVec, GenericStoredVec, StoredIndex, StoredRaw, VecIterator,
};

30
crates/brk_indexer/src/lib.rs
View File

@@ -3,7 +3,7 @@
#![doc = include_str!("../examples/indexer.rs")]
#![doc = "```"]
use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc, thread, time::Instant};
use std::{collections::BTreeMap, path::Path, str::FromStr, thread, time::Instant};
use bitcoin::{Transaction, TxIn, TxOut};
use brk_error::{Error, Result};
@@ -15,9 +15,9 @@ use brk_structs::{
OutputType, Sats, StoredBool, Timestamp, TxIndex, Txid, TxidPrefix, TypeIndex,
TypeIndexWithOutputindex, Unit, Version, Vin, Vout,
};
use brk_vecs::{AnyVec, Exit, File, GenericStoredVec, PAGE_SIZE, Reader, VecIterator};
use log::{error, info};
use rayon::prelude::*;
use vecdb::{AnyVec, Database, Exit, GenericStoredVec, PAGE_SIZE, Reader, VecIterator};
mod indexes;
mod stores;
mod vecs;
@@ -27,23 +27,23 @@ pub use stores::*;
pub use vecs::*;
const SNAPSHOT_BLOCK_RANGE: usize = 1_000;
const COLLISIONS_CHECKED_UP_TO: Height = Height::new(908_700);
const COLLISIONS_CHECKED_UP_TO: Height = Height::new(909_150);
const VERSION: Version = Version::ONE;
#[derive(Clone)]
pub struct Indexer {
pub file: Arc<File>,
pub db: Database,
pub vecs: Vecs,
pub stores: Stores,
}
impl Indexer {
pub fn forced_import(outputs_dir: &Path) -> Result<Self> {
let file = Arc::new(File::open(&outputs_dir.join("indexed/vecs"))?);
let db = Database::open(&outputs_dir.join("indexed/vecs"))?;
let vecs = Vecs::forced_import(&file, VERSION + Version::ZERO)?;
let vecs = Vecs::forced_import(&db, VERSION + Version::ZERO)?;
file.set_min_len(PAGE_SIZE * 50_000_000)?;
db.set_min_len(PAGE_SIZE * 50_000_000)?;
Ok(Self {
vecs,
@@ -51,7 +51,7 @@ impl Indexer {
&outputs_dir.join("indexed/stores"),
VERSION + Version::ZERO,
)?,
file,
db,
})
}
@@ -62,10 +62,10 @@ impl Indexer {
exit: &Exit,
check_collisions: bool,
) -> Result<Indexes> {
let file = self.file.clone();
let db = self.db.clone();
// dbg!(self.file.regions().id_to_index());
// dbg!(self.file.layout());
// dbg!(self.db.regions().id_to_index());
// dbg!(self.db.layout());
let starting_indexes = Indexes::try_from((&mut self.vecs, &self.stores, rpc))
.unwrap_or_else(|_report| Indexes::default());
@@ -111,8 +111,8 @@ impl Indexer {
vecs.flush(height)?;
info!("Flushed vecs in {}s", i.elapsed().as_secs());
let i = Instant::now();
file.flush()?;
info!("Flushed file in {}s", i.elapsed().as_secs());
db.flush()?;
info!("Flushed db in {}s", i.elapsed().as_secs());
Ok(())
};
@@ -799,8 +799,8 @@ impl Indexer {
}
let i = Instant::now();
file.punch_holes()?;
info!("Punched holes in file in {}s", i.elapsed().as_secs());
db.punch_holes()?;
info!("Punched holes in db in {}s", i.elapsed().as_secs());
Ok(starting_indexes)
}

2
crates/brk_indexer/src/stores.rs
View File

@@ -6,9 +6,9 @@ use brk_structs::{
AddressBytes, AddressBytesHash, BlockHashPrefix, ByAddressType, Height, OutputIndex,
OutputType, TxIndex, TxidPrefix, TypeIndex, TypeIndexWithOutputindex, Unit, Version,
};
use brk_vecs::VecIterator;
use fjall::{PersistMode, TransactionalKeyspace};
use rayon::prelude::*;
use vecdb::VecIterator;
use crate::Indexes;

102
crates/brk_indexer/src/vecs.rs
View File

@@ -1,5 +1,3 @@
use std::sync::Arc;
use brk_error::Result;
use brk_structs::{
AddressBytes, BlockHash, EmptyOutputIndex, Height, InputIndex, OpReturnIndex, OutputIndex,
@@ -9,10 +7,10 @@ use brk_structs::{
RawLockTime, Sats, StoredBool, StoredF64, StoredU32, StoredU64, Timestamp, TxIndex, TxVersion,
Txid, TypeIndex, UnknownOutputIndex, Version, Weight,
};
use brk_vecs::{
AnyCollectableVec, AnyStoredVec, CompressedVec, File, GenericStoredVec, RawVec, Stamp,
};
use rayon::prelude::*;
use vecdb::{
AnyCollectableVec, AnyStoredVec, CompressedVec, Database, GenericStoredVec, RawVec, Stamp,
};
use crate::Indexes;
@@ -69,225 +67,221 @@ pub struct Vecs {
}
impl Vecs {
pub fn forced_import(file: &Arc<File>, version: Version) -> Result<Self> {
pub fn forced_import(db: &Database, version: Version) -> Result<Self> {
Ok(Self {
emptyoutputindex_to_txindex: CompressedVec::forced_import(
file,
db,
"txindex",
version + VERSION + Version::ZERO,
)?,
height_to_blockhash: RawVec::forced_import(
file,
db,
"blockhash",
version + VERSION + Version::ZERO,
)?,
height_to_difficulty: CompressedVec::forced_import(
file,
db,
"difficulty",
version + VERSION + Version::ZERO,
)?,
height_to_first_emptyoutputindex: CompressedVec::forced_import(
file,
db,
"first_emptyoutputindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_inputindex: CompressedVec::forced_import(
file,
db,
"first_inputindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_opreturnindex: CompressedVec::forced_import(
file,
db,
"first_opreturnindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_outputindex: CompressedVec::forced_import(
file,
db,
"first_outputindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_p2aaddressindex: CompressedVec::forced_import(
file,
db,
"first_p2aaddressindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_p2msoutputindex: CompressedVec::forced_import(
file,
db,
"first_p2msoutputindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_p2pk33addressindex: CompressedVec::forced_import(
file,
db,
"first_p2pk33addressindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_p2pk65addressindex: CompressedVec::forced_import(
file,
db,
"first_p2pk65addressindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_p2pkhaddressindex: CompressedVec::forced_import(
file,
db,
"first_p2pkhaddressindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_p2shaddressindex: CompressedVec::forced_import(
file,
db,
"first_p2shaddressindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_p2traddressindex: CompressedVec::forced_import(
file,
db,
"first_p2traddressindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_p2wpkhaddressindex: CompressedVec::forced_import(
file,
db,
"first_p2wpkhaddressindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_p2wshaddressindex: CompressedVec::forced_import(
file,
db,
"first_p2wshaddressindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_txindex: CompressedVec::forced_import(
file,
db,
"first_txindex",
version + VERSION + Version::ZERO,
)?,
height_to_first_unknownoutputindex: CompressedVec::forced_import(
file,
db,
"first_unknownoutputindex",
version + VERSION + Version::ZERO,
)?,
height_to_timestamp: CompressedVec::forced_import(
file,
db,
"timestamp",
version + VERSION + Version::ZERO,
)?,
height_to_total_size: CompressedVec::forced_import(
file,
db,
"total_size",
version + VERSION + Version::ZERO,
)?,
height_to_weight: CompressedVec::forced_import(
file,
db,
"weight",
version + VERSION + Version::ZERO,
)?,
inputindex_to_outputindex: RawVec::forced_import(
file,
db,
"outputindex",
version + VERSION + Version::ZERO,
)?,
opreturnindex_to_txindex: CompressedVec::forced_import(
file,
db,
"txindex",
version + VERSION + Version::ZERO,
)?,
outputindex_to_outputtype: RawVec::forced_import(
file,
db,
"outputtype",
version + VERSION + Version::ZERO,
)?,
outputindex_to_typeindex: RawVec::forced_import(
file,
db,
"typeindex",
version + VERSION + Version::ZERO,
)?,
outputindex_to_value: RawVec::forced_import(
file,
db,
"value",
version + VERSION + Version::ZERO,
)?,
p2aaddressindex_to_p2abytes: RawVec::forced_import(
file,
db,
"p2abytes",
version + VERSION + Version::ZERO,
)?,
p2msoutputindex_to_txindex: CompressedVec::forced_import(
file,
db,
"txindex",
version + VERSION + Version::ZERO,
)?,
p2pk33addressindex_to_p2pk33bytes: RawVec::forced_import(
file,
db,
"p2pk33bytes",
version + VERSION + Version::ZERO,
)?,
p2pk65addressindex_to_p2pk65bytes: RawVec::forced_import(
file,
db,
"p2pk65bytes",
version + VERSION + Version::ZERO,
)?,
p2pkhaddressindex_to_p2pkhbytes: RawVec::forced_import(
file,
db,
"p2pkhbytes",
version + VERSION + Version::ZERO,
)?,
p2shaddressindex_to_p2shbytes: RawVec::forced_import(
file,
db,
"p2shbytes",
version + VERSION + Version::ZERO,
)?,
p2traddressindex_to_p2trbytes: RawVec::forced_import(
file,
db,
"p2trbytes",
version + VERSION + Version::ZERO,
)?,
p2wpkhaddressindex_to_p2wpkhbytes: RawVec::forced_import(
file,
db,
"p2wpkhbytes",
version + VERSION + Version::ZERO,
)?,
p2wshaddressindex_to_p2wshbytes: RawVec::forced_import(
file,
db,
"p2wshbytes",
version + VERSION + Version::ZERO,
)?,
txindex_to_base_size: CompressedVec::forced_import(
file,
db,
"base_size",
version + VERSION + Version::ZERO,
)?,
txindex_to_first_inputindex: CompressedVec::forced_import(
file,
db,
"first_inputindex",
version + VERSION + Version::ZERO,
)?,
txindex_to_first_outputindex: CompressedVec::forced_import(
file,
db,
"first_outputindex",
version + VERSION + Version::ZERO,
)?,
txindex_to_is_explicitly_rbf: CompressedVec::forced_import(
file,
db,
"is_explicitly_rbf",
version + VERSION + Version::ZERO,
)?,
txindex_to_rawlocktime: CompressedVec::forced_import(
file,
db,
"rawlocktime",
version + VERSION + Version::ZERO,
)?,
txindex_to_total_size: CompressedVec::forced_import(
file,
db,
"total_size",
version + VERSION + Version::ZERO,
)?,
txindex_to_txid: RawVec::forced_import(
file,
"txid",
version + VERSION + Version::ZERO,
)?,
txindex_to_txid: RawVec::forced_import(db, "txid", version + VERSION + Version::ZERO)?,
txindex_to_txversion: CompressedVec::forced_import(
file,
db,
"txversion",
version + VERSION + Version::ZERO,
)?,
unknownoutputindex_to_txindex: CompressedVec::forced_import(
file,
db,
"txindex",
version + VERSION + Version::ZERO,
)?,

2
crates/brk_interface/Cargo.toml
View File

@@ -13,7 +13,7 @@ brk_computer = { workspace = true }
brk_error = { workspace = true }
brk_indexer = { workspace = true }
brk_structs = { workspace = true }
brk_vecs = { workspace = true }
vecdb = { workspace = true }
derive_deref = { workspace = true }
schemars = "1.0.4"
serde = { workspace = true }

2
crates/brk_interface/src/lib.rs
View File

@@ -9,8 +9,8 @@ use brk_computer::Computer;
use brk_error::Result;
use brk_indexer::Indexer;
use brk_structs::Height;
use brk_vecs::{AnyCollectableVec, AnyStoredVec};
use tabled::settings::Style;
use vecdb::{AnyCollectableVec, AnyStoredVec};
mod deser;
mod format;

2
crates/brk_interface/src/vecs.rs
View File

@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
use brk_computer::Computer;
use brk_indexer::Indexer;
use brk_vecs::AnyCollectableVec;
use vecdb::AnyCollectableVec;
use derive_deref::{Deref, DerefMut};
use crate::pagination::{PaginatedIndexParam, PaginationParam};

2
crates/brk_mcp/Cargo.toml
View File

@@ -12,7 +12,7 @@ build = "build.rs"
axum = { workspace = true }
brk_interface = { workspace = true }
log = { workspace = true }
brk_rmcp = { version = "0.4.1", features = [
brk_rmcp = { version = "0.5.0", features = [
"transport-worker",
"transport-streamable-http-server",
] }

475
crates/brk_parser/Cargo.lock generated
View File

@@ -1,475 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "arrayvec"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "base58ck"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c8d66485a3a2ea485c1913c4572ce0256067a5377ac8c75c4960e1cda98605f"
dependencies = [
"bitcoin-internals",
"bitcoin_hashes",
]
[[package]]
name = "base64"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "bech32"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d"
[[package]]
name = "bitcoin"
version = "0.32.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce6bc65742dea50536e35ad42492b234c27904a27f0abdcbce605015cb4ea026"
dependencies = [
"base58ck",
"bech32",
"bitcoin-internals",
"bitcoin-io",
"bitcoin-units",
"bitcoin_hashes",
"hex-conservative",
"hex_lit",
"secp256k1",
"serde",
]
[[package]]
name = "bitcoin-internals"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30bdbe14aa07b06e6cfeffc529a1f099e5fbe249524f8125358604df99a4bed2"
dependencies = [
"serde",
]
[[package]]
name = "bitcoin-io"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "340e09e8399c7bd8912f495af6aa58bea0c9214773417ffaa8f6460f93aaee56"
[[package]]
name = "bitcoin-units"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5285c8bcaa25876d07f37e3d30c303f2609179716e11d688f51e8f1fe70063e2"
dependencies = [
"bitcoin-internals",
"serde",
]
[[package]]
name = "bitcoin_hashes"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16"
dependencies = [
"bitcoin-io",
"hex-conservative",
"serde",
]
[[package]]
name = "bitcoincore-rpc"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aedd23ae0fd321affb4bbbc36126c6f49a32818dc6b979395d24da8c9d4e80ee"
dependencies = [
"bitcoincore-rpc-json",
"jsonrpc",
"log",
"serde",
"serde_json",
]
[[package]]
name = "bitcoincore-rpc-json"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8909583c5fab98508e80ef73e5592a651c954993dc6b7739963257d19f0e71a"
dependencies = [
"bitcoin",
"serde",
"serde_json",
]
[[package]]
name = "biter"
version = "0.2.2"
dependencies = [
"bitcoin",
"bitcoincore-rpc",
"crossbeam",
"derived-deref",
"rayon",
"serde",
"serde_json",
]
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "cc"
version = "1.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f"
dependencies = [
"shlex",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "crossbeam"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-epoch",
"crossbeam-queue",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
[[package]]
name = "derived-deref"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "805ef2023ccd65425743a91ecd11fc020979a0b01921db3104fb606d18a7b43e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "either"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
[[package]]
name = "getrandom"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "hex-conservative"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5313b072ce3c597065a808dbf612c4c8e8590bdbf8b579508bf7a762c5eae6cd"
dependencies = [
"arrayvec",
]
[[package]]
name = "hex_lit"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd"
[[package]]
name = "itoa"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
[[package]]
name = "jsonrpc"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3662a38d341d77efecb73caf01420cfa5aa63c0253fd7bc05289ef9f6616e1bf"
dependencies = [
"base64",
"minreq",
"serde",
"serde_json",
]
[[package]]
name = "libc"
version = "0.2.161"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1"
[[package]]
name = "log"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
[[package]]
name = "memchr"
version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "minreq"
version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "763d142cdff44aaadd9268bebddb156ef6c65a0e13486bb81673cf2d8739f9b0"
dependencies = [
"log",
"serde",
"serde_json",
]
[[package]]
name = "ppv-lite86"
version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
dependencies = [
"zerocopy",
]
[[package]]
name = "proc-macro2"
version = "1.0.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "rayon"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "ryu"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "secp256k1"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113"
dependencies = [
"bitcoin_hashes",
"rand",
"secp256k1-sys",
"serde",
]
[[package]]
name = "secp256k1-sys"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9"
dependencies = [
"cc",
]
[[package]]
name = "serde"
version = "1.0.217"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.217"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.135"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9"
dependencies = [
"itoa",
"memchr",
"ryu",
"serde",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "syn"
version = "2.0.96"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "unicode-ident"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "zerocopy"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
dependencies = [
"byteorder",
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]

2
crates/brk_server/Cargo.toml
View File

@@ -19,7 +19,7 @@ brk_interface = { workspace = true }
brk_logger = { workspace = true }
brk_mcp = { workspace = true }
brk_parser = { workspace = true }
brk_vecs = { workspace = true }
vecdb = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
quick_cache = "0.6.16"

4
crates/brk_server/examples/main.rs
View File

@@ -9,7 +9,7 @@ use brk_indexer::Indexer;
use brk_interface::Interface;
use brk_parser::Parser;
use brk_server::Server;
use brk_vecs::Exit;
use vecdb::Exit;
pub fn main() -> Result<()> {
brk_logger::init(Some(Path::new(".log")));
@@ -32,7 +32,7 @@ pub fn main() -> Result<()> {
let mut indexer = Indexer::forced_import(outputs_dir)?;
let fetcher = Some(Fetcher::import(None)?);
let fetcher = Some(Fetcher::import(true, None)?);
let mut computer = Computer::forced_import(outputs_dir, &indexer, fetcher)?;

3
crates/brk_server/src/api/interface.rs
View File

@@ -9,8 +9,8 @@ use axum::{
};
use brk_error::{Error, Result};
use brk_interface::{Format, Output, Params};
use brk_vecs::Stamp;
use quick_cache::sync::GuardResult;
use vecdb::Stamp;
use crate::{HeaderMapExtended, ResponseExtended};
@@ -112,6 +112,7 @@ fn req_to_response_res(
headers.insert_cors();
headers.insert_etag(&etag);
headers.insert_cache_control_must_revalidate();
match format {
Some(format) => {

2
crates/brk_server/src/extended/header_map.rs
View File

@@ -59,7 +59,7 @@ impl HeaderMapExtended for HeaderMap {
fn insert_cache_control_must_revalidate(&mut self) {
self.insert(
header::CACHE_CONTROL,
"public, max-age=0, must-revalidate".parse().unwrap(),
"public, max-age=1, must-revalidate".parse().unwrap(),
);
}

4
crates/brk_structs/Cargo.toml
View File

@@ -12,11 +12,11 @@ build = "build.rs"
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_error = {workspace = true}
brk_vecs = {workspace = true}
vecdb = {workspace = true}
byteview = { workspace = true }
derive_deref = { workspace = true }
jiff = { workspace = true }
rapidhash = "3.0.0"
rapidhash = "3.1.0"
serde = { workspace = true }
serde_bytes = { workspace = true }
zerocopy = { workspace = true }

2
crates/brk_structs/src/lib.rs
View File

@@ -6,4 +6,4 @@ mod structs;
pub use groups::*;
pub use structs::*;
pub use brk_vecs::{CheckedSub, Exit, Printable, Version};
pub use vecdb::{CheckedSub, Exit, Printable, Version};

2
crates/brk_structs/src/structs/bitcoin.rs
View File

@@ -3,8 +3,8 @@ use std::{
ops::{Add, AddAssign, Div, Mul},
};
use brk_vecs::{CheckedSub, StoredCompressed};
use serde::Serialize;
use vecdb::{CheckedSub, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::{Sats, StoredF64};

2
crates/brk_structs/src/structs/cents.rs
View File

@@ -1,7 +1,7 @@
use std::ops::{Add, Div, Mul};
use brk_vecs::{CheckedSub, StoredCompressed};
use serde::Serialize;
use vecdb::{CheckedSub, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Dollars;

2
crates/brk_structs/src/structs/date.rs
View File

@@ -1,6 +1,6 @@
use brk_vecs::StoredCompressed;
use jiff::{Span, civil::Date as Date_, tz::TimeZone};
use serde::{Serialize, Serializer};
use vecdb::StoredCompressed;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::{DateIndex, Timestamp};

2
crates/brk_structs/src/structs/dateindex.rs
View File

@@ -4,7 +4,7 @@ use std::{
};
use brk_error::Error;
use brk_vecs::{CheckedSub, FromCoarserIndex, Printable, StoredCompressed};
use vecdb::{CheckedSub, FromCoarserIndex, Printable, StoredCompressed};
use jiff::Span;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

2
crates/brk_structs/src/structs/decadeindex.rs
View File

@@ -3,7 +3,7 @@ use std::{
ops::{Add, AddAssign, Div},
};
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use vecdb::{CheckedSub, Printable, StoredCompressed};
use serde::{Deserialize, Serialize};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

2
crates/brk_structs/src/structs/difficultyepoch.rs
View File

@@ -3,8 +3,8 @@ use std::{
ops::{Add, AddAssign, Div},
};
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use serde::{Deserialize, Serialize};
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Height;

2
crates/brk_structs/src/structs/dollars.rs
View File

@@ -4,9 +4,9 @@ use std::{
ops::{Add, AddAssign, Div, Mul},
};
use brk_vecs::{CheckedSub, StoredCompressed};
use derive_deref::Deref;
use serde::{Deserialize, Serialize};
use vecdb::{CheckedSub, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::{Bitcoin, Cents, Close, High, Sats, StoredF32, StoredF64};

2
crates/brk_structs/src/structs/emptyaddressindex.rs
View File

@@ -1,6 +1,6 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable};
use vecdb::{CheckedSub, Printable};
use derive_deref::Deref;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

2
crates/brk_structs/src/structs/emptyoutputindex.rs
View File

@@ -1,8 +1,8 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::TypeIndex;

2
crates/brk_structs/src/structs/feerate.rs
View File

@@ -3,8 +3,8 @@ use std::{
ops::{Add, AddAssign, Div},
};
use brk_vecs::StoredCompressed;
use serde::Serialize;
use vecdb::StoredCompressed;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::{Sats, StoredU64};

2
crates/brk_structs/src/structs/halvingepoch.rs
View File

@@ -3,8 +3,8 @@ use std::{
ops::{Add, AddAssign, Div},
};
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use serde::{Deserialize, Serialize};
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Height;

2
crates/brk_structs/src/structs/height.rs
View File

@@ -4,10 +4,10 @@ use std::{
};
use bitcoincore_rpc::{Client, RpcApi};
use brk_vecs::{CheckedSub, Printable, Stamp, StoredCompressed};
use byteview::ByteView;
use derive_deref::Deref;
use serde::{Deserialize, Serialize};
use vecdb::{CheckedSub, Printable, Stamp, StoredCompressed};
use zerocopy::{FromBytes, IntoBytes};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

2
crates/brk_structs/src/structs/inputindex.rs
View File

@@ -1,8 +1,8 @@
use std::ops::{Add, AddAssign};
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::Vin;

2
crates/brk_structs/src/structs/loadedaddressdata.rs
View File

@@ -1,6 +1,6 @@
use brk_error::{Error, Result};
use brk_vecs::CheckedSub;
use serde::Serialize;
use vecdb::CheckedSub;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::{Bitcoin, Dollars, EmptyAddressData, Sats};

2
crates/brk_structs/src/structs/loadedaddressindex.rs
View File

@@ -1,8 +1,8 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable};
use derive_deref::Deref;
use serde::Serialize;
use vecdb::{CheckedSub, Printable};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::TypeIndex;

2
crates/brk_structs/src/structs/monthindex.rs
View File

@@ -3,7 +3,7 @@ use std::{
ops::{Add, AddAssign, Div},
};
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use vecdb::{CheckedSub, Printable, StoredCompressed};
use serde::{Deserialize, Serialize};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

2
crates/brk_structs/src/structs/ohlc.rs
View File

@@ -1,8 +1,8 @@
use std::ops::{Add, AddAssign, Div};
use brk_vecs::StoredCompressed;
use derive_deref::{Deref, DerefMut};
use serde::{Serialize, Serializer, ser::SerializeTuple};
use vecdb::StoredCompressed;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::{Cents, Dollars, Sats};

2
crates/brk_structs/src/structs/opreturnindex.rs
View File

@@ -1,8 +1,8 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::TypeIndex;

2
crates/brk_structs/src/structs/outputindex.rs
View File

@@ -1,8 +1,8 @@
use std::ops::{Add, AddAssign};
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::copy_first_8bytes;

2
crates/brk_structs/src/structs/p2aaddressindex.rs
View File

@@ -1,8 +1,8 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::TypeIndex;

2
crates/brk_structs/src/structs/p2msoutputindex.rs
View File

@@ -1,8 +1,8 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::TypeIndex;

2
crates/brk_structs/src/structs/p2pk33addressindex.rs
View File

@@ -1,8 +1,8 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::TypeIndex;

2
crates/brk_structs/src/structs/p2pk65addressindex.rs
View File

@@ -1,8 +1,8 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::TypeIndex;

2
crates/brk_structs/src/structs/p2pkhaddressindex.rs
View File

@@ -1,8 +1,8 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::TypeIndex;

2
crates/brk_structs/src/structs/p2shaddressindex.rs
View File

@@ -1,6 +1,6 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use vecdb::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

2
crates/brk_structs/src/structs/p2traddressindex.rs
View File

@@ -1,8 +1,8 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::TypeIndex;

2
crates/brk_structs/src/structs/p2wpkhaddressindex.rs
View File

@@ -1,8 +1,8 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use crate::TypeIndex;

2
crates/brk_structs/src/structs/p2wshaddressindex.rs
View File

@@ -1,6 +1,6 @@
use std::ops::Add;
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use vecdb::{CheckedSub, Printable, StoredCompressed};
use derive_deref::{Deref, DerefMut};
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};

2
crates/brk_structs/src/structs/quarterindex.rs
View File

@@ -3,8 +3,8 @@ use std::{
ops::{Add, AddAssign, Div},
};
use brk_vecs::{CheckedSub, Printable, StoredCompressed};
use serde::{Deserialize, Serialize};
use vecdb::{CheckedSub, Printable, StoredCompressed};
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
use super::MonthIndex;

Some files were not shown because too many files have changed in this diff Show More