Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-05-12 10:38:36 -07:00
Code Issues Packages Projects Releases Wiki Activity

kibo: part 2 (broken)

Browse Source
This commit is contained in:
nym21
2025-03-25 20:55:47 +01:00
parent 7a8896864f
commit 50bf670931
126 changed files with 7131 additions and 2758 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
crates/brk_cli/README.md
View File

@@ -17,9 +17,15 @@
<a href="https://deps.rs/crate/brk_cli">
<img src="https://deps.rs/crate/brk_cli/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/EScZAYX4">
<a href="https://discord.gg/Cvrwpv3zEG">
<img src="https://img.shields.io/discord/1350431684562124850" alt="Chat" />
</a>
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
</p>
A command line interface to interact with the full Bitcoin Research Kit. It's built on top of every other create and gives the possility to use BRK using the terminal instead of Rust.

8
crates/brk_cli/src/run.rs
View File

@@ -50,9 +50,13 @@ pub fn run(config: RunConfig) -> color_eyre::Result<()> {
let server = Server::new(served_indexer, served_computer, config.website())?;
Some(tokio::spawn(async move {
let opt = Some(tokio::spawn(async move {
server.serve().await.unwrap();
}))
}));
sleep(Duration::from_secs(1));
opt
} else {
None
};

8
crates/brk_computer/README.md
View File

@@ -17,9 +17,15 @@
<a href="https://deps.rs/crate/brk_computer">
<img src="https://deps.rs/crate/brk_computer/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/EScZAYX4">
<a href="https://discord.gg/Cvrwpv3zEG">
<img src="https://img.shields.io/discord/1350431684562124850" alt="Chat" />
</a>
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
</p>
A dataset computer, built on top of `brk_indexer` and `brk_fetcher`. It computes any dataset you can think of and if it doesn't feel free to create an issue.

2
crates/brk_computer/src/storage/vecs/stats/from_height.rs
View File

@@ -18,7 +18,7 @@ where
pub difficultyepoch: StorableVecBuilder<Difficultyepoch, T>,
pub monthindex: StorableVecBuilder<Monthindex, T>,
pub yearindex: StorableVecBuilder<Yearindex, T>,
// pub halvingepoch: StorableVecGeneator<Halvingepoch, T>, // TODO
// TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
pub decadeindex: StorableVecBuilder<Decadeindex, T>,
}

2
crates/brk_computer/src/storage/vecs/stats/from_height_strict.rs
View File

@@ -14,7 +14,7 @@ where
T: ComputedType + PartialOrd,
{
pub difficultyepoch: StorableVecBuilder<Difficultyepoch, T>,
// pub halvingepoch: StorableVecGeneator<Halvingepoch, T>, // TODO
// TODO: pub halvingepoch: StorableVecGeneator<Halvingepoch, T>,
}
impl<T> StorableVecsStatsFromHeightStrict<T>

8
crates/brk_core/README.md
View File

@@ -17,9 +17,15 @@
<a href="https://deps.rs/crate/brk_core">
<img src="https://deps.rs/crate/brk_core/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/EScZAYX4">
<a href="https://discord.gg/Cvrwpv3zEG">
<img src="https://img.shields.io/discord/1350431684562124850" alt="Chat" />
</a>
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
</p>
A list of structs that are used throughout the project as units, think of `Date`, `Height`, `Sats`, `Txindex` or anything that can be either a key and/or a value of a dataset.

8
crates/brk_exit/README.md
View File

@@ -17,9 +17,15 @@
<a href="https://deps.rs/crate/brk_exit">
<img src="https://deps.rs/crate/brk_exit/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/EScZAYX4">
<a href="https://discord.gg/Cvrwpv3zEG">
<img src="https://img.shields.io/discord/1350431684562124850" alt="Chat" />
</a>
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
</p>
A simple crate that stops the program from exitting when blocking is activated until it is released. The purpose of that is to prevent exitting when a program is in the middle of saving data and thus prevent partial writes.

8
crates/brk_fetcher/README.md
View File

@@ -17,9 +17,15 @@
<a href="https://deps.rs/crate/brk_fetcher">
<img src="https://deps.rs/crate/brk_fetcher/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/EScZAYX4">
<a href="https://discord.gg/Cvrwpv3zEG">
<img src="https://img.shields.io/discord/1350431684562124850" alt="Chat" />
</a>
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
</p>
A crate that can fetch the Bitcoin price, either by date or height, from multiple APIs such Kraken, Binance and Kibo.money.

26
crates/brk_fetcher/src/lib.rs
View File

@@ -64,18 +64,20 @@ impl Fetcher {
let date = Date::from(timestamp);
panic!(
"Can't find the price for: height: {height} - date: {date}
1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's
How to fix this:
1. Go to https://www.binance.com/en/trade/BTC_USDT?type=spot
2. Select 1mn interval
3. Open the inspector/dev tools
4. Go to the Network Tab
5. Filter URLs by 'uiKlines'
6. Go back to the chart and scroll until you pass the date mentioned few lines ago
7. Go back to the dev tools
8. Export to a har file (if there is no explicit button, click on the cog button)
9. Move the file to 'parser/imports/binance.har'
"
Can't find the price for: height: {height} - date: {date}
1mn APIs are limited to the last 16 hours for Binance's and the last 10 hours for Kraken's
How to fix this:
0. Try rerunning the program first, it usually fixes the problem
1. If it didn't, go to https://www.binance.com/en/trade/BTC_USDT?type=spot
2. Select 1mn interval
3. Open the inspector/dev tools
4. Go to the Network Tab
5. Filter URLs by 'uiKlines'
6. Go back to the chart and scroll until you pass the date mentioned few lines ago
7. Go back to the dev tools
8. Export to a har file (if there is no explicit button, click on the cog button)
9. Move the file to 'parser/imports/binance.har'
"
)
})

8
crates/brk_indexer/README.md
View File

@@ -17,9 +17,15 @@
<a href="https://deps.rs/crate/brk_indexer">
<img src="https://deps.rs/crate/brk_indexer/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/EScZAYX4">
<a href="https://discord.gg/Cvrwpv3zEG">
<img src="https://img.shields.io/discord/1350431684562124850" alt="Chat" />
</a>
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
</p>
A [Bitcoin Core](https://bitcoincore.org/en/about/) node indexer which iterates over the chain (via `../brk_parser`) and creates a database of the vecs (`brk_vec`) and key/value stores ([`fjall`](https://crates.io/crates/fjall)) that can be used in your Rust code.

45
crates/brk_indexer/src/stores/base.rs
View File

@@ -10,7 +10,8 @@ use brk_core::Height;
use brk_vec::{Value, Version};
use byteview::ByteView;
use fjall::{
PartitionCreateOptions, PersistMode, ReadTransaction, Result, TransactionalKeyspace, TransactionalPartitionHandle,
PartitionCreateOptions, PersistMode, ReadTransaction, Result, TransactionalKeyspace,
TransactionalPartitionHandle,
};
use zerocopy::{Immutable, IntoBytes};
@@ -71,7 +72,7 @@ where
if let Some(v) = self.puts.get(key) {
Ok(Some(Value::Ref(v)))
} else if let Some(slice) = self.rtx.get(&self.part, key.as_bytes())? {
Ok(Some(Value::Owned(V::try_from(slice.into())?)))
Ok(Some(Value::Owned(V::try_from(slice.as_bytes().into())?)))
} else {
Ok(None)
}
@@ -109,23 +110,29 @@ where
mem::take(&mut self.dels)
.into_iter()
.for_each(|key| wtx.remove(&self.part, key.into()));
.for_each(|key| wtx.remove(&self.part, key.as_bytes()));
mem::take(&mut self.puts).into_iter().for_each(|(key, value)| {
if CHECK_COLLISISONS {
#[allow(unused_must_use)]
if let Ok(Some(value)) = wtx.get(&self.part, key.as_bytes()) {
dbg!(
&key,
V::try_from(value.into()).unwrap(),
&self.meta,
self.rtx.get(&self.part, key.as_bytes())
);
unreachable!();
mem::take(&mut self.puts)
.into_iter()
.for_each(|(key, value)| {
if CHECK_COLLISISONS {
#[allow(unused_must_use)]
if let Ok(Some(value)) = wtx.get(&self.part, key.as_bytes()) {
dbg!(
&key,
V::try_from(value.as_bytes().into()).unwrap(),
&self.meta,
self.rtx.get(&self.part, key.as_bytes())
);
unreachable!();
}
}
}
wtx.insert(&self.part, key.into(), value.into())
});
wtx.insert(
&self.part,
key.as_bytes(),
&*ByteView::try_from(value).unwrap(),
)
});
wtx.commit()?;
@@ -164,7 +171,9 @@ where
.open_transactional()
}
fn open_partition_handle(keyspace: &TransactionalKeyspace) -> Result<TransactionalPartitionHandle> {
fn open_partition_handle(
keyspace: &TransactionalKeyspace,
) -> Result<TransactionalPartitionHandle> {
keyspace.open_partition(
"partition",
PartitionCreateOptions::default()

8
crates/brk_logger/README.md
View File

@@ -17,9 +17,15 @@
<a href="https://deps.rs/crate/brk_logger">
<img src="https://deps.rs/crate/brk_logger/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/EScZAYX4">
<a href="https://discord.gg/Cvrwpv3zEG">
<img src="https://img.shields.io/discord/1350431684562124850" alt="Chat" />
</a>
<a href="https://primal.net/p/nprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6">
<img src="https://img.shields.io/badge/nostr-purple?link=https%3A%2F%2Fprimal.net%2Fp%2Fnprofile1qqsfw5dacngjlahye34krvgz7u0yghhjgk7gxzl5ptm9v6n2y3sn03sqxu2e6" alt="Nostr" />
</a>
<a href="https://bsky.app/profile/bitcoinresearchkit.org">
<img src="https://img.shields.io/badge/bluesky-blue?link=https%3A%2F%2Fbsky.app%2Fprofile%2Fbitcoinresearchkit.org" alt="Bluesky" />
</a>
</p>
A simple crate built on top of [`env_logger`](https://crates.io/crates/env_logger) to display logs from the [`log`](https://crates.io/crates/log) crate in a colorful and clean format.

2
crates/brk_parser/README.md
View File

@@ -17,7 +17,7 @@
<a href="https://deps.rs/crate/brk_parser">
<img src="https://deps.rs/crate/brk_parser/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/EScZAYX4">
<a href="https://discord.gg/Cvrwpv3zEG">
<img src="https://img.shields.io/discord/1350431684562124850" alt="Chat" />
</a>
</p>

2
crates/brk_query/README.md
View File

@@ -17,7 +17,7 @@
<a href="https://deps.rs/crate/brk_query">
<img src="https://deps.rs/crate/brk_query/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/EScZAYX4">
<a href="https://discord.gg/Cvrwpv3zEG">
<img src="https://img.shields.io/discord/1350431684562124850" alt="Chat" />
</a>
</p>

4
crates/brk_server/Cargo.toml
View File

@@ -21,8 +21,8 @@ color-eyre = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }
minreq = { workspace = true }
oxc = { version = "0.61.1", features = ["codegen", "minifier"] }
oxc = { version = "0.61.2", features = ["codegen", "minifier"] }
serde = { workspace = true }
tokio = { version = "1.44.1", features = ["full"] }
tower-http = { version = "0.6.2", features = ["compression-full"] }
zip = "2.4.2"
zip = "2.5.0"

2
crates/brk_server/README.md
View File

@@ -17,7 +17,7 @@
<a href="https://deps.rs/crate/brk_server">
<img src="https://deps.rs/crate/brk_server/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/EScZAYX4">
<a href="https://discord.gg/Cvrwpv3zEG">
<img src="https://img.shields.io/discord/1350431684562124850" alt="Chat" />
</a>
</p>

19
crates/brk_server/src/api/query/dts.rs
View File

@@ -30,14 +30,25 @@ impl DTS for Query<'static> {
let path = path.join(Path::new("vecid-to-indexes.d.ts"));
let mut contents = Index::all()
.into_iter()
let indexes = Index::all();
let mut contents = indexes
.iter()
.enumerate()
.map(|(i_of_i, i)| format!("type {} = {};", i, i_of_i))
.collect::<Vec<_>>()
.join("\n");
contents += "\n\ninterface VecIdToIndexes {\n";
contents += &format!(
"\n\nexport type Index = {};",
indexes
.iter()
.map(|i| i.to_string())
.collect::<Vec<_>>()
.join(" | ")
);
contents += "\n\nexport interface VecIdToIndexes {\n";
self.vecid_to_index_to_vec
.iter()
@@ -60,6 +71,8 @@ impl DTS for Query<'static> {
contents.push('}');
contents += "\n\nexport type VecId = keyof VecIdToIndexes;";
fs::write(path, contents)
}
}

4
crates/brk_server/src/lib.rs
View File

@@ -47,7 +47,6 @@ const DEV_PATH: &str = "../..";
const DOWNLOADS: &str = "downloads";
const WEBSITES: &str = "websites";
// TODO
pub struct Server(AppState);
impl Server {
@@ -69,8 +68,7 @@ impl Server {
if !fs::exists(&downloaded_websites_path)? {
info!("Downloading websites from Github...");
// TODO
// Need to download versioned, this is only for testing !
// TODO: Need to download versioned, main is only for testing
let url =
"https://github.com/bitcoinresearchkit/brk/archive/refs/heads/main.zip";

2
crates/brk_vec/README.md
View File

@@ -17,7 +17,7 @@
<a href="https://deps.rs/crate/brk_vec">
<img src="https://deps.rs/crate/brk_vec/latest/status.svg" alt="Dependency status">
</a>
<a href="https://discord.gg/EScZAYX4">
<a href="https://discord.gg/Cvrwpv3zEG">
<img src="https://img.shields.io/discord/1350431684562124850" alt="Chat" />
</a>
</p>