Mirror/brk
1
0
Fork 0
mirror of https://github.com/bitcoinresearchkit/brk.git synced 2026-04-29 17:19:57 -07:00
Code Issues Packages Projects Releases Wiki Activity

global: snapshot

Browse Source
This commit is contained in:
nym21
2025-09-14 23:13:18 +02:00
parent ce50b14591
commit 17dc4bde5e
13 changed files with 295 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
Cargo.lock generated
View File

@@ -1057,6 +1057,7 @@ name = "brk_server"
version = "0.0.107"
dependencies = [
"axum",
"bitcoin",
"bitcoincore-rpc",
"brk_computer",
"brk_error",
@@ -1066,6 +1067,7 @@ dependencies = [
"brk_logger",
"brk_mcp",
"brk_parser",
"brk_structs",
"jiff",
"log",
"quick_cache",
@@ -1085,6 +1087,7 @@ dependencies = [
"byteview",
"fjall",
"log",
"parking_lot 0.12.4",
]
[[package]]
@@ -2590,9 +2593,9 @@ checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"
[[package]]
name = "libredox"
version = "0.1.9"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3"
checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb"
dependencies = [
"bitflags 2.9.4",
"libc",
@@ -4122,12 +4125,12 @@ dependencies = [
[[package]]
name = "seize"
version = "0.5.0"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4b8d813387d566f627f3ea1b914c068aac94c40ae27ec43f5f33bde65abefe7"
checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.61.0",
]
[[package]]
@@ -4161,27 +4164,38 @@ dependencies = [
[[package]]
name = "serde"
version = "1.0.219"
version = "1.0.223"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
checksum = "a505d71960adde88e293da5cb5eda57093379f64e61cf77bf0e6a63af07a7bac"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]]
name = "serde_bytes"
version = "0.11.17"
version = "0.11.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96"
checksum = "fe07b5d88710e3b807c16a06ccbc9dfecd5fff6a4d2745c59e3e26774f10de6a"
dependencies = [
"serde",
"serde_core",
]
[[package]]
name = "serde_core"
version = "1.0.223"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20f57cbd357666aa7b3ac84a90b4ea328f1d4ddb6772b430caa5d9e1309bb9e9"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.219"
version = "1.0.223"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
checksum = "3d428d07faf17e306e699ec1e91996e5a165ba5d6bce5b5155173e91a8a01a56"
dependencies = [
"proc-macro2",
"quote",
@@ -4201,25 +4215,27 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.143"
version = "1.0.145"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a"
checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
dependencies = [
"indexmap 2.11.1",
"itoa",
"memchr",
"ryu",
"serde",
"serde_core",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.17"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a"
checksum = "a30a8abed938137c7183c173848e3c9b3517f5e038226849a4ecc9b21a4b4e2a"
dependencies = [
"itoa",
"serde",
"serde_core",
]
[[package]]

8
Cargo.toml
View File

@@ -68,10 +68,10 @@ minreq = { version = "2.14.1", features = ["https", "serde_json"] }
parking_lot = "0.12.4"
quick_cache = "0.6.16"
rayon = "1.11.0"
serde = "1.0.219"
serde_bytes = "0.11.17"
serde_derive = "1.0.219"
serde_json = { version = "1.0.143", features = ["float_roundtrip"] }
serde = "1.0.223"
serde_bytes = "0.11.18"
serde_derive = "1.0.223"
serde_json = { version = "1.0.145", features = ["float_roundtrip"] }
tokio = { version = "1.47.1", features = ["rt-multi-thread"] }
# vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]}
vecdb = { version = "0.2.14", features = ["derive"]}

18
crates/brk_computer/src/stateful/mod.rs
View File

@@ -1495,35 +1495,35 @@ impl Vecs {
return None;
}
let mmap = addresstypeindex_to_anyaddressindex_reader_opt
let reader = addresstypeindex_to_anyaddressindex_reader_opt
.get_unwrap(address_type)
.as_ref()
.unwrap();
let anyaddressindex = match address_type {
OutputType::P2PK33 => {
p2pk33addressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
p2pk33addressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
}
OutputType::P2PK65 => {
p2pk65addressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
p2pk65addressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
}
OutputType::P2PKH => {
p2pkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
p2pkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
}
OutputType::P2SH => {
p2shaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
p2shaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
}
OutputType::P2TR => {
p2traddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
p2traddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
}
OutputType::P2WPKH => {
p2wpkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
p2wpkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
}
OutputType::P2WSH => {
p2wshaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
p2wshaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
}
OutputType::P2A => {
p2aaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap)
p2aaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader)
}
_ => unreachable!(),
}

8
crates/brk_interface/src/lib.rs
View File

@@ -261,4 +261,12 @@ impl<'a> Interface<'a> {
pub fn get_vecid_to_indexes(&self, id: String) -> Option<&Vec<&'static str>> {
self.vecs.id_to_indexes(id)
}
pub fn indexer(&self) -> &Indexer {
self.indexer
}
pub fn computer(&self) -> &Computer {
self.computer
}
}

2
crates/brk_server/Cargo.toml
View File

@@ -11,6 +11,7 @@ build = "build.rs"
[dependencies]
axum = { workspace = true }
bitcoin = { workspace = true }
bitcoincore-rpc = { workspace = true }
brk_computer = { workspace = true }
brk_error = { workspace = true }
@@ -20,6 +21,7 @@ brk_interface = { workspace = true }
brk_logger = { workspace = true }
brk_mcp = { workspace = true }
brk_parser = { workspace = true }
brk_structs = { workspace = true }
vecdb = { workspace = true }
jiff = { workspace = true }
log = { workspace = true }

170
crates/brk_server/src/api/mod.rs
View File

@@ -1,3 +1,5 @@
use std::str::FromStr;
use axum::{
Json, Router,
extract::{Path, Query, State},
@@ -5,12 +7,20 @@ use axum::{
response::{IntoResponse, Redirect, Response},
routing::get,
};
use bitcoin::{Address, Network, absolute::LockTime};
use bitcoincore_rpc::bitcoin;
use brk_interface::{IdParam, Index, PaginatedIndexParam, PaginationParam, Params, ParamsOpt};
use brk_structs::{
AddressBytesHash, AnyAddressDataIndexEnum, Bitcoin, OutputType, Txid, TxidPrefix,
};
use serde_json::Number;
use tracing::info;
use vecdb::{AnyIterableVec, VecIterator};
use super::AppState;
mod explorer;
mod interface;
mod vecs;
pub trait ApiRoutes {
fn add_api_routes(self) -> Self;
@@ -21,6 +31,160 @@ const TO_SEPARATOR: &str = "_to_";
impl ApiRoutes for Router<AppState> {
fn add_api_routes(self) -> Self {
self.route(
"/api/address/{address}",
get(
async |Path(address): Path<String>, state: State<AppState>| -> Response {
info!(address);
let address = Address::from_str(&address).unwrap();
if !address.is_valid_for_network(Network::Bitcoin) {
return "Invalid address".into_response();
}
let address = address.assume_checked();
let interface = state.interface;
let indexer = interface.indexer();
let computer = interface.computer();
let stores = &indexer.stores;
let hash = AddressBytesHash::from(&address);
dbg!(&hash);
dbg!(
&address,
address.address_type(),
address.script_pubkey(),
OutputType::from(&address)
);
let addri = stores
.addressbyteshash_to_typeindex
.get(&hash)
.unwrap()
.unwrap()
.into_owned();
println!("Script pubkey: {}", address.script_pubkey());
println!("Address type: {:?}", address.address_type());
let output_type = OutputType::from(&address);
let stateful = &computer.stateful;
let price = computer.price.as_ref().map(|v| {
*v.timeindexes_to_price_close
.dateindex
.as_ref()
.unwrap()
.iter()
.last()
.unwrap()
.1
.into_owned()
});
let anyaddri = match output_type {
OutputType::P2PK33 => stateful
.p2pk33addressindex_to_anyaddressindex
.iter()
.unwrap_get_inner(addri.into()),
OutputType::P2PK65 => stateful
.p2pk65addressindex_to_anyaddressindex
.iter()
.unwrap_get_inner(addri.into()),
OutputType::P2PKH => stateful
.p2pkhaddressindex_to_anyaddressindex
.iter()
.unwrap_get_inner(addri.into()),
OutputType::P2SH => stateful
.p2shaddressindex_to_anyaddressindex
.iter()
.unwrap_get_inner(addri.into()),
OutputType::P2TR => stateful
.p2traddressindex_to_anyaddressindex
.iter()
.unwrap_get_inner(addri.into()),
OutputType::P2WPKH => stateful
.p2wpkhaddressindex_to_anyaddressindex
.iter()
.unwrap_get_inner(addri.into()),
OutputType::P2WSH => stateful
.p2wshaddressindex_to_anyaddressindex
.iter()
.unwrap_get_inner(addri.into()),
OutputType::P2A => stateful
.p2aaddressindex_to_anyaddressindex
.iter()
.unwrap_get_inner(addri.into()),
_ => unreachable!(),
};
let addr_data = match anyaddri.to_enum() {
AnyAddressDataIndexEnum::Loaded(loadedi) => stateful
.loadedaddressindex_to_loadedaddressdata
.iter()
.unwrap_get_inner(loadedi),
AnyAddressDataIndexEnum::Empty(emptyi) => stateful
.emptyaddressindex_to_emptyaddressdata
.iter()
.unwrap_get_inner(emptyi)
.into(),
};
let amount = addr_data.amount();
Json(serde_json::json!({
"address": address,
"type": output_type,
"index": addri,
"chain_stats": {
"funded_txo_count": serde_json::Value::Null,
"funded_txo_sum": addr_data.received,
"spent_txo_count": serde_json::Value::Null,
"spent_txo_sum": addr_data.sent,
"utxo_count": addr_data.utxos,
"balance": amount,
"balance_usd": price.map_or(serde_json::Value::Null, |p| serde_json::Value::Number(Number::from_f64( *(p * Bitcoin::from(amount))).unwrap())),
"realized_value": addr_data.realized_cap,
"tx_count": serde_json::Value::Null,
"avg_cost_basis": addr_data.realized_price()
},
"mempool_stats": serde_json::Value::Null
}))
.into_response()
},
),
)
.route(
"/api/tx/{txid}",
get(
async |Path(txid): Path<String>, state: State<AppState>| -> Response {
let txid = bitcoin::Txid::from_str(&txid).unwrap();
let txid = Txid::from(txid);
let prefix = TxidPrefix::from(&txid);
let interface = state.interface;
let indexer = interface.indexer();
let txindex = indexer
.stores
.txidprefix_to_txindex
.get(&prefix)
.unwrap()
.unwrap()
.into_owned();
let version = indexer
.vecs
.txindex_to_txversion
.iter()
.unwrap_get_inner(txindex);
let rawlocktime = indexer
.vecs
.txindex_to_rawlocktime
.iter()
.unwrap_get_inner(txindex);
let locktime = LockTime::from(rawlocktime);
Json(serde_json::json!({
"txid": txid,
"index": txindex,
"version": version,
"locktime": locktime
}))
.into_response()
},
),
)
.route(
"/api/vecs/index-count",
get(async |State(app_state): State<AppState>| -> Response {
Json(app_state.interface.get_index_count()).into_response()
@@ -81,7 +245,7 @@ impl ApiRoutes for Router<AppState> {
),
)
// .route("/api/vecs/variants", get(variants_handler))
.route("/api/vecs/query", get(interface::handler))
.route("/api/vecs/query", get(vecs::handler))
.route(
"/api/vecs/{variant}",
get(
@@ -99,7 +263,7 @@ impl ApiRoutes for Router<AppState> {
(index, split.collect::<Vec<_>>().join(TO_SEPARATOR)),
params_opt,
));
interface::handler(uri, headers, Query(params), state).await
vecs::handler(uri, headers, Query(params), state).await
} else {
"Bad variant".into_response()
}

0
crates/brk_server/src/api/interface.rs → crates/brk_server/src/api/vecs.rs
View File

1
crates/brk_store/Cargo.toml
View File

@@ -17,3 +17,4 @@ brk_structs = { workspace = true }
byteview = { workspace = true }
fjall = { workspace = true }
log = { workspace = true }
parking_lot = { workspace = true }

25
crates/brk_store/src/lib.rs
View File

@@ -6,6 +6,7 @@ use std::{
fmt::Debug,
fs, mem,
path::Path,
sync::Arc,
};
use brk_error::Result;
@@ -22,13 +23,13 @@ mod meta;
pub use any::*;
use log::info;
use meta::*;
use parking_lot::RwLock;
pub struct Store<Key, Value> {
meta: StoreMeta,
name: &'static str,
keyspace: TransactionalKeyspace,
// Arc it
partition: Option<TransactionalPartitionHandle>,
partition: Arc<RwLock<Option<TransactionalPartitionHandle>>>,
rtx: ReadTransaction,
puts: BTreeMap<Key, Value>,
dels: BTreeSet<Key>,
@@ -77,7 +78,7 @@ where
meta,
name: Box::leak(Box::new(name.to_string())),
keyspace: keyspace.clone(),
partition: Some(partition),
partition: Arc::new(RwLock::new(Some(partition))),
rtx,
puts: BTreeMap::new(),
dels: BTreeSet::new(),
@@ -90,7 +91,7 @@ where
Ok(Some(Cow::Borrowed(v)))
} else if let Some(slice) = self
.rtx
.get(self.partition.as_ref().unwrap(), ByteView::from(key))?
.get(self.partition.read().as_ref().unwrap(), ByteView::from(key))?
{
Ok(Some(Cow::Owned(V::from(ByteView::from(slice)))))
} else {
@@ -100,7 +101,7 @@ where
pub fn is_empty(&self) -> Result<bool> {
self.rtx
.is_empty(self.partition.as_ref().unwrap())
.is_empty(self.partition.read().as_ref().unwrap())
.map_err(|e| e.into())
}
@@ -128,7 +129,7 @@ where
pub fn iter(&self) -> impl Iterator<Item = (K, V)> {
self.rtx
.iter(self.partition.as_ref().unwrap())
.iter(self.partition.read().as_ref().unwrap())
.map(|res| res.unwrap())
.map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v))))
}
@@ -203,7 +204,9 @@ where
let mut wtx = self.keyspace.write_tx();
let partition = self.partition.as_ref().unwrap();
let partition = self.partition.read();
let partition = partition.as_ref().unwrap();
remove.for_each(|key| wtx.remove(partition, ByteView::from(key)));
@@ -262,7 +265,9 @@ where
fn reset(&mut self) -> Result<()> {
info!("Resetting {}...", self.name);
let partition: TransactionalPartitionHandle = self.partition.take().unwrap();
let mut opt = self.partition.write();
let partition = opt.take().unwrap();
self.keyspace.delete_partition(partition)?;
@@ -270,7 +275,7 @@ where
let partition = Self::open_partition_handle(&self.keyspace, self.name, self.bloom_filters)?;
self.partition.replace(partition);
opt.replace(partition);
Ok(())
}
@@ -301,7 +306,7 @@ where
meta: self.meta.clone(),
name: self.name,
keyspace: self.keyspace.clone(),
partition: None,
partition: self.partition.clone(),
rtx: self.keyspace.read_tx(),
puts: self.puts.clone(),
dels: self.dels.clone(),

6
crates/brk_structs/src/structs/addressbytes.rs
View File

@@ -59,6 +59,12 @@ impl AddressBytes {
}
}
impl From<&Address> for AddressBytes {
fn from(value: &Address) -> Self {
Self::try_from((&value.script_pubkey(), OutputType::from(value))).unwrap()
}
}
impl TryFrom<(&ScriptBuf, OutputType)> for AddressBytes {
type Error = Error;
fn try_from(tuple: (&ScriptBuf, OutputType)) -> Result<Self, Self::Error> {

7
crates/brk_structs/src/structs/addressbyteshash.rs
View File

@@ -1,3 +1,4 @@
use bitcoin::Address;
use byteview::ByteView;
use derive_deref::Deref;
use zerocopy::{FromBytes, IntoBytes};
@@ -21,6 +22,12 @@ use super::{AddressBytes, OutputType};
)]
pub struct AddressBytesHash([u8; 8]);
impl From<&Address> for AddressBytesHash {
fn from(value: &Address) -> Self {
Self::from((&AddressBytes::from(value), OutputType::from(value)))
}
}
impl From<(&AddressBytes, OutputType)> for AddressBytesHash {
fn from((address_bytes, outputtype): (&AddressBytes, OutputType)) -> Self {
let mut slice = rapidhash::v3::rapidhash_v3(address_bytes.as_slice()).to_le_bytes();

38
crates/brk_structs/src/structs/outputtype.rs
View File

@@ -1,4 +1,5 @@
use bitcoin::{ScriptBuf, opcodes::all::OP_PUSHBYTES_2};
use bitcoin::{Address, AddressType, ScriptBuf, opcodes::all::OP_PUSHBYTES_2};
use brk_error::Error;
use serde::Serialize;
use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout};
@@ -380,3 +381,38 @@ impl From<&ScriptBuf> for OutputType {
}
}
}
impl From<&Address> for OutputType {
fn from(value: &Address) -> Self {
Self::from(&value.script_pubkey())
}
}
impl From<AddressType> for OutputType {
fn from(value: AddressType) -> Self {
match value {
AddressType::P2a => Self::P2A,
AddressType::P2pkh => Self::P2PKH,
AddressType::P2sh => Self::P2SH,
AddressType::P2tr => Self::P2TR,
AddressType::P2wpkh => Self::P2WPKH,
AddressType::P2wsh => Self::P2WSH,
_ => unreachable!(),
}
}
}
impl TryFrom<OutputType> for AddressType {
type Error = Error;
fn try_from(value: OutputType) -> Result<Self, Self::Error> {
Ok(match value {
OutputType::P2A => Self::P2a,
OutputType::P2PKH => Self::P2pkh,
OutputType::P2SH => Self::P2sh,
OutputType::P2TR => Self::P2tr,
OutputType::P2WPKH => Self::P2wpkh,
OutputType::P2WSH => Self::P2wsh,
_ => return Err(Error::Str("Bad output format")),
})
}
}

12
docs/TODO.md
View File

@@ -52,13 +52,12 @@
- _LOGGER_
- remove colors from file
- _PARSER_
- save `vec` file instead of `json`
- support lock file, process in read only if already opened in write mode
- Stateless
- if less than X (10 maybe ?) get block using rpc instead of parsing the block files
- support `None` output_dir
- _SERVER_
- api
- copy mempool's rest api
- https://mempool.space/docs/api/rest
- add extensions support (.json .csv …) instead of only format
- if format instead of extension then don't download file
- ddos protection
@@ -89,6 +88,8 @@
- _PACKAGES_
- move packages from `bitview` to `/packages` or `/websites/packages` or else
- move the fetching logic from `bitview` website to an independent `brk` package which could be published to npm
- https://www.npmjs.com/package/@mempool/mempool.js
- auto publish with github actions
- _BITVIEW_
- explorer
- blocks (interval as length between)
@@ -126,7 +127,10 @@
- global
- improve behavior when local storage is unavailable
- by having a global state
- font:
- https://fonts.google.com/specimen/Space+Mono
- keep as many files as possible [under 14kb](https://endtimes.dev/why-your-website-should-be-under-14kb-in-size/)
- __GLOBAL__
- check `TODO`s in codebase
- rename `output` to `txout`, `input` to `txin`
- rename `output` to `txout` or `vout`, `input` to `txin` or `vin`