diff --git a/Cargo.lock b/Cargo.lock index 92fef5998..427cef917 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1057,6 +1057,7 @@ name = "brk_server" version = "0.0.107" dependencies = [ "axum", + "bitcoin", "bitcoincore-rpc", "brk_computer", "brk_error", @@ -1066,6 +1067,7 @@ dependencies = [ "brk_logger", "brk_mcp", "brk_parser", + "brk_structs", "jiff", "log", "quick_cache", @@ -1085,6 +1087,7 @@ dependencies = [ "byteview", "fjall", "log", + "parking_lot 0.12.4", ] [[package]] @@ -2590,9 +2593,9 @@ checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libredox" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ "bitflags 2.9.4", "libc", @@ -4122,12 +4125,12 @@ dependencies = [ [[package]] name = "seize" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4b8d813387d566f627f3ea1b914c068aac94c40ae27ec43f5f33bde65abefe7" +checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -4161,27 +4164,38 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.219" +version = "1.0.223" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "a505d71960adde88e293da5cb5eda57093379f64e61cf77bf0e6a63af07a7bac" dependencies = [ + "serde_core", "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.17" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" +checksum = "fe07b5d88710e3b807c16a06ccbc9dfecd5fff6a4d2745c59e3e26774f10de6a" dependencies = [ "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.223" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f57cbd357666aa7b3ac84a90b4ea328f1d4ddb6772b430caa5d9e1309bb9e9" +dependencies = [ + "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.223" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "3d428d07faf17e306e699ec1e91996e5a165ba5d6bce5b5155173e91a8a01a56" dependencies = [ "proc-macro2", "quote", @@ -4201,25 +4215,27 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.143" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "indexmap 2.11.1", "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] name = "serde_path_to_error" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" +checksum = "a30a8abed938137c7183c173848e3c9b3517f5e038226849a4ecc9b21a4b4e2a" dependencies = [ "itoa", "serde", + "serde_core", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b679c6f87..82bf059a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,10 +68,10 @@ minreq = { version = "2.14.1", features = ["https", "serde_json"] } parking_lot = "0.12.4" quick_cache = "0.6.16" rayon = "1.11.0" -serde = "1.0.219" -serde_bytes = "0.11.17" -serde_derive = "1.0.219" -serde_json = { version = "1.0.143", features = ["float_roundtrip"] } +serde = "1.0.223" +serde_bytes = "0.11.18" +serde_derive = "1.0.223" +serde_json = { version = "1.0.145", features = ["float_roundtrip"] } tokio = { version = "1.47.1", features = ["rt-multi-thread"] } # vecdb = { path = "../seqdb/crates/vecdb", features = ["derive"]} vecdb = { version = "0.2.14", features = ["derive"]} diff --git a/crates/brk_computer/src/stateful/mod.rs b/crates/brk_computer/src/stateful/mod.rs index c040d52f6..f5cd9e354 100644 --- a/crates/brk_computer/src/stateful/mod.rs +++ b/crates/brk_computer/src/stateful/mod.rs @@ -1495,35 +1495,35 @@ impl Vecs { return None; } - let mmap = addresstypeindex_to_anyaddressindex_reader_opt + let reader = addresstypeindex_to_anyaddressindex_reader_opt .get_unwrap(address_type) .as_ref() .unwrap(); let anyaddressindex = match address_type { OutputType::P2PK33 => { - p2pk33addressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + p2pk33addressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader) } OutputType::P2PK65 => { - p2pk65addressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + p2pk65addressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader) } OutputType::P2PKH => { - p2pkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + p2pkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader) } OutputType::P2SH => { - p2shaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + p2shaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader) } OutputType::P2TR => { - p2traddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + p2traddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader) } OutputType::P2WPKH => { - p2wpkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + p2wpkhaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader) } OutputType::P2WSH => { - p2wshaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + p2wshaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader) } OutputType::P2A => { - p2aaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), mmap) + p2aaddressindex_to_anyaddressindex.get_or_read(typeindex.into(), reader) } _ => unreachable!(), } diff --git a/crates/brk_interface/src/lib.rs b/crates/brk_interface/src/lib.rs index 94e6a09ba..86b48e731 100644 --- a/crates/brk_interface/src/lib.rs +++ b/crates/brk_interface/src/lib.rs @@ -261,4 +261,12 @@ impl<'a> Interface<'a> { pub fn get_vecid_to_indexes(&self, id: String) -> Option<&Vec<&'static str>> { self.vecs.id_to_indexes(id) } + + pub fn indexer(&self) -> &Indexer { + self.indexer + } + + pub fn computer(&self) -> &Computer { + self.computer + } } diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index a5e5183b6..bc3f28d05 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -11,6 +11,7 @@ build = "build.rs" [dependencies] axum = { workspace = true } +bitcoin = { workspace = true } bitcoincore-rpc = { workspace = true } brk_computer = { workspace = true } brk_error = { workspace = true } @@ -20,6 +21,7 @@ brk_interface = { workspace = true } brk_logger = { workspace = true } brk_mcp = { workspace = true } brk_parser = { workspace = true } +brk_structs = { workspace = true } vecdb = { workspace = true } jiff = { workspace = true } log = { workspace = true } diff --git a/crates/brk_server/src/api/mod.rs b/crates/brk_server/src/api/mod.rs index 8a3a5579b..74a649c31 100644 --- a/crates/brk_server/src/api/mod.rs +++ b/crates/brk_server/src/api/mod.rs @@ -1,3 +1,5 @@ +use std::str::FromStr; + use axum::{ Json, Router, extract::{Path, Query, State}, @@ -5,12 +7,20 @@ use axum::{ response::{IntoResponse, Redirect, Response}, routing::get, }; +use bitcoin::{Address, Network, absolute::LockTime}; +use bitcoincore_rpc::bitcoin; use brk_interface::{IdParam, Index, PaginatedIndexParam, PaginationParam, Params, ParamsOpt}; +use brk_structs::{ + AddressBytesHash, AnyAddressDataIndexEnum, Bitcoin, OutputType, Txid, TxidPrefix, +}; +use serde_json::Number; +use tracing::info; +use vecdb::{AnyIterableVec, VecIterator}; use super::AppState; mod explorer; -mod interface; +mod vecs; pub trait ApiRoutes { fn add_api_routes(self) -> Self; @@ -21,6 +31,160 @@ const TO_SEPARATOR: &str = "_to_"; impl ApiRoutes for Router { fn add_api_routes(self) -> Self { self.route( + "/api/address/{address}", + get( + async |Path(address): Path, state: State| -> Response { + info!(address); + let address = Address::from_str(&address).unwrap(); + if !address.is_valid_for_network(Network::Bitcoin) { + return "Invalid address".into_response(); + } + let address = address.assume_checked(); + let interface = state.interface; + let indexer = interface.indexer(); + let computer = interface.computer(); + let stores = &indexer.stores; + let hash = AddressBytesHash::from(&address); + dbg!(&hash); + dbg!( + &address, + address.address_type(), + address.script_pubkey(), + OutputType::from(&address) + ); + let addri = stores + .addressbyteshash_to_typeindex + .get(&hash) + .unwrap() + .unwrap() + .into_owned(); + println!("Script pubkey: {}", address.script_pubkey()); + println!("Address type: {:?}", address.address_type()); + let output_type = OutputType::from(&address); + let stateful = &computer.stateful; + let price = computer.price.as_ref().map(|v| { + *v.timeindexes_to_price_close + .dateindex + .as_ref() + .unwrap() + .iter() + .last() + .unwrap() + .1 + .into_owned() + }); + + let anyaddri = match output_type { + OutputType::P2PK33 => stateful + .p2pk33addressindex_to_anyaddressindex + .iter() + .unwrap_get_inner(addri.into()), + OutputType::P2PK65 => stateful + .p2pk65addressindex_to_anyaddressindex + .iter() + .unwrap_get_inner(addri.into()), + OutputType::P2PKH => stateful + .p2pkhaddressindex_to_anyaddressindex + .iter() + .unwrap_get_inner(addri.into()), + OutputType::P2SH => stateful + .p2shaddressindex_to_anyaddressindex + .iter() + .unwrap_get_inner(addri.into()), + OutputType::P2TR => stateful + .p2traddressindex_to_anyaddressindex + .iter() + .unwrap_get_inner(addri.into()), + OutputType::P2WPKH => stateful + .p2wpkhaddressindex_to_anyaddressindex + .iter() + .unwrap_get_inner(addri.into()), + OutputType::P2WSH => stateful + .p2wshaddressindex_to_anyaddressindex + .iter() + .unwrap_get_inner(addri.into()), + OutputType::P2A => stateful + .p2aaddressindex_to_anyaddressindex + .iter() + .unwrap_get_inner(addri.into()), + + _ => unreachable!(), + }; + + let addr_data = match anyaddri.to_enum() { + AnyAddressDataIndexEnum::Loaded(loadedi) => stateful + .loadedaddressindex_to_loadedaddressdata + .iter() + .unwrap_get_inner(loadedi), + AnyAddressDataIndexEnum::Empty(emptyi) => stateful + .emptyaddressindex_to_emptyaddressdata + .iter() + .unwrap_get_inner(emptyi) + .into(), + }; + + let amount = addr_data.amount(); + Json(serde_json::json!({ + "address": address, + "type": output_type, + "index": addri, + "chain_stats": { + "funded_txo_count": serde_json::Value::Null, + "funded_txo_sum": addr_data.received, + "spent_txo_count": serde_json::Value::Null, + "spent_txo_sum": addr_data.sent, + "utxo_count": addr_data.utxos, + "balance": amount, + "balance_usd": price.map_or(serde_json::Value::Null, |p| serde_json::Value::Number(Number::from_f64( *(p * Bitcoin::from(amount))).unwrap())), + "realized_value": addr_data.realized_cap, + "tx_count": serde_json::Value::Null, + "avg_cost_basis": addr_data.realized_price() + }, + "mempool_stats": serde_json::Value::Null + })) + .into_response() + }, + ), + ) + .route( + "/api/tx/{txid}", + get( + async |Path(txid): Path, state: State| -> Response { + let txid = bitcoin::Txid::from_str(&txid).unwrap(); + let txid = Txid::from(txid); + let prefix = TxidPrefix::from(&txid); + let interface = state.interface; + let indexer = interface.indexer(); + let txindex = indexer + .stores + .txidprefix_to_txindex + .get(&prefix) + .unwrap() + .unwrap() + .into_owned(); + let version = indexer + .vecs + .txindex_to_txversion + .iter() + .unwrap_get_inner(txindex); + let rawlocktime = indexer + .vecs + .txindex_to_rawlocktime + .iter() + .unwrap_get_inner(txindex); + let locktime = LockTime::from(rawlocktime); + + Json(serde_json::json!({ + "txid": txid, + "index": txindex, + "version": version, + "locktime": locktime + })) + .into_response() + }, + ), + ) + .route( "/api/vecs/index-count", get(async |State(app_state): State| -> Response { Json(app_state.interface.get_index_count()).into_response() @@ -81,7 +245,7 @@ impl ApiRoutes for Router { ), ) // .route("/api/vecs/variants", get(variants_handler)) - .route("/api/vecs/query", get(interface::handler)) + .route("/api/vecs/query", get(vecs::handler)) .route( "/api/vecs/{variant}", get( @@ -99,7 +263,7 @@ impl ApiRoutes for Router { (index, split.collect::>().join(TO_SEPARATOR)), params_opt, )); - interface::handler(uri, headers, Query(params), state).await + vecs::handler(uri, headers, Query(params), state).await } else { "Bad variant".into_response() } diff --git a/crates/brk_server/src/api/interface.rs b/crates/brk_server/src/api/vecs.rs similarity index 100% rename from crates/brk_server/src/api/interface.rs rename to crates/brk_server/src/api/vecs.rs diff --git a/crates/brk_store/Cargo.toml b/crates/brk_store/Cargo.toml index 1b42a5d07..f018f27de 100644 --- a/crates/brk_store/Cargo.toml +++ b/crates/brk_store/Cargo.toml @@ -17,3 +17,4 @@ brk_structs = { workspace = true } byteview = { workspace = true } fjall = { workspace = true } log = { workspace = true } +parking_lot = { workspace = true } diff --git a/crates/brk_store/src/lib.rs b/crates/brk_store/src/lib.rs index f9bc24236..63ea6a9d9 100644 --- a/crates/brk_store/src/lib.rs +++ b/crates/brk_store/src/lib.rs @@ -6,6 +6,7 @@ use std::{ fmt::Debug, fs, mem, path::Path, + sync::Arc, }; use brk_error::Result; @@ -22,13 +23,13 @@ mod meta; pub use any::*; use log::info; use meta::*; +use parking_lot::RwLock; pub struct Store { meta: StoreMeta, name: &'static str, keyspace: TransactionalKeyspace, - // Arc it - partition: Option, + partition: Arc>>, rtx: ReadTransaction, puts: BTreeMap, dels: BTreeSet, @@ -77,7 +78,7 @@ where meta, name: Box::leak(Box::new(name.to_string())), keyspace: keyspace.clone(), - partition: Some(partition), + partition: Arc::new(RwLock::new(Some(partition))), rtx, puts: BTreeMap::new(), dels: BTreeSet::new(), @@ -90,7 +91,7 @@ where Ok(Some(Cow::Borrowed(v))) } else if let Some(slice) = self .rtx - .get(self.partition.as_ref().unwrap(), ByteView::from(key))? + .get(self.partition.read().as_ref().unwrap(), ByteView::from(key))? { Ok(Some(Cow::Owned(V::from(ByteView::from(slice))))) } else { @@ -100,7 +101,7 @@ where pub fn is_empty(&self) -> Result { self.rtx - .is_empty(self.partition.as_ref().unwrap()) + .is_empty(self.partition.read().as_ref().unwrap()) .map_err(|e| e.into()) } @@ -128,7 +129,7 @@ where pub fn iter(&self) -> impl Iterator { self.rtx - .iter(self.partition.as_ref().unwrap()) + .iter(self.partition.read().as_ref().unwrap()) .map(|res| res.unwrap()) .map(|(k, v)| (K::from(ByteView::from(k)), V::from(ByteView::from(v)))) } @@ -203,7 +204,9 @@ where let mut wtx = self.keyspace.write_tx(); - let partition = self.partition.as_ref().unwrap(); + let partition = self.partition.read(); + + let partition = partition.as_ref().unwrap(); remove.for_each(|key| wtx.remove(partition, ByteView::from(key))); @@ -262,7 +265,9 @@ where fn reset(&mut self) -> Result<()> { info!("Resetting {}...", self.name); - let partition: TransactionalPartitionHandle = self.partition.take().unwrap(); + let mut opt = self.partition.write(); + + let partition = opt.take().unwrap(); self.keyspace.delete_partition(partition)?; @@ -270,7 +275,7 @@ where let partition = Self::open_partition_handle(&self.keyspace, self.name, self.bloom_filters)?; - self.partition.replace(partition); + opt.replace(partition); Ok(()) } @@ -301,7 +306,7 @@ where meta: self.meta.clone(), name: self.name, keyspace: self.keyspace.clone(), - partition: None, + partition: self.partition.clone(), rtx: self.keyspace.read_tx(), puts: self.puts.clone(), dels: self.dels.clone(), diff --git a/crates/brk_structs/src/structs/addressbytes.rs b/crates/brk_structs/src/structs/addressbytes.rs index b9a3b88f6..2dbd9de79 100644 --- a/crates/brk_structs/src/structs/addressbytes.rs +++ b/crates/brk_structs/src/structs/addressbytes.rs @@ -59,6 +59,12 @@ impl AddressBytes { } } +impl From<&Address> for AddressBytes { + fn from(value: &Address) -> Self { + Self::try_from((&value.script_pubkey(), OutputType::from(value))).unwrap() + } +} + impl TryFrom<(&ScriptBuf, OutputType)> for AddressBytes { type Error = Error; fn try_from(tuple: (&ScriptBuf, OutputType)) -> Result { diff --git a/crates/brk_structs/src/structs/addressbyteshash.rs b/crates/brk_structs/src/structs/addressbyteshash.rs index c6b453fb1..8ce488d5d 100644 --- a/crates/brk_structs/src/structs/addressbyteshash.rs +++ b/crates/brk_structs/src/structs/addressbyteshash.rs @@ -1,3 +1,4 @@ +use bitcoin::Address; use byteview::ByteView; use derive_deref::Deref; use zerocopy::{FromBytes, IntoBytes}; @@ -21,6 +22,12 @@ use super::{AddressBytes, OutputType}; )] pub struct AddressBytesHash([u8; 8]); +impl From<&Address> for AddressBytesHash { + fn from(value: &Address) -> Self { + Self::from((&AddressBytes::from(value), OutputType::from(value))) + } +} + impl From<(&AddressBytes, OutputType)> for AddressBytesHash { fn from((address_bytes, outputtype): (&AddressBytes, OutputType)) -> Self { let mut slice = rapidhash::v3::rapidhash_v3(address_bytes.as_slice()).to_le_bytes(); diff --git a/crates/brk_structs/src/structs/outputtype.rs b/crates/brk_structs/src/structs/outputtype.rs index 4699e0eb0..20df5ab35 100644 --- a/crates/brk_structs/src/structs/outputtype.rs +++ b/crates/brk_structs/src/structs/outputtype.rs @@ -1,4 +1,5 @@ -use bitcoin::{ScriptBuf, opcodes::all::OP_PUSHBYTES_2}; +use bitcoin::{Address, AddressType, ScriptBuf, opcodes::all::OP_PUSHBYTES_2}; +use brk_error::Error; use serde::Serialize; use zerocopy_derive::{FromBytes, Immutable, IntoBytes, KnownLayout}; @@ -380,3 +381,38 @@ impl From<&ScriptBuf> for OutputType { } } } + +impl From<&Address> for OutputType { + fn from(value: &Address) -> Self { + Self::from(&value.script_pubkey()) + } +} + +impl From for OutputType { + fn from(value: AddressType) -> Self { + match value { + AddressType::P2a => Self::P2A, + AddressType::P2pkh => Self::P2PKH, + AddressType::P2sh => Self::P2SH, + AddressType::P2tr => Self::P2TR, + AddressType::P2wpkh => Self::P2WPKH, + AddressType::P2wsh => Self::P2WSH, + _ => unreachable!(), + } + } +} + +impl TryFrom for AddressType { + type Error = Error; + fn try_from(value: OutputType) -> Result { + Ok(match value { + OutputType::P2A => Self::P2a, + OutputType::P2PKH => Self::P2pkh, + OutputType::P2SH => Self::P2sh, + OutputType::P2TR => Self::P2tr, + OutputType::P2WPKH => Self::P2wpkh, + OutputType::P2WSH => Self::P2wsh, + _ => return Err(Error::Str("Bad output format")), + }) + } +} diff --git a/docs/TODO.md b/docs/TODO.md index 1e77488db..eba2440f8 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -52,13 +52,12 @@ - _LOGGER_ - remove colors from file - _PARSER_ - - save `vec` file instead of `json` - - support lock file, process in read only if already opened in write mode + - Stateless - if less than X (10 maybe ?) get block using rpc instead of parsing the block files - - support `None` output_dir - _SERVER_ - api - copy mempool's rest api + - https://mempool.space/docs/api/rest - add extensions support (.json .csv …) instead of only format - if format instead of extension then don't download file - ddos protection @@ -89,6 +88,8 @@ - _PACKAGES_ - move packages from `bitview` to `/packages` or `/websites/packages` or else - move the fetching logic from `bitview` website to an independent `brk` package which could be published to npm + - https://www.npmjs.com/package/@mempool/mempool.js + - auto publish with github actions - _BITVIEW_ - explorer - blocks (interval as length between) @@ -126,7 +127,10 @@ - global - improve behavior when local storage is unavailable - by having a global state + - font: + - https://fonts.google.com/specimen/Space+Mono + - keep as many files as possible [under 14kb](https://endtimes.dev/why-your-website-should-be-under-14kb-in-size/) - __GLOBAL__ - check `TODO`s in codebase - - rename `output` to `txout`, `input` to `txin` + - rename `output` to `txout` or `vout`, `input` to `txin` or `vin`