mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-07-25 17:58:11 -07:00
global: snapshot
This commit is contained in:
Generated
+16
-6
@@ -506,10 +506,13 @@ dependencies = [
|
|||||||
"brk_fetcher",
|
"brk_fetcher",
|
||||||
"brk_grouper",
|
"brk_grouper",
|
||||||
"brk_indexer",
|
"brk_indexer",
|
||||||
|
"brk_iterator",
|
||||||
"brk_logger",
|
"brk_logger",
|
||||||
"brk_mcp",
|
"brk_mcp",
|
||||||
|
"brk_monitor",
|
||||||
"brk_query",
|
"brk_query",
|
||||||
"brk_reader",
|
"brk_reader",
|
||||||
|
"brk_rpc",
|
||||||
"brk_server",
|
"brk_server",
|
||||||
"brk_store",
|
"brk_store",
|
||||||
"brk_traversable",
|
"brk_traversable",
|
||||||
@@ -612,9 +615,11 @@ dependencies = [
|
|||||||
"brk_error",
|
"brk_error",
|
||||||
"brk_fetcher",
|
"brk_fetcher",
|
||||||
"brk_indexer",
|
"brk_indexer",
|
||||||
|
"brk_iterator",
|
||||||
"brk_logger",
|
"brk_logger",
|
||||||
"brk_query",
|
"brk_query",
|
||||||
"brk_reader",
|
"brk_reader",
|
||||||
|
"brk_rpc",
|
||||||
"brk_server",
|
"brk_server",
|
||||||
"clap",
|
"clap",
|
||||||
"color-eyre",
|
"color-eyre",
|
||||||
@@ -637,8 +642,10 @@ dependencies = [
|
|||||||
"brk_fetcher",
|
"brk_fetcher",
|
||||||
"brk_grouper",
|
"brk_grouper",
|
||||||
"brk_indexer",
|
"brk_indexer",
|
||||||
|
"brk_iterator",
|
||||||
"brk_logger",
|
"brk_logger",
|
||||||
"brk_reader",
|
"brk_reader",
|
||||||
|
"brk_rpc",
|
||||||
"brk_store",
|
"brk_store",
|
||||||
"brk_traversable",
|
"brk_traversable",
|
||||||
"brk_types",
|
"brk_types",
|
||||||
@@ -783,6 +790,7 @@ dependencies = [
|
|||||||
"brk_error",
|
"brk_error",
|
||||||
"brk_indexer",
|
"brk_indexer",
|
||||||
"brk_reader",
|
"brk_reader",
|
||||||
|
"brk_rpc",
|
||||||
"brk_traversable",
|
"brk_traversable",
|
||||||
"brk_types",
|
"brk_types",
|
||||||
"derive_deref",
|
"derive_deref",
|
||||||
@@ -1224,10 +1232,12 @@ dependencies = [
|
|||||||
"brk_error",
|
"brk_error",
|
||||||
"brk_fetcher",
|
"brk_fetcher",
|
||||||
"brk_indexer",
|
"brk_indexer",
|
||||||
|
"brk_iterator",
|
||||||
"brk_logger",
|
"brk_logger",
|
||||||
"brk_mcp",
|
"brk_mcp",
|
||||||
"brk_query",
|
"brk_query",
|
||||||
"brk_reader",
|
"brk_reader",
|
||||||
|
"brk_rpc",
|
||||||
"brk_traversable",
|
"brk_traversable",
|
||||||
"brk_types",
|
"brk_types",
|
||||||
"jiff",
|
"jiff",
|
||||||
@@ -1375,9 +1385,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cc"
|
name = "cc"
|
||||||
version = "1.2.41"
|
version = "1.2.42"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7"
|
checksum = "81bbf3b3619004ad9bd139f62a9ab5cfe467f307455a0d307b0cf58bf070feaa"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"find-msvc-tools",
|
"find-msvc-tools",
|
||||||
"jobserver",
|
"jobserver",
|
||||||
@@ -2018,9 +2028,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "flate2"
|
name = "flate2"
|
||||||
version = "1.1.4"
|
version = "1.1.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9"
|
checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crc32fast",
|
"crc32fast",
|
||||||
"libz-rs-sys",
|
"libz-rs-sys",
|
||||||
@@ -3757,9 +3767,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proc-macro2"
|
name = "proc-macro2"
|
||||||
version = "1.0.102"
|
version = "1.0.103"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8e0f6df8eaa422d97d72edcd152e1451618fed47fabbdbd5a8864167b1d4aff7"
|
checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"unicode-ident",
|
"unicode-ident",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ pub trait Bridge {
|
|||||||
fn generate_js_files(&self, modules_path: &Path) -> io::Result<()>;
|
fn generate_js_files(&self, modules_path: &Path) -> io::Result<()>;
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Bridge for Query<'static> {
|
impl Bridge for Query {
|
||||||
fn generate_js_files(&self, modules_path: &Path) -> io::Result<()> {
|
fn generate_js_files(&self, modules_path: &Path) -> io::Result<()> {
|
||||||
let path = modules_path.join("brk-client");
|
let path = modules_path.join("brk-client");
|
||||||
|
|
||||||
@@ -81,7 +81,7 @@ export const POOL_ID_TO_POOL_NAME = /** @type {const} */ ({
|
|||||||
fs::write(path, contents)
|
fs::write(path, contents)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn generate_metrics_file(query: &Query<'static>, parent: &Path) -> io::Result<()> {
|
fn generate_metrics_file(query: &Query, parent: &Path) -> io::Result<()> {
|
||||||
let path = parent.join(Path::new("metrics.js"));
|
let path = parent.join(Path::new("metrics.js"));
|
||||||
|
|
||||||
let indexes = Index::all();
|
let indexes = Index::all();
|
||||||
|
|||||||
@@ -16,9 +16,11 @@ brk_computer = { workspace = true }
|
|||||||
brk_error = { workspace = true }
|
brk_error = { workspace = true }
|
||||||
brk_fetcher = { workspace = true }
|
brk_fetcher = { workspace = true }
|
||||||
brk_indexer = { workspace = true }
|
brk_indexer = { workspace = true }
|
||||||
|
brk_iterator = { workspace = true }
|
||||||
brk_query = { workspace = true }
|
brk_query = { workspace = true }
|
||||||
brk_logger = { workspace = true }
|
brk_logger = { workspace = true }
|
||||||
brk_reader = { workspace = true }
|
brk_reader = { workspace = true }
|
||||||
|
brk_rpc = { workspace = true }
|
||||||
brk_server = { workspace = true }
|
brk_server = { workspace = true }
|
||||||
vecdb = { workspace = true }
|
vecdb = { workspace = true }
|
||||||
clap = { version = "4.5.50", features = ["derive", "string"] }
|
clap = { version = "4.5.50", features = ["derive", "string"] }
|
||||||
|
|||||||
@@ -3,10 +3,10 @@ use std::{
|
|||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
};
|
};
|
||||||
|
|
||||||
use bitcoincore_rpc::{self, Auth, Client};
|
use brk_error::{Error, Result};
|
||||||
use brk_fetcher::Fetcher;
|
use brk_fetcher::Fetcher;
|
||||||
|
use brk_rpc::{Auth, Client};
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use color_eyre::eyre::eyre;
|
|
||||||
use serde::{Deserialize, Deserializer, Serialize};
|
use serde::{Deserialize, Deserializer, Serialize};
|
||||||
|
|
||||||
use crate::{default_bitcoin_path, default_brk_path, dot_brk_path, website::Website};
|
use crate::{default_bitcoin_path, default_brk_path, dot_brk_path, website::Website};
|
||||||
@@ -78,7 +78,7 @@ pub struct Config {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Config {
|
impl Config {
|
||||||
pub fn import() -> color_eyre::Result<Self> {
|
pub fn import() -> Result<Self> {
|
||||||
let config_args = Some(Config::parse());
|
let config_args = Some(Config::parse());
|
||||||
|
|
||||||
let path = dot_brk_path();
|
let path = dot_brk_path();
|
||||||
@@ -196,18 +196,18 @@ Finally, you can run the program with '-h' for help."
|
|||||||
fs::write(path, toml::to_string(self).unwrap())
|
fs::write(path, toml::to_string(self).unwrap())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn rpc(&self) -> color_eyre::Result<&'static Client> {
|
pub fn rpc(&self) -> Result<Client> {
|
||||||
Ok(Box::leak(Box::new(Client::new(
|
Ok(Client::new(
|
||||||
&format!(
|
&format!(
|
||||||
"http://{}:{}",
|
"http://{}:{}",
|
||||||
self.rpcconnect().unwrap_or(&"localhost".to_string()),
|
self.rpcconnect().unwrap_or(&"localhost".to_string()),
|
||||||
self.rpcport().unwrap_or(8332)
|
self.rpcport().unwrap_or(8332)
|
||||||
),
|
),
|
||||||
self.rpc_auth().unwrap(),
|
self.rpc_auth().unwrap(),
|
||||||
)?)))
|
)?)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn rpc_auth(&self) -> color_eyre::Result<Auth> {
|
fn rpc_auth(&self) -> Result<Auth> {
|
||||||
let cookie = self.path_cookiefile();
|
let cookie = self.path_cookiefile();
|
||||||
|
|
||||||
if cookie.is_file() {
|
if cookie.is_file() {
|
||||||
@@ -218,7 +218,7 @@ Finally, you can run the program with '-h' for help."
|
|||||||
self.rpcpassword.clone().unwrap(),
|
self.rpcpassword.clone().unwrap(),
|
||||||
))
|
))
|
||||||
} else {
|
} else {
|
||||||
Err(eyre!("Failed to find correct auth"))
|
Err(Error::Str("Failed to find correct auth"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+19
-32
@@ -8,12 +8,12 @@ use std::{
|
|||||||
time::Duration,
|
time::Duration,
|
||||||
};
|
};
|
||||||
|
|
||||||
use bitcoincore_rpc::{self, RpcApi};
|
|
||||||
use brk_binder::Bridge;
|
use brk_binder::Bridge;
|
||||||
use brk_bundler::bundle;
|
use brk_bundler::bundle;
|
||||||
use brk_computer::Computer;
|
use brk_computer::Computer;
|
||||||
use brk_error::Result;
|
use brk_error::Result;
|
||||||
use brk_indexer::Indexer;
|
use brk_indexer::Indexer;
|
||||||
|
use brk_iterator::Blocks;
|
||||||
use brk_query::Query;
|
use brk_query::Query;
|
||||||
use brk_reader::Reader;
|
use brk_reader::Reader;
|
||||||
use brk_server::{Server, VERSION};
|
use brk_server::{Server, VERSION};
|
||||||
@@ -27,12 +27,7 @@ mod website;
|
|||||||
use crate::{config::Config, paths::*};
|
use crate::{config::Config, paths::*};
|
||||||
|
|
||||||
pub fn main() -> color_eyre::Result<()> {
|
pub fn main() -> color_eyre::Result<()> {
|
||||||
color_eyre::install()?;
|
// Can't increase main thread's stack size, thus we need to use another thread
|
||||||
|
|
||||||
fs::create_dir_all(dot_brk_path())?;
|
|
||||||
|
|
||||||
brk_logger::init(Some(&dot_brk_log_path()))?;
|
|
||||||
|
|
||||||
thread::Builder::new()
|
thread::Builder::new()
|
||||||
.stack_size(512 * 1024 * 1024)
|
.stack_size(512 * 1024 * 1024)
|
||||||
.spawn(run)?
|
.spawn(run)?
|
||||||
@@ -41,14 +36,22 @@ pub fn main() -> color_eyre::Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn run() -> color_eyre::Result<()> {
|
pub fn run() -> color_eyre::Result<()> {
|
||||||
|
color_eyre::install()?;
|
||||||
|
|
||||||
|
fs::create_dir_all(dot_brk_path())?;
|
||||||
|
|
||||||
|
brk_logger::init(Some(&dot_brk_log_path()))?;
|
||||||
|
|
||||||
let config = Config::import()?;
|
let config = Config::import()?;
|
||||||
|
|
||||||
let rpc = config.rpc()?;
|
let client = config.rpc()?;
|
||||||
|
|
||||||
let exit = Exit::new();
|
let exit = Exit::new();
|
||||||
exit.set_ctrlc_handler();
|
exit.set_ctrlc_handler();
|
||||||
|
|
||||||
let reader = Reader::new(config.blocksdir(), rpc);
|
let reader = Reader::new(config.blocksdir(), &client);
|
||||||
|
|
||||||
|
let blocks = Blocks::new(&client, &reader);
|
||||||
|
|
||||||
let mut indexer = Indexer::forced_import(&config.brkdir())?;
|
let mut indexer = Indexer::forced_import(&config.brkdir())?;
|
||||||
|
|
||||||
@@ -112,7 +115,7 @@ pub fn run() -> color_eyre::Result<()> {
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
let server = Server::new(query, bundle_path);
|
let server = Server::new(&query, bundle_path);
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
server.serve(true).await.unwrap();
|
server.serve(true).await.unwrap();
|
||||||
@@ -128,40 +131,24 @@ pub fn run() -> color_eyre::Result<()> {
|
|||||||
let _handle = runtime.spawn(future);
|
let _handle = runtime.spawn(future);
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
wait_for_synced_node(rpc)?;
|
client.wait_for_synced_node()?;
|
||||||
|
|
||||||
let block_count = rpc.get_block_count()?;
|
let last_height = client.get_last_height()?;
|
||||||
|
|
||||||
info!("{} blocks found.", block_count + 1);
|
info!("{} blocks found.", u32::from(last_height) + 1);
|
||||||
|
|
||||||
let starting_indexes = if config.check_collisions() {
|
let starting_indexes = if config.check_collisions() {
|
||||||
indexer.checked_index(&reader, rpc, &exit)?;
|
indexer.checked_index(&blocks, &client, &exit)?
|
||||||
} else {
|
} else {
|
||||||
indexer.index(&reader, rpc, &exit)?;
|
indexer.index(&blocks, &client, &exit)?
|
||||||
};
|
};
|
||||||
|
|
||||||
computer.compute(&indexer, starting_indexes, &reader, &exit)?;
|
computer.compute(&indexer, starting_indexes, &reader, &exit)?;
|
||||||
|
|
||||||
info!("Waiting for new blocks...");
|
info!("Waiting for new blocks...");
|
||||||
|
|
||||||
while block_count == rpc.get_block_count()? {
|
while last_height == client.get_last_height()? {
|
||||||
sleep(Duration::from_secs(1))
|
sleep(Duration::from_secs(1))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn wait_for_synced_node(rpc_client: &bitcoincore_rpc::Client) -> color_eyre::Result<()> {
|
|
||||||
let is_synced = || -> color_eyre::Result<bool> {
|
|
||||||
let info = rpc_client.get_blockchain_info()?;
|
|
||||||
Ok(info.headers == info.blocks)
|
|
||||||
};
|
|
||||||
|
|
||||||
if !is_synced()? {
|
|
||||||
info!("Waiting for node to sync...");
|
|
||||||
while !is_synced()? {
|
|
||||||
sleep(Duration::from_secs(1))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -14,6 +14,15 @@ use brk_rpc::{Auth, Client};
|
|||||||
use vecdb::Exit;
|
use vecdb::Exit;
|
||||||
|
|
||||||
pub fn main() -> Result<()> {
|
pub fn main() -> Result<()> {
|
||||||
|
// Can't increase main thread's stack size, thus we need to use another thread
|
||||||
|
thread::Builder::new()
|
||||||
|
.stack_size(512 * 1024 * 1024)
|
||||||
|
.spawn(run)?
|
||||||
|
.join()
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run() -> Result<()> {
|
||||||
brk_logger::init(Some(Path::new(".log")))?;
|
brk_logger::init(Some(Path::new(".log")))?;
|
||||||
|
|
||||||
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
|
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
|
||||||
@@ -22,6 +31,9 @@ pub fn main() -> Result<()> {
|
|||||||
.join("Bitcoin");
|
.join("Bitcoin");
|
||||||
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
|
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK/bitcoin");
|
||||||
|
|
||||||
|
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
|
||||||
|
// let outputs_dir = Path::new("../../_outputs");
|
||||||
|
|
||||||
let client = Client::new(
|
let client = Client::new(
|
||||||
"http://localhost:8332",
|
"http://localhost:8332",
|
||||||
Auth::CookieFile(bitcoin_dir.join(".cookie")),
|
Auth::CookieFile(bitcoin_dir.join(".cookie")),
|
||||||
@@ -31,9 +43,6 @@ pub fn main() -> Result<()> {
|
|||||||
|
|
||||||
let blocks = Blocks::new(&client, &reader);
|
let blocks = Blocks::new(&client, &reader);
|
||||||
|
|
||||||
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
|
|
||||||
// let outputs_dir = Path::new("../../_outputs");
|
|
||||||
|
|
||||||
let mut indexer = Indexer::forced_import(&outputs_dir)?;
|
let mut indexer = Indexer::forced_import(&outputs_dir)?;
|
||||||
|
|
||||||
let fetcher = Fetcher::import(true, None)?;
|
let fetcher = Fetcher::import(true, None)?;
|
||||||
@@ -41,20 +50,13 @@ pub fn main() -> Result<()> {
|
|||||||
let exit = Exit::new();
|
let exit = Exit::new();
|
||||||
exit.set_ctrlc_handler();
|
exit.set_ctrlc_handler();
|
||||||
|
|
||||||
// Can't increase main thread's stack size, thus we need to use another thread
|
let mut computer = Computer::forced_import(&outputs_dir, &indexer, Some(fetcher))?;
|
||||||
thread::Builder::new()
|
|
||||||
.stack_size(256 * 1024 * 1024)
|
|
||||||
.spawn(move || -> Result<()> {
|
|
||||||
let mut computer = Computer::forced_import(&outputs_dir, &indexer, Some(fetcher))?;
|
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let i = Instant::now();
|
let i = Instant::now();
|
||||||
let starting_indexes = indexer.checked_index(&blocks, &client, &exit)?;
|
let starting_indexes = indexer.checked_index(&blocks, &client, &exit)?;
|
||||||
computer.compute(&indexer, starting_indexes, &reader, &exit)?;
|
computer.compute(&indexer, starting_indexes, &reader, &exit)?;
|
||||||
dbg!(i.elapsed());
|
dbg!(i.elapsed());
|
||||||
sleep(Duration::from_secs(10));
|
sleep(Duration::from_secs(10));
|
||||||
}
|
}
|
||||||
})?
|
|
||||||
.join()
|
|
||||||
.unwrap()
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -124,20 +124,12 @@ impl Indexer {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let mut readers = Readers::new(&self.vecs);
|
let mut readers = Readers::new(&self.vecs);
|
||||||
let mut already_added_addressbyteshash: FxHashMap<AddressBytesHash, TypeIndex> =
|
|
||||||
FxHashMap::default();
|
|
||||||
let mut same_block_spent_outpoints: FxHashSet<OutPoint> = FxHashSet::default();
|
|
||||||
let mut same_block_output_info: FxHashMap<OutPoint, (OutputType, TypeIndex)> =
|
|
||||||
FxHashMap::default();
|
|
||||||
|
|
||||||
let vecs = &mut self.vecs;
|
let vecs = &mut self.vecs;
|
||||||
let stores = &mut self.stores;
|
let stores = &mut self.stores;
|
||||||
|
|
||||||
for block in blocks.after(prev_hash)? {
|
for block in blocks.after(prev_hash)? {
|
||||||
// let i_tot = Instant::now();
|
// let i_tot = Instant::now();
|
||||||
already_added_addressbyteshash.clear();
|
|
||||||
same_block_spent_outpoints.clear();
|
|
||||||
same_block_output_info.clear();
|
|
||||||
|
|
||||||
let height = block.height();
|
let height = block.height();
|
||||||
let blockhash = block.hash();
|
let blockhash = block.hash();
|
||||||
@@ -305,16 +297,19 @@ impl Indexer {
|
|||||||
// println!("txinindex_and_txindata = : {:?}", i.elapsed());
|
// println!("txinindex_and_txindata = : {:?}", i.elapsed());
|
||||||
|
|
||||||
// let i = Instant::now();
|
// let i = Instant::now();
|
||||||
same_block_spent_outpoints.extend(txins.iter().filter_map(|(_, input_source)| {
|
let same_block_spent_outpoints: FxHashSet<OutPoint> = txins
|
||||||
let InputSource::SameBlock((_, _, _, outpoint)) = input_source else {
|
.iter()
|
||||||
return None;
|
.filter_map(|(_, input_source)| {
|
||||||
};
|
let InputSource::SameBlock((_, _, _, outpoint)) = input_source else {
|
||||||
if !outpoint.is_coinbase() {
|
return None;
|
||||||
Some(*outpoint)
|
};
|
||||||
} else {
|
if !outpoint.is_coinbase() {
|
||||||
None
|
Some(*outpoint)
|
||||||
}
|
} else {
|
||||||
}));
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
// println!("same_block_spent_outpoints = : {:?}", i.elapsed());
|
// println!("same_block_spent_outpoints = : {:?}", i.elapsed());
|
||||||
|
|
||||||
// let i = Instant::now();
|
// let i = Instant::now();
|
||||||
@@ -475,6 +470,10 @@ impl Indexer {
|
|||||||
let tx_len = block.txdata.len();
|
let tx_len = block.txdata.len();
|
||||||
|
|
||||||
// let i = Instant::now();
|
// let i = Instant::now();
|
||||||
|
let mut already_added_addressbyteshash: FxHashMap<AddressBytesHash, TypeIndex> =
|
||||||
|
FxHashMap::default();
|
||||||
|
let mut same_block_output_info: FxHashMap<OutPoint, (OutputType, TypeIndex)> =
|
||||||
|
FxHashMap::default();
|
||||||
txouts
|
txouts
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.try_for_each(|data| -> Result<()> {
|
.try_for_each(|data| -> Result<()> {
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ use brk_types::{
|
|||||||
TxOutIndex, TxidPrefix, TypeIndex, TypeIndexAndOutPoint, TypeIndexAndTxIndex, Unit, Version,
|
TxOutIndex, TxidPrefix, TypeIndex, TypeIndexAndOutPoint, TypeIndexAndTxIndex, Unit, Version,
|
||||||
Vout,
|
Vout,
|
||||||
};
|
};
|
||||||
use fjall2::{Keyspace, PersistMode};
|
use fjall2::{PersistMode, TransactionalKeyspace};
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
use vecdb::{AnyVec, StoredIndex, VecIterator};
|
use vecdb::{AnyVec, StoredIndex, VecIterator};
|
||||||
|
|
||||||
@@ -18,7 +18,7 @@ use super::Vecs;
|
|||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct Stores {
|
pub struct Stores {
|
||||||
pub keyspace: Keyspace,
|
pub keyspace: TransactionalKeyspace,
|
||||||
|
|
||||||
pub addressbyteshash_to_typeindex: Store<AddressBytesHash, TypeIndex>,
|
pub addressbyteshash_to_typeindex: Store<AddressBytesHash, TypeIndex>,
|
||||||
pub blockhashprefix_to_height: Store<BlockHashPrefix, Height>,
|
pub blockhashprefix_to_height: Store<BlockHashPrefix, Height>,
|
||||||
|
|||||||
@@ -1,8 +1,11 @@
|
|||||||
|
use std::thread::sleep;
|
||||||
use std::{mem, sync::Arc, time::Duration};
|
use std::{mem, sync::Arc, time::Duration};
|
||||||
|
|
||||||
use bitcoin::block::Header;
|
use bitcoin::block::Header;
|
||||||
use bitcoin::consensus::encode;
|
use bitcoin::consensus::encode;
|
||||||
use bitcoincore_rpc::json::{GetBlockHeaderResult, GetBlockResult, GetTxOutResult};
|
use bitcoincore_rpc::json::{
|
||||||
|
GetBlockHeaderResult, GetBlockResult, GetBlockchainInfoResult, GetTxOutResult,
|
||||||
|
};
|
||||||
use bitcoincore_rpc::{Client as CoreClient, Error as RpcError, RpcApi};
|
use bitcoincore_rpc::{Client as CoreClient, Error as RpcError, RpcApi};
|
||||||
use brk_error::Result;
|
use brk_error::Result;
|
||||||
use brk_types::{BlockHash, Height, Sats, Transaction, TxIn, TxOut, TxStatus, Txid, Vout};
|
use brk_types::{BlockHash, Height, Sats, Transaction, TxIn, TxOut, TxStatus, Txid, Vout};
|
||||||
@@ -12,6 +15,7 @@ pub use bitcoincore_rpc::Auth;
|
|||||||
mod inner;
|
mod inner;
|
||||||
|
|
||||||
use inner::ClientInner;
|
use inner::ClientInner;
|
||||||
|
use log::info;
|
||||||
|
|
||||||
///
|
///
|
||||||
/// Bitcoin Core RPC Client
|
/// Bitcoin Core RPC Client
|
||||||
@@ -40,6 +44,13 @@ impl Client {
|
|||||||
)?)))
|
)?)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns a data structure containing various state info regarding
|
||||||
|
/// blockchain processing.
|
||||||
|
pub fn get_blockchain_info(&self) -> Result<GetBlockchainInfoResult> {
|
||||||
|
self.call(move |c| c.get_blockchain_info())
|
||||||
|
.map_err(Into::into)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn get_block<'a, H>(&self, hash: &'a H) -> Result<bitcoin::Block>
|
pub fn get_block<'a, H>(&self, hash: &'a H) -> Result<bitcoin::Block>
|
||||||
where
|
where
|
||||||
&'a H: Into<&'a bitcoin::BlockHash>,
|
&'a H: Into<&'a bitcoin::BlockHash>,
|
||||||
@@ -230,6 +241,22 @@ impl Client {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn wait_for_synced_node(&self) -> Result<()> {
|
||||||
|
let is_synced = || -> Result<bool> {
|
||||||
|
let info = self.get_blockchain_info()?;
|
||||||
|
Ok(info.headers == info.blocks)
|
||||||
|
};
|
||||||
|
|
||||||
|
if !is_synced()? {
|
||||||
|
info!("Waiting for node to sync...");
|
||||||
|
while !is_synced()? {
|
||||||
|
sleep(Duration::from_secs(1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn call<F, T>(&self, f: F) -> Result<T, RpcError>
|
pub fn call<F, T>(&self, f: F) -> Result<T, RpcError>
|
||||||
where
|
where
|
||||||
F: Fn(&CoreClient) -> Result<T, RpcError>,
|
F: Fn(&CoreClient) -> Result<T, RpcError>,
|
||||||
|
|||||||
@@ -1,4 +1,8 @@
|
|||||||
use std::{path::Path, thread::sleep, time::Duration};
|
use std::{
|
||||||
|
path::Path,
|
||||||
|
thread::{self, sleep},
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
|
||||||
use brk_computer::Computer;
|
use brk_computer::Computer;
|
||||||
|
|
||||||
@@ -10,14 +14,29 @@ use brk_query::Query;
|
|||||||
use brk_reader::Reader;
|
use brk_reader::Reader;
|
||||||
use brk_rpc::{Auth, Client};
|
use brk_rpc::{Auth, Client};
|
||||||
use brk_server::Server;
|
use brk_server::Server;
|
||||||
|
use log::info;
|
||||||
use vecdb::Exit;
|
use vecdb::Exit;
|
||||||
|
|
||||||
pub fn main() -> Result<()> {
|
pub fn main() -> Result<()> {
|
||||||
|
// Can't increase main thread's stack size, thus we need to use another thread
|
||||||
|
thread::Builder::new()
|
||||||
|
.stack_size(512 * 1024 * 1024)
|
||||||
|
.spawn(run)?
|
||||||
|
.join()
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run() -> Result<()> {
|
||||||
brk_logger::init(Some(Path::new(".log")))?;
|
brk_logger::init(Some(Path::new(".log")))?;
|
||||||
|
|
||||||
let process = true;
|
let bitcoin_dir = Path::new(&std::env::var("HOME").unwrap())
|
||||||
|
.join("Library")
|
||||||
|
.join("Application Support")
|
||||||
|
.join("Bitcoin");
|
||||||
|
// let bitcoin_dir = Path::new("/Volumes/WD_BLACK1/bitcoin");
|
||||||
|
|
||||||
let bitcoin_dir = Path::new("");
|
let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
|
||||||
|
// let outputs_dir = Path::new("../../_outputs");
|
||||||
|
|
||||||
let client = Client::new(
|
let client = Client::new(
|
||||||
"http://localhost:8332",
|
"http://localhost:8332",
|
||||||
@@ -28,46 +47,48 @@ pub fn main() -> Result<()> {
|
|||||||
|
|
||||||
let blocks = Blocks::new(&client, &reader);
|
let blocks = Blocks::new(&client, &reader);
|
||||||
|
|
||||||
let outputs_dir = Path::new("../../_outputs");
|
let mut indexer = Indexer::forced_import(&outputs_dir)?;
|
||||||
|
|
||||||
let mut indexer = Indexer::forced_import(outputs_dir)?;
|
|
||||||
|
|
||||||
let fetcher = Some(Fetcher::import(true, None)?);
|
let fetcher = Some(Fetcher::import(true, None)?);
|
||||||
|
|
||||||
let mut computer = Computer::forced_import(outputs_dir, &indexer, fetcher)?;
|
let mut computer = Computer::forced_import(&outputs_dir, &indexer, fetcher)?;
|
||||||
|
|
||||||
let exit = Exit::new();
|
let exit = Exit::new();
|
||||||
exit.set_ctrlc_handler();
|
exit.set_ctrlc_handler();
|
||||||
|
|
||||||
tokio::runtime::Builder::new_multi_thread()
|
let query = Query::build(&reader, &indexer, &computer);
|
||||||
|
|
||||||
|
let future = async move {
|
||||||
|
let server = Server::new(&query, None);
|
||||||
|
|
||||||
|
tokio::spawn(async move {
|
||||||
|
server.serve(true).await.unwrap();
|
||||||
|
});
|
||||||
|
|
||||||
|
Ok(()) as Result<()>
|
||||||
|
};
|
||||||
|
|
||||||
|
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||||
.enable_all()
|
.enable_all()
|
||||||
.build()?
|
.build()?;
|
||||||
.block_on(async {
|
|
||||||
let query = Query::build(&reader, &indexer, &computer);
|
|
||||||
|
|
||||||
let server = Server::new(&query, None);
|
let _handle = runtime.spawn(future);
|
||||||
|
|
||||||
let server = tokio::spawn(async move {
|
loop {
|
||||||
server.serve(true).await.unwrap();
|
client.wait_for_synced_node()?;
|
||||||
});
|
|
||||||
|
|
||||||
if process {
|
let last_height = client.get_last_height()?;
|
||||||
loop {
|
|
||||||
let last_height = client.get_last_height()?;
|
|
||||||
|
|
||||||
let starting_indexes = indexer.checked_index(&blocks, &client, &exit)?;
|
info!("{} blocks found.", u32::from(last_height) + 1);
|
||||||
|
|
||||||
computer.compute(&indexer, starting_indexes, &reader, &exit)?;
|
let starting_indexes = indexer.checked_index(&blocks, &client, &exit)?;
|
||||||
|
|
||||||
while last_height == client.get_last_height()? {
|
computer.compute(&indexer, starting_indexes, &reader, &exit)?;
|
||||||
sleep(Duration::from_secs(1))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[allow(unreachable_code)]
|
info!("Waiting for new blocks...");
|
||||||
server.await.unwrap();
|
|
||||||
|
|
||||||
Ok(())
|
while last_height == client.get_last_height()? {
|
||||||
})
|
sleep(Duration::from_secs(1))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ use std::{
|
|||||||
|
|
||||||
use brk_error::Result;
|
use brk_error::Result;
|
||||||
use brk_types::Version;
|
use brk_types::Version;
|
||||||
use fjall2::{Keyspace, PartitionHandle, PersistMode};
|
use fjall2::{PersistMode, TransactionalKeyspace, TransactionalPartitionHandle};
|
||||||
|
|
||||||
use super::Height;
|
use super::Height;
|
||||||
|
|
||||||
@@ -18,13 +18,13 @@ pub struct StoreMeta {
|
|||||||
|
|
||||||
impl StoreMeta {
|
impl StoreMeta {
|
||||||
pub fn checked_open<F>(
|
pub fn checked_open<F>(
|
||||||
keyspace: &Keyspace,
|
keyspace: &TransactionalKeyspace,
|
||||||
path: &Path,
|
path: &Path,
|
||||||
version: Version,
|
version: Version,
|
||||||
open_partition_handle: F,
|
open_partition_handle: F,
|
||||||
) -> Result<(Self, PartitionHandle)>
|
) -> Result<(Self, TransactionalPartitionHandle)>
|
||||||
where
|
where
|
||||||
F: Fn() -> Result<PartitionHandle>,
|
F: Fn() -> Result<TransactionalPartitionHandle>,
|
||||||
{
|
{
|
||||||
fs::create_dir_all(path)?;
|
fs::create_dir_all(path)?;
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
use std::{borrow::Cow, cmp, fmt::Debug, fs, hash::Hash, path::Path};
|
use std::{borrow::Cow, cmp, fmt::Debug, fs, hash::Hash, mem, path::Path};
|
||||||
|
|
||||||
use brk_error::Result;
|
use brk_error::Result;
|
||||||
use brk_types::{Height, Version};
|
use brk_types::{Height, Version};
|
||||||
use byteview6::ByteView;
|
use byteview6::ByteView;
|
||||||
use fjall2::{
|
use fjall2::{
|
||||||
InnerItem, Keyspace, PartitionCreateOptions, PartitionHandle, PersistMode, ValueType,
|
InnerItem, PartitionCreateOptions, PersistMode, TransactionalKeyspace,
|
||||||
|
TransactionalPartitionHandle, ValueType,
|
||||||
};
|
};
|
||||||
use rustc_hash::{FxHashMap, FxHashSet};
|
use rustc_hash::{FxHashMap, FxHashSet};
|
||||||
|
|
||||||
@@ -18,18 +19,18 @@ use meta::*;
|
|||||||
pub struct StoreV2<Key, Value> {
|
pub struct StoreV2<Key, Value> {
|
||||||
meta: StoreMeta,
|
meta: StoreMeta,
|
||||||
name: &'static str,
|
name: &'static str,
|
||||||
keyspace: Keyspace,
|
keyspace: TransactionalKeyspace,
|
||||||
partition: PartitionHandle,
|
partition: TransactionalPartitionHandle,
|
||||||
puts: FxHashMap<Key, Value>,
|
puts: FxHashMap<Key, Value>,
|
||||||
dels: FxHashSet<Key>,
|
dels: FxHashSet<Key>,
|
||||||
}
|
}
|
||||||
|
|
||||||
const MAJOR_FJALL_VERSION: Version = Version::TWO;
|
const MAJOR_FJALL_VERSION: Version = Version::TWO;
|
||||||
|
|
||||||
pub fn open_keyspace(path: &Path) -> fjall2::Result<Keyspace> {
|
pub fn open_keyspace(path: &Path) -> fjall2::Result<TransactionalKeyspace> {
|
||||||
fjall2::Config::new(path.join("fjall"))
|
fjall2::Config::new(path.join("fjall"))
|
||||||
.max_write_buffer_size(32 * 1024 * 1024)
|
.max_write_buffer_size(32 * 1024 * 1024)
|
||||||
.open()
|
.open_transactional()
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<K, V> StoreV2<K, V>
|
impl<K, V> StoreV2<K, V>
|
||||||
@@ -39,23 +40,25 @@ where
|
|||||||
ByteView: From<K> + From<V>,
|
ByteView: From<K> + From<V>,
|
||||||
{
|
{
|
||||||
fn open_partition_handle(
|
fn open_partition_handle(
|
||||||
keyspace: &Keyspace,
|
keyspace: &TransactionalKeyspace,
|
||||||
name: &str,
|
name: &str,
|
||||||
bloom_filters: Option<bool>,
|
bloom_filters: Option<bool>,
|
||||||
) -> Result<PartitionHandle> {
|
) -> Result<TransactionalPartitionHandle> {
|
||||||
let mut options = PartitionCreateOptions::default()
|
let mut options = PartitionCreateOptions::default()
|
||||||
.max_memtable_size(8 * 1024 * 1024)
|
.max_memtable_size(8 * 1024 * 1024)
|
||||||
.manual_journal_persist(true);
|
.manual_journal_persist(true);
|
||||||
|
|
||||||
if bloom_filters.is_some_and(|b| !b) {
|
if bloom_filters.is_some_and(|b| !b) {
|
||||||
options = options.bloom_filter_bits(None);
|
options = options.bloom_filter_bits(None);
|
||||||
|
} else {
|
||||||
|
options = options.bloom_filter_bits(Some(5));
|
||||||
}
|
}
|
||||||
|
|
||||||
keyspace.open_partition(name, options).map_err(|e| e.into())
|
keyspace.open_partition(name, options).map_err(|e| e.into())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn import(
|
pub fn import(
|
||||||
keyspace: &Keyspace,
|
keyspace: &TransactionalKeyspace,
|
||||||
path: &Path,
|
path: &Path,
|
||||||
name: &str,
|
name: &str,
|
||||||
version: Version,
|
version: Version,
|
||||||
@@ -100,12 +103,16 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_empty(&self) -> Result<bool> {
|
pub fn is_empty(&self) -> Result<bool> {
|
||||||
self.partition.is_empty().map_err(|e| e.into())
|
self.keyspace
|
||||||
|
.read_tx()
|
||||||
|
.is_empty(&self.partition)
|
||||||
|
.map_err(|e| e.into())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn iter(&self) -> impl Iterator<Item = (K, V)> {
|
pub fn iter(&self) -> impl Iterator<Item = (K, V)> {
|
||||||
self.partition
|
self.keyspace
|
||||||
.iter()
|
.read_tx()
|
||||||
|
.iter(&self.partition)
|
||||||
.map(|res| res.unwrap())
|
.map(|res| res.unwrap())
|
||||||
.map(|(k, v)| (K::from(ByteView::from(&*k)), V::from(ByteView::from(&*v))))
|
.map(|(k, v)| (K::from(ByteView::from(&*k)), V::from(ByteView::from(&*v))))
|
||||||
}
|
}
|
||||||
@@ -140,6 +147,11 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn approximate_len(&self) -> usize {
|
||||||
|
self.partition.approximate_len()
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
fn has(&self, height: Height) -> bool {
|
fn has(&self, height: Height) -> bool {
|
||||||
self.meta.has(height)
|
self.meta.has(height)
|
||||||
@@ -169,16 +181,19 @@ where
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut items = self
|
let mut items = mem::take(&mut self.puts)
|
||||||
.puts
|
.into_iter()
|
||||||
.drain()
|
|
||||||
.map(|(key, value)| Item::Value { key, value })
|
.map(|(key, value)| Item::Value { key, value })
|
||||||
.chain(self.dels.drain().map(|key| Item::Tomb(key)))
|
.chain(
|
||||||
|
mem::take(&mut self.dels)
|
||||||
|
.into_iter()
|
||||||
|
.map(|key| Item::Tomb(key)),
|
||||||
|
)
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
items.sort_unstable();
|
items.sort_unstable();
|
||||||
|
|
||||||
self.keyspace.batch().commit_partition(
|
self.keyspace.inner().batch().commit_partition(
|
||||||
&self.partition,
|
self.partition.inner(),
|
||||||
items.into_iter().map(InnerItem::from).collect::<Vec<_>>(),
|
items.into_iter().map(InnerItem::from).collect::<Vec<_>>(),
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user