From d2ca6f1d46ad0036f75a001fc5b4e03d2b062d90 Mon Sep 17 00:00:00 2001 From: nym21 Date: Wed, 5 Mar 2025 16:55:37 +0100 Subject: [PATCH] server: use query for search --- Cargo.lock | 118 +++++++++++++------------- Cargo.toml | 2 +- crates/brk_cli/src/query.rs | 8 +- crates/brk_computer/src/lib.rs | 3 - crates/brk_query/src/format.rs | 5 ++ crates/brk_query/src/index.rs | 2 +- crates/brk_query/src/lib.rs | 13 ++- crates/brk_server/Cargo.toml | 2 +- crates/brk_server/src/api/vecs/mod.rs | 103 +++++----------------- crates/brk_server/src/lib.rs | 3 + 10 files changed, 104 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 45b1a7c04..012011df9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -374,7 +374,7 @@ dependencies = [ [[package]] name = "brk" -version = "0.0.6" +version = "0.0.7" dependencies = [ "brk_computer", "brk_core", @@ -390,7 +390,7 @@ dependencies = [ [[package]] name = "brk_cli" -version = "0.0.6" +version = "0.0.7" dependencies = [ "brk_computer", "brk_core", @@ -410,7 +410,7 @@ dependencies = [ [[package]] name = "brk_computer" -version = "0.0.6" +version = "0.0.7" dependencies = [ "brk_core", "brk_exit", @@ -424,7 +424,7 @@ dependencies = [ [[package]] name = "brk_core" -version = "0.0.6" +version = "0.0.7" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -441,7 +441,7 @@ dependencies = [ [[package]] name = "brk_exit" -version = "0.0.6" +version = "0.0.7" dependencies = [ "ctrlc", "log", @@ -449,7 +449,7 @@ dependencies = [ [[package]] name = "brk_fetcher" -version = "0.0.6" +version = "0.0.7" dependencies = [ "brk_core", "brk_logger", @@ -462,7 +462,7 @@ dependencies = [ [[package]] name = "brk_indexer" -version = "0.0.6" +version = "0.0.7" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -481,7 +481,7 @@ dependencies = [ [[package]] name = "brk_logger" -version = "0.0.6" +version = "0.0.7" dependencies = [ "color-eyre", "env_logger", @@ -490,7 +490,7 @@ dependencies = [ [[package]] name = "brk_parser" -version = "0.0.6" +version = "0.0.7" dependencies = [ "bitcoin", "bitcoincore-rpc", @@ -505,7 +505,7 @@ dependencies = [ [[package]] name = "brk_query" -version = "0.0.6" +version = "0.0.7" dependencies = [ "brk_computer", "brk_indexer", @@ -521,7 +521,7 @@ dependencies = [ [[package]] name = "brk_server" -version = "0.0.6" +version = "0.0.7" dependencies = [ "axum", "brk_computer", @@ -547,7 +547,7 @@ dependencies = [ [[package]] name = "brk_vec" -version = "0.0.6" +version = "0.0.7" dependencies = [ "brk_exit", "memmap2", @@ -601,9 +601,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "byteview" @@ -1773,9 +1773,9 @@ checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" [[package]] name = "oxc" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24855da39b565013176401474f89408479b6086f81bc0d670e3a318bc985b158" +checksum = "ea36ba3258ba8c21e44927caf3af5c6ac6b4c3ac4b38c3c0660564b796ffa08e" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1816,9 +1816,9 @@ dependencies = [ [[package]] name = "oxc_allocator" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b18e24923867bc335264d0ee98b60263dabd50ff19983d781f6a5ce0ca964429" +checksum = "eef1633a33a1a8ec5e88c877aa50c8db61e2e341dfe8e3ffeda87ab4b493afa9" dependencies = [ "allocator-api2", "bumpalo", @@ -1829,14 +1829,15 @@ dependencies = [ [[package]] name = "oxc_ast" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded17501d4e1535e765d2e5cadf28cad03ec87d39931d1481b9e34f733b98474" +checksum = "87589abc25ec0975b4f22c05f55ee8546e08743950adc262e94e90db9ebc8262" dependencies = [ "bitflags", "cow-utils", "oxc_allocator", "oxc_ast_macros", + "oxc_data_structures", "oxc_estree", "oxc_regular_expression", "oxc_span", @@ -1845,9 +1846,9 @@ dependencies = [ [[package]] name = "oxc_ast_macros" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade1cdc274922b8246436c8722afb70d07bf18d4b126354a21cbca1796dab041" +checksum = "631d65bd4d6b16d3e56ff19754282ef094cb5427648495d23f39c96d6cdc523e" dependencies = [ "proc-macro2", "quote", @@ -1856,9 +1857,9 @@ dependencies = [ [[package]] name = "oxc_ast_visit" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1222c2d9e93ee199bb4f9919c3d8a99c5ffe8db2a3a9e67fa0c4f5c54634d5f4" +checksum = "da5e14e31c9d02657061db04405dd1071724d262ded4574a74a3874cb44fcca2" dependencies = [ "oxc_allocator", "oxc_ast", @@ -1868,9 +1869,9 @@ dependencies = [ [[package]] name = "oxc_cfg" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a8980b4c13c150ca80e9896392db78cd5d2c250675fcfa4c8d89972dfa60126" +checksum = "fa3fe7457827a2bbe13e3bdccd7fd93ee502d00a6240ebe0ca65e9ebf7166aee" dependencies = [ "bitflags", "itertools", @@ -1883,9 +1884,9 @@ dependencies = [ [[package]] name = "oxc_codegen" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187c5ab8a0a7f6e2ea4da99f0bc22645133ff0192f6385561e7050d04796c4a4" +checksum = "32f9c00cb3d0d02ac9ee802048fc37bd00155ec21f588cf3b3a5a0884e6331ab" dependencies = [ "bitflags", "cow-utils", @@ -1904,9 +1905,9 @@ dependencies = [ [[package]] name = "oxc_data_structures" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59020f1b37242b5041cb039898c0234f22ebe193910622968552353d74ed885e" +checksum = "53b4292a782b93493456f56ec35ba09f41680b2cb7d0294dd7b39f664fb6dd4d" dependencies = [ "assert-unchecked", "ropey", @@ -1914,9 +1915,9 @@ dependencies = [ [[package]] name = "oxc_diagnostics" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa9d41fe75156b30adcfbce4cba8b7e3316c1e650da0c90f90cd1923980d4de5" +checksum = "4c00829ead1c6ac5344a575207214a53003fab41dd0ac8a5e35bfb57aedfde2f" dependencies = [ "cow-utils", "oxc-miette", @@ -1924,9 +1925,9 @@ dependencies = [ [[package]] name = "oxc_ecmascript" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea8a969324e2fb6ba5c48f1f48125ef283df9aa66c69a6a9f88ce66c2017d0fb" +checksum = "488a6e543ab952ca3efb72bcefe9250d34777d7010955bfac3978c7e03bf4613" dependencies = [ "cow-utils", "num-bigint", @@ -1938,9 +1939,9 @@ dependencies = [ [[package]] name = "oxc_estree" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dbd7bb1b12e006fd02c7d61a1aece96e4c6336c9b9e8d568ad5f6dfd4c315c2" +checksum = "78290e1c47759f1b7a6eb871e99adbb12054e817a052cacc898dfc89183f444b" [[package]] name = "oxc_index" @@ -1950,14 +1951,15 @@ checksum = "2fa07b0cfa997730afed43705766ef27792873fdf5215b1391949fec678d2392" [[package]] name = "oxc_mangler" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a297a0c747bb800bf6ac05fd5c281cf045d9cd7b65b5f7c212f65834b85fac9" +checksum = "878c8f316cd0af5e85eabd8b65085f9b415b71f76b5f6b564992d2a554bf1f44" dependencies = [ "fixedbitset", "itertools", "oxc_allocator", "oxc_ast", + "oxc_data_structures", "oxc_index", "oxc_semantic", "oxc_span", @@ -1966,9 +1968,9 @@ dependencies = [ [[package]] name = "oxc_minifier" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52fa721e44be1eb5e1231cccce0d1e03aba06e3515d59107608cb23498ce9fad" +checksum = "11a5af6dd1b470715c12aae1428789faa174d9bdbee300c66ae77deaea34a13e" dependencies = [ "cow-utils", "oxc_allocator", @@ -1988,9 +1990,9 @@ dependencies = [ [[package]] name = "oxc_parser" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc8294a24383bcb7f518604769b07915a14c21b53077725a272736ef90b3c99" +checksum = "1227c203986a3b000cab2f9ef4adc613cf61a39e2f3902bbef6f9072f41c833d" dependencies = [ "assert-unchecked", "bitflags", @@ -2011,9 +2013,9 @@ dependencies = [ [[package]] name = "oxc_regular_expression" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e84b8e555281da67deac9b6a88d6f95e390f9fe692fa8f8a6868011076a2ee33" +checksum = "64f053a6df456bf575cbf3c38ffea3013799748012d4a7c9ecd70d3f47a214d4" dependencies = [ "oxc_allocator", "oxc_ast_macros", @@ -2027,9 +2029,9 @@ dependencies = [ [[package]] name = "oxc_semantic" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb18fa7e368a33a54bd7dc4730de56cd8e26bca6cbffc3798811645e63f3a57" +checksum = "56322936cd15d784b4bb82a92eeca754ee8dde3c499f3165eef227aebdcfff98" dependencies = [ "assert-unchecked", "itertools", @@ -2064,9 +2066,9 @@ dependencies = [ [[package]] name = "oxc_span" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704d1f7229aafcd1291dacd1d39927cd9fd77ac3c05e6edca32e25f52329dcd5" +checksum = "680170488923ec30f75228b93c3b08d42754e983bca138063c973ca5ba0e71ff" dependencies = [ "compact_str", "oxc-miette", @@ -2077,9 +2079,9 @@ dependencies = [ [[package]] name = "oxc_syntax" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2697e97541b2b8f8b95152ea00931d73b254bbcb2eb590192ebc4e76a7748d3f" +checksum = "abe1784332acdab786ae3904930a7274a84eda184144202490e65f7a0e2db436" dependencies = [ "assert-unchecked", "bitflags", @@ -2098,9 +2100,9 @@ dependencies = [ [[package]] name = "oxc_traverse" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c70727b18b6e5b66d068e5e282a44674bf8dd72f39d9c503d665a0b61e9b5667" +checksum = "55ee3adf17f4dd4819d333f7952e7d644f24e433c9fdf5ceaadc93bb9f57a36c" dependencies = [ "compact_str", "itoa", @@ -2909,9 +2911,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.37" +version = "0.3.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +checksum = "bb041120f25f8fbe8fd2dbe4671c7c2ed74d83be2e7a77529bf7e0790ae3f472" dependencies = [ "deranged", "itoa", @@ -2924,15 +2926,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "765c97a5b985b7c11d7bc27fa927dc4fe6af3a6dfb021d28deb60d3bf51e76ef" [[package]] name = "time-macros" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +checksum = "e8093bc3e81c3bc5f7879de09619d06c9a5a5e45ca44dfeeb7225bae38005c5c" dependencies = [ "num-conv", "time-core", diff --git a/Cargo.toml b/Cargo.toml index 80d45c372..9d6f2c5c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["crates/*"] package.description = "The Bitcoin Research Kit is a suite of tools designed to extract, compute and display data stored on a Bitcoin Core node" package.license = "MIT" package.edition = "2024" -package.version = "0.0.6" +package.version = "0.0.7" package.repository = "https://github.com/bitcoinresearchkit/brk" [workspace.dependencies] diff --git a/crates/brk_cli/src/query.rs b/crates/brk_cli/src/query.rs index fd90148c9..328b2b15f 100644 --- a/crates/brk_cli/src/query.rs +++ b/crates/brk_cli/src/query.rs @@ -16,14 +16,10 @@ pub fn query(params: QueryParams) -> color_eyre::Result<()> { let query = Query::build(&indexer, &computer); - let ids = params - .values - .iter() - .flat_map(|v| v.split(",")) - .collect::>(); - let index = Index::try_from(params.index.as_str())?; + let ids = params.values.iter().map(|s| s.as_str()).collect::>(); + let res = query.search(index, &ids, params.from, params.to, params.format)?; if params.format.is_some() { diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 8d877a368..1e90c5d51 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -58,9 +58,6 @@ impl Computer { let txinindexes_count = indexer.vecs().txinindex_to_txoutindex.len(); let txoutindexes_count = indexer.vecs().txoutindex_to_addressindex.len(); - // TODO: Remove all outdated - - // self.vecs.txindex_to_last_txinindex.compute_last_index_from_first( // starting_indexes.txindex, // &mut indexer.vecs().txindex_to_first_txinindex, // txinindexes_count, diff --git a/crates/brk_query/src/format.rs b/crates/brk_query/src/format.rs index e8972d072..137dd7cae 100644 --- a/crates/brk_query/src/format.rs +++ b/crates/brk_query/src/format.rs @@ -5,9 +5,14 @@ use serde::Deserialize; #[allow(clippy::upper_case_acronyms)] #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum, Deserialize)] pub enum Format { + #[serde(alias = "json")] JSON, + #[serde(alias = "csv")] CSV, + #[serde(alias = "tsv")] TSV, + #[serde(alias = "md", alias = "markdown")] + #[value(alias("markdown"))] MD, } diff --git a/crates/brk_query/src/index.rs b/crates/brk_query/src/index.rs index 3fedd6677..e8e0d492a 100644 --- a/crates/brk_query/src/index.rs +++ b/crates/brk_query/src/index.rs @@ -68,7 +68,7 @@ impl Index { impl TryFrom<&str> for Index { type Error = color_eyre::Report; fn try_from(value: &str) -> Result { - Ok(match value { + Ok(match value.to_lowercase().as_str() { v if (Self::Dateindex).possible_values().contains(&v) => Self::Dateindex, v if (Self::Height).possible_values().contains(&v) => Self::Height, v if (Self::Txindex).possible_values().contains(&v) => Self::Txindex, diff --git a/crates/brk_query/src/lib.rs b/crates/brk_query/src/lib.rs index 3536cbacd..78d359f93 100644 --- a/crates/brk_query/src/lib.rs +++ b/crates/brk_query/src/lib.rs @@ -60,8 +60,17 @@ impl<'a> Query<'a> { ) -> color_eyre::Result { let tuples = ids .iter() - .map(|s| { - let mut id = s.to_lowercase().replace("_", "-"); + .flat_map(|s| { + s.to_lowercase() + .replace("_", "-") + .split_whitespace() + .flat_map(|s| { + s.split(',') + .flat_map(|s| s.split('+').map(|s| s.to_string())) + }) + .collect::>() + }) + .map(|mut id| { let mut res = self.vecid_to_index_to_vec.get(&id); if res.is_none() { if let Ok(index) = Index::try_from(id.as_str()) { diff --git a/crates/brk_server/Cargo.toml b/crates/brk_server/Cargo.toml index fc9960b7e..70f886ac0 100644 --- a/crates/brk_server/Cargo.toml +++ b/crates/brk_server/Cargo.toml @@ -21,7 +21,7 @@ color-eyre = { workspace = true } jiff = { workspace = true } log = { workspace = true } minreq = { workspace = true } -oxc = { version = "0.54.0", features = ["codegen", "minifier"] } +oxc = { version = "0.55.0", features = ["codegen", "minifier"] } serde = { workspace = true } serde_json = { workspace = true } tokio = { version = "1.43.0", features = ["full"] } diff --git a/crates/brk_server/src/api/vecs/mod.rs b/crates/brk_server/src/api/vecs/mod.rs index adafcd2c8..7aacbad3f 100644 --- a/crates/brk_server/src/api/vecs/mod.rs +++ b/crates/brk_server/src/api/vecs/mod.rs @@ -6,7 +6,7 @@ use axum::{ http::{HeaderMap, StatusCode, Uri}, response::{IntoResponse, Response}, }; -use brk_query::{Format, Index, Params}; +use brk_query::{Format, Index, Output, Params}; use color_eyre::eyre::eyre; use serde_json::Value; @@ -34,7 +34,8 @@ pub async fn handler( response } Err(error) => { - let mut response = (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(); + let mut response = + (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(); log_result(response.status(), path, instant); response.headers_mut().insert_cors(); response @@ -53,89 +54,25 @@ fn req_to_response_res( }): AxumQuery, AppState { query, .. }: AppState, ) -> color_eyre::Result { - let indexes = index - .to_lowercase() - .split(",") - .flat_map(|s| Index::try_from(s).ok()) - .collect::>(); + let index = Index::try_from(index.as_str())?; - if indexes.len() > 1 { - return Err(eyre!("Multiple indexes aren't supported")); - } else if indexes.is_empty() { - return Err(eyre!("Unknown index")); - } + let output = query.search( + index, + &values.iter().map(|v| v.as_str()).collect::>(), + from, + to, + format, + )?; - let ids = values - .into_iter() - .map(|v| v.to_lowercase()) - .flat_map(|v| v.split(",").map(|v| v.to_owned()).collect::>()) - .map(|s| { - let opt = query.vecid_to_index_to_vec.get(&s.replace("_", "-")); - (s, opt) - }) - .filter(|(_, opt)| opt.is_some()) - .map(|(id, vec)| (id, vec.unwrap())) - .collect::>(); - - if ids.is_empty() { - return Ok(Json(()).into_response()); - } - - let values = ids - .iter() - .flat_map(|(_, i_to_v)| i_to_v.get(indexes.first().unwrap())) - .map(|vec| -> brk_vec::Result> { vec.collect_range_values(from, to) }) - .collect::>>()?; - - if ids.is_empty() { - return Ok(Json(()).into_response()); - } - - let ids_last_i = ids.len() - 1; - - let mut response = match format { - Some(Format::CSV) | Some(Format::TSV) => { - let delimiter = if format == Some(Format::CSV) { ',' } else { '\t' }; - - let mut csv = ids - .into_iter() - .map(|(id, _)| id) - .collect::>() - .join(&delimiter.to_string()); - - csv.push('\n'); - - let values_len = values.first().unwrap().len(); - - (0..values_len).for_each(|i| { - let mut line = "".to_string(); - values.iter().enumerate().for_each(|(id_i, v)| { - line += &v.get(i).unwrap().to_string(); - if id_i == ids_last_i { - line.push('\n'); - } else { - line.push(delimiter); - } - }); - csv += &line; - }); - - csv.into_response() - } - Some(Format::MD) => "".into_response(), - Some(Format::JSON) | None => { - if values.len() == 1 { - let values = values.first().unwrap(); - if values.len() == 1 { - let value = values.first().unwrap(); - Json(value).into_response() - } else { - Json(values).into_response() - } - } else { - Json(values).into_response() - } - } + let mut response = match output { + Output::CSV(s) => s.into_response(), + Output::TSV(s) => s.into_response(), + Output::Json(v) => match v { + brk_query::Value::Single(v) => Json(v).into_response(), + brk_query::Value::List(l) => Json(l).into_response(), + brk_query::Value::Matrix(m) => Json(m).into_response(), + }, + Output::MD(s) => s.into_response(), }; let headers = response.headers_mut(); diff --git a/crates/brk_server/src/lib.rs b/crates/brk_server/src/lib.rs index 432387d30..6b3d589bf 100644 --- a/crates/brk_server/src/lib.rs +++ b/crates/brk_server/src/lib.rs @@ -40,6 +40,9 @@ const DEV_PATH: &str = "../.."; const DOWNLOADS: &str = "downloads"; const WEBSITES: &str = "websites"; +// TODO +pub struct Server; + pub async fn main( indexer: Indexer, computer: Computer,