diff --git a/Cargo.lock b/Cargo.lock index 917948b15..888de5c84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -352,6 +352,8 @@ dependencies = [ "clap", "color-eyre", "log", + "tabled", + "terminal_size", ] [[package]] @@ -461,6 +463,7 @@ dependencies = [ "derive_deref", "serde", "serde_json", + "tabled", ] [[package]] @@ -526,6 +529,12 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "bytecount" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" + [[package]] name = "byteorder" version = "1.5.0" @@ -1749,6 +1758,17 @@ dependencies = [ "rustc-hash", ] +[[package]] +name = "papergrid" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b915f831b85d984193fdc3d3611505871dc139b2534530fa01c1a6a6707b6723" +dependencies = [ + "bytecount", + "fnv", + "unicode-width 0.2.0", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -1890,6 +1910,28 @@ dependencies = [ "zerocopy 0.7.35", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.98", +] + [[package]] name = "proc-macro2" version = "1.0.93" @@ -2332,6 +2374,29 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +[[package]] +name = "tabled" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121d8171ee5687a4978d1b244f7d99c43e7385a272185a2f1e1fa4dc0979d444" +dependencies = [ + "papergrid", + "tabled_derive", +] + +[[package]] +name = "tabled_derive" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d9946811baad81710ec921809e2af67ad77719418673b2a3794932d57b7538" +dependencies = [ + "heck", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.98", +] + [[package]] name = "tempfile" version = "3.17.1" @@ -2346,6 +2411,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "terminal_size" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5352447f921fda68cf61b4101566c0bdb5104eff6804d0678e5227580ab6a4e9" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "textwrap" version = "0.16.1" diff --git a/Cargo.toml b/Cargo.toml index 58c1b9f5e..cc440a989 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,4 +31,5 @@ minreq = { version = "2.13.2", features = ["https", "serde_json"] } rayon = "1.10.0" serde = { version = "1.0.218", features = ["derive"] } serde_json = { version = "1.0.139", features = ["float_roundtrip"] } +tabled = "0.18.0" zerocopy = { version = "0.8.21", features = ["derive"] } diff --git a/crates/brk_cli/Cargo.toml b/crates/brk_cli/Cargo.toml index 4d8e0a3ff..119ad066c 100644 --- a/crates/brk_cli/Cargo.toml +++ b/crates/brk_cli/Cargo.toml @@ -14,9 +14,11 @@ brk_logger = { workspace = true } brk_parser = { workspace = true } brk_query = { workspace = true } brk_server = { workspace = true } -clap = { workspace = true , features = ["string"] } +clap = { workspace = true, features = ["string"] } color-eyre = { workspace = true } log = { workspace = true } +tabled = { workspace = true } +terminal_size = "0.4.1" [[bin]] name = "brk" diff --git a/crates/brk_cli/src/main.rs b/crates/brk_cli/src/main.rs index 60b5f43f8..f64803d69 100644 --- a/crates/brk_cli/src/main.rs +++ b/crates/brk_cli/src/main.rs @@ -1,13 +1,14 @@ -use std::{path::Path, thread::sleep, time::Duration}; +use std::path::Path; use brk_computer::Computer; -use brk_exit::Exit; use brk_indexer::Indexer; -use brk_parser::rpc::{self, RpcApi}; -use brk_query::{Index, Params as QueryParams, Query}; -use brk_server::tokio; +use brk_query::Params as QueryArgs; use clap::{Parser, Subcommand}; -use log::info; +use query::query; +use run::{RunArgs, run}; + +mod query; +mod run; #[derive(Parser)] #[command(version, about)] @@ -19,13 +20,10 @@ struct Cli { #[derive(Subcommand, Debug)] enum Commands { + /// Run the indexer, computer and server Run(RunArgs), - Query(QueryParams), -} - -#[derive(Parser, Debug)] -struct RunArgs { - name: Option, + /// Query generated datasets via the `run` command in a similar fashion as the server's API + Query(QueryArgs), } fn main() -> color_eyre::Result<()> { @@ -37,67 +35,12 @@ fn main() -> color_eyre::Result<()> { let outputs_dir = Path::new("../../_outputs"); - let mut indexer = Indexer::import(&outputs_dir.join("indexed"))?; + let indexer = Indexer::import(&outputs_dir.join("indexed"))?; - let mut computer = Computer::import(&outputs_dir.join("computed"))?; + let computer = Computer::import(&outputs_dir.join("computed"))?; match &cli.command { - Commands::Run(_) => { - let data_dir = Path::new("../../../bitcoin"); - let rpc = Box::leak(Box::new(rpc::Client::new( - "http://localhost:8332", - rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")), - )?)); - let exit = Exit::new(); - - let parser = brk_parser::Parser::new(data_dir, rpc); - - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build()? - .block_on(async { - let served_indexer = indexer.clone(); - let served_computer = computer.clone(); - - tokio::spawn(async move { - brk_server::main(served_indexer, served_computer).await.unwrap(); - }); - - loop { - let block_count = rpc.get_block_count()?; - - info!("{block_count} blocks found."); - - let starting_indexes = indexer.index(&parser, rpc, &exit)?; - - computer.compute(&mut indexer, starting_indexes, &exit)?; - - info!("Waiting for new blocks..."); - - while block_count == rpc.get_block_count()? { - sleep(Duration::from_secs(1)) - } - } - - #[allow(unreachable_code)] - Ok(()) - }) - } - Commands::Query(args) => { - let query = Query::build(&indexer, &computer); - - println!( - "{}", - query.search( - Index::try_from(args.index.as_str())?, - &args.values.iter().flat_map(|v| v.split(",")).collect::>(), - args.from, - args.to, - args.format - )? - ); - - Ok(()) - } + Commands::Run(_) => run(indexer, computer), + Commands::Query(args) => query(indexer, computer, args), } } diff --git a/crates/brk_cli/src/query.rs b/crates/brk_cli/src/query.rs new file mode 100644 index 000000000..7eb41a112 --- /dev/null +++ b/crates/brk_cli/src/query.rs @@ -0,0 +1,40 @@ +use brk_computer::Computer; +use brk_indexer::Indexer; +use brk_query::{Index, Output, Params as QueryParams, Query, Tabled, Value}; +use tabled::settings::Style; + +pub fn query(indexer: Indexer, computer: Computer, params: &QueryParams) -> color_eyre::Result<()> { + let query = Query::build(&indexer, &computer); + + let ids = params.values.iter().flat_map(|v| v.split(",")).collect::>(); + + let index = Index::try_from(params.index.as_str())?; + + let res = query.search(index, &ids, params.from, params.to, params.format)?; + + if params.format.is_some() { + println!("{}", res); + } else { + println!( + "{}", + match res { + Output::Json(v) => match v { + Value::Single(v) => v.to_string(), + v => { + let v = match v { + Value::Single(_) => unreachable!("Already processed"), + Value::List(v) => vec![v], + Value::Matrix(v) => v, + }; + let mut table = v.to_table(ids.iter().map(|id| id.to_string()).collect::>()); + table.with(Style::psql()); + table.to_string() + } + }, + _ => unreachable!(), + } + ); + } + + Ok(()) +} diff --git a/crates/brk_cli/src/run.rs b/crates/brk_cli/src/run.rs new file mode 100644 index 000000000..608cf760b --- /dev/null +++ b/crates/brk_cli/src/run.rs @@ -0,0 +1,58 @@ +use std::{path::Path, thread::sleep, time::Duration}; + +use brk_computer::Computer; +use brk_exit::Exit; +use brk_indexer::Indexer; +use brk_parser::rpc::{self, RpcApi}; +use brk_server::tokio; +use clap::Parser; +use log::info; + +#[derive(Parser, Debug)] +pub struct RunArgs { + name: Option, +} + +pub fn run(mut indexer: Indexer, mut computer: Computer) -> color_eyre::Result<()> { + let data_dir = Path::new("../../../bitcoin"); + + let rpc = Box::leak(Box::new(rpc::Client::new( + "http://localhost:8332", + rpc::Auth::CookieFile(Path::new(data_dir).join(".cookie")), + )?)); + + let exit = Exit::new(); + + let parser = brk_parser::Parser::new(data_dir, rpc); + + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()? + .block_on(async { + let served_indexer = indexer.clone(); + let served_computer = computer.clone(); + + tokio::spawn(async move { + brk_server::main(served_indexer, served_computer).await.unwrap(); + }); + + loop { + let block_count = rpc.get_block_count()?; + + info!("{block_count} blocks found."); + + let starting_indexes = indexer.index(&parser, rpc, &exit)?; + + computer.compute(&mut indexer, starting_indexes, &exit)?; + + info!("Waiting for new blocks..."); + + while block_count == rpc.get_block_count()? { + sleep(Duration::from_secs(1)) + } + } + + #[allow(unreachable_code)] + Ok(()) + }) +} diff --git a/crates/brk_query/Cargo.toml b/crates/brk_query/Cargo.toml index c885007fa..ecbc4772b 100644 --- a/crates/brk_query/Cargo.toml +++ b/crates/brk_query/Cargo.toml @@ -15,3 +15,4 @@ color-eyre = { workspace = true } derive_deref = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +tabled = { workspace = true } diff --git a/crates/brk_query/src/format.rs b/crates/brk_query/src/format.rs index 32a4b8e87..e8972d072 100644 --- a/crates/brk_query/src/format.rs +++ b/crates/brk_query/src/format.rs @@ -8,6 +8,7 @@ pub enum Format { JSON, CSV, TSV, + MD, } impl TryFrom> for Format { @@ -16,7 +17,9 @@ impl TryFrom> for Format { if let Some(value) = value { let value = value.to_lowercase(); let value = value.as_str(); - if value == "csv" { + if value == "md" || value == "markdown" { + Ok(Self::MD) + } else if value == "csv" { Ok(Self::CSV) } else if value == "tsv" { Ok(Self::TSV) diff --git a/crates/brk_query/src/index.rs b/crates/brk_query/src/index.rs index 5179f0b70..1bde51c2b 100644 --- a/crates/brk_query/src/index.rs +++ b/crates/brk_query/src/index.rs @@ -56,7 +56,7 @@ impl Index { } } - pub fn ids() -> Vec { + pub fn possible_values() -> Vec { Self::all() .iter() .flat_map(|i| i.self_to_ids().iter().map(|s| s.to_string())) diff --git a/crates/brk_query/src/lib.rs b/crates/brk_query/src/lib.rs index eee21da87..fb39853af 100644 --- a/crates/brk_query/src/lib.rs +++ b/crates/brk_query/src/lib.rs @@ -3,19 +3,22 @@ #![doc = include_str!("main.rs")] #![doc = "```"] -mod format; -mod index; -mod params; -mod tree; - -use std::fmt; - use brk_computer::Computer; use brk_indexer::Indexer; +use tabled::settings::Style; + +mod format; +mod index; +mod output; +mod params; +mod table; +mod tree; + pub use format::Format; pub use index::Index; +pub use output::{Output, Value}; pub use params::Params; -use serde::Serialize; +pub use table::Tabled; use tree::VecIdToIndexToVec; pub struct Query<'a> { @@ -41,12 +44,12 @@ impl<'a> Query<'a> { pub fn search( &self, index: Index, - values: &[&str], + ids: &[&str], from: Option, to: Option, format: Option, - ) -> color_eyre::Result { - let ids = values + ) -> color_eyre::Result { + let tuples = ids .iter() .map(|s| { ( @@ -58,27 +61,27 @@ impl<'a> Query<'a> { .map(|(id, vec)| (id, vec.unwrap())) .collect::>(); - if ids.is_empty() { - return Ok(QueryResponse::default(format)); + if tuples.is_empty() { + return Ok(Output::default(format)); } - let mut values = ids + let mut values = tuples .iter() .flat_map(|(_, i_to_v)| i_to_v.get(&index)) .map(|vec| -> brk_vec::Result> { vec.collect_range_values(from, to) }) .collect::>>()?; if values.is_empty() { - return Ok(QueryResponse::default(format)); + return Ok(Output::default(format)); } - let ids_last_i = ids.len() - 1; + let ids_last_i = tuples.len() - 1; Ok(match format { Some(Format::CSV) | Some(Format::TSV) => { let delimiter = if format == Some(Format::CSV) { ',' } else { '\t' }; - let mut text = ids + let mut text = tuples .into_iter() .map(|(id, _)| id) .collect::>() @@ -102,59 +105,31 @@ impl<'a> Query<'a> { }); if format == Some(Format::CSV) { - QueryResponse::CSV(text) + Output::CSV(text) } else { - QueryResponse::TSV(text) + Output::TSV(text) } } + Some(Format::MD) => { + let mut table = values.to_table(ids.iter().map(|s| s.to_string()).collect::>()); + + table.with(Style::markdown()); + + Output::MD(table.to_string()) + } Some(Format::JSON) | None => { if values.len() == 1 { let mut values = values.pop().unwrap(); if values.len() == 1 { let value = values.pop().unwrap(); - QueryResponse::Json(Value::Single(value)) + Output::Json(Value::Single(value)) } else { - QueryResponse::Json(Value::List(values)) + Output::Json(Value::List(values)) } } else { - QueryResponse::Json(Value::Matrix(values)) + Output::Json(Value::Matrix(values)) } } }) } } - -#[derive(Debug)] -pub enum QueryResponse { - Json(Value), - CSV(String), - TSV(String), -} - -#[derive(Debug, Serialize)] -#[serde(untagged)] -pub enum Value { - Matrix(Vec>), - List(Vec), - Single(serde_json::Value), -} - -impl QueryResponse { - fn default(format: Option) -> Self { - match format { - Some(Format::CSV) => QueryResponse::CSV("".to_string()), - Some(Format::TSV) => QueryResponse::TSV("".to_string()), - _ => QueryResponse::Json(Value::Single(serde_json::Value::Null)), - } - } -} - -impl fmt::Display for QueryResponse { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Json(value) => write!(f, "{}", serde_json::to_string_pretty(value).unwrap()), - Self::CSV(string) => write!(f, "{}", string), - Self::TSV(string) => write!(f, "{}", string), - } - } -} diff --git a/crates/brk_query/src/output.rs b/crates/brk_query/src/output.rs new file mode 100644 index 000000000..80300ae54 --- /dev/null +++ b/crates/brk_query/src/output.rs @@ -0,0 +1,43 @@ +use std::fmt; + +use serde::Serialize; +use tabled::Tabled as TabledTabled; + +use crate::Format; + +#[derive(Debug)] +pub enum Output { + Json(Value), + CSV(String), + TSV(String), + MD(String), +} + +#[derive(Debug, Serialize, TabledTabled)] +#[serde(untagged)] +pub enum Value { + Matrix(Vec>), + List(Vec), + Single(serde_json::Value), +} + +impl Output { + pub fn default(format: Option) -> Self { + match format { + Some(Format::CSV) => Output::CSV("".to_string()), + Some(Format::TSV) => Output::TSV("".to_string()), + _ => Output::Json(Value::Single(serde_json::Value::Null)), + } + } +} + +impl fmt::Display for Output { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Json(value) => write!(f, "{}", serde_json::to_string_pretty(value).unwrap()), + Self::CSV(string) => write!(f, "{}", string), + Self::TSV(string) => write!(f, "{}", string), + Self::MD(string) => write!(f, "{}", string), + } + } +} diff --git a/crates/brk_query/src/params.rs b/crates/brk_query/src/params.rs index c904e2e98..91c77e084 100644 --- a/crates/brk_query/src/params.rs +++ b/crates/brk_query/src/params.rs @@ -5,14 +5,19 @@ use crate::{Format, Index}; #[derive(Debug, Deserialize, Parser)] pub struct Params { - #[clap(short, long, value_parser = PossibleValuesParser::new(Index::ids()))] + #[clap(short, long, value_parser = PossibleValuesParser::new(Index::possible_values()))] + /// Index of the values requested pub index: String, #[clap(short, long, value_delimiter = ' ', num_args = 1..)] + /// Names of the values requested pub values: Vec, #[clap(short, long, allow_hyphen_values = true)] + /// Inclusive starting index, if negative will be from the end pub from: Option, #[clap(short, long, allow_hyphen_values = true)] + /// Inclusive ending index, if negative will be from the end pub to: Option, #[clap(long)] + /// Format of the output pub format: Option, } diff --git a/crates/brk_query/src/table.rs b/crates/brk_query/src/table.rs new file mode 100644 index 000000000..0de9f6c6a --- /dev/null +++ b/crates/brk_query/src/table.rs @@ -0,0 +1,23 @@ +use tabled::{Table, builder::Builder}; + +pub trait Tabled { + fn to_table(&self, ids: Vec) -> Table; +} + +impl Tabled for Vec> { + fn to_table(&self, ids: Vec) -> Table { + let mut builder = Builder::default(); + + builder.push_record(ids); + + if let Some(first) = self.first() { + let len = first.len(); + + (0..len).for_each(|index| { + builder.push_record(self.iter().map(|vec| vec.get(index).unwrap().to_string())); + }); + } + + builder.build() + } +} diff --git a/crates/brk_server/src/api/vecs/mod.rs b/crates/brk_server/src/api/vecs/mod.rs index 00c674137..adafcd2c8 100644 --- a/crates/brk_server/src/api/vecs/mod.rs +++ b/crates/brk_server/src/api/vecs/mod.rs @@ -122,6 +122,7 @@ fn req_to_response_res( csv.into_response() } + Some(Format::MD) => "".into_response(), Some(Format::JSON) | None => { if values.len() == 1 { let values = values.first().unwrap(); @@ -147,6 +148,7 @@ fn req_to_response_res( headers.insert_content_disposition_attachment(); match format { Format::CSV => headers.insert_content_type_text_csv(), + Format::MD => headers.insert_content_type_text_plain(), Format::TSV => headers.insert_content_type_text_tsv(), Format::JSON => headers.insert_content_type_application_json(), } diff --git a/crates/brk_vec/src/lib.rs b/crates/brk_vec/src/lib.rs index eea78d687..a5530e0c9 100644 --- a/crates/brk_vec/src/lib.rs +++ b/crates/brk_vec/src/lib.rs @@ -325,15 +325,15 @@ where } }); - let to = to.map_or(len, |to| { + let to = to.map_or(len - 1, |to| { if to >= 0 { to as usize } else { - (len as i64 + to) as usize + ((len - 1) as i64 + to) as usize } }); - if from >= to { + if from > to { return Err(Error::RangeFromAfterTo); } @@ -341,8 +341,8 @@ where let mut buf = Self::create_buffer(); - Ok((from..to) - .map(|_| Self::read_exact(&mut file, &mut buf).map(|v| v.to_owned()).unwrap()) + Ok((from..=to) + .flat_map(|_| Self::read_exact(&mut file, &mut buf).map(|v| v.to_owned())) .collect::>()) }