global: serialization optimizations for faster responses

crates/brk_interface/src/format.rs

@@ -1,40 +1,26 @@
-use brk_error::Error;
 use schemars::JsonSchema;
 use serde::Deserialize;
 
 #[allow(clippy::upper_case_acronyms)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, JsonSchema)]
+#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Deserialize, JsonSchema)]
 #[serde(rename_all = "lowercase")]
 pub enum Format {
+    #[default]
     #[serde(alias = "json")]
     JSON,
     #[serde(alias = "csv")]
     CSV,
-    #[serde(alias = "tsv")]
-    TSV,
-    #[serde(alias = "md", alias = "markdown")]
-    MD,
 }
 
-impl TryFrom<Option<String>> for Format {
-    type Error = Error;
-    fn try_from(value: Option<String>) -> Result<Self, Self::Error> {
+impl From<Option<String>> for Format {
+    fn from(value: Option<String>) -> Self {
         if let Some(value) = value {
             let value = value.to_lowercase();
             let value = value.as_str();
-            if value == "md" || value == "markdown" {
-                Ok(Self::MD)
-            } else if value == "csv" {
-                Ok(Self::CSV)
-            } else if value == "tsv" {
-                Ok(Self::TSV)
-            } else if value == "json" {
-                Ok(Self::JSON)
-            } else {
-                Err(Error::Str("Fail"))
+            if value == "csv" {
+                return Self::CSV;
             }
-        } else {
-            Err(Error::Str("Fail"))
         }
+        Self::JSON
     }
 }

crates/brk_interface/src/index.rs

@@ -5,7 +5,7 @@ use brk_structs::{
     DateIndex, DecadeIndex, DifficultyEpoch, EmptyAddressIndex, EmptyOutputIndex, HalvingEpoch,
     Height, InputIndex, LoadedAddressIndex, MonthIndex, OpReturnIndex, OutputIndex,
     P2AAddressIndex, P2MSOutputIndex, P2PK33AddressIndex, P2PK65AddressIndex, P2PKHAddressIndex,
-    P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex, Printable,
+    P2SHAddressIndex, P2TRAddressIndex, P2WPKHAddressIndex, P2WSHAddressIndex, PrintableIndex,
     QuarterIndex, SemesterIndex, TxIndex, UnknownOutputIndex, WeekIndex, YearIndex,
 };
 use schemars::JsonSchema;

crates/brk_interface/src/lib.rs

@@ -12,7 +12,6 @@ use nucleo_matcher::{
     pattern::{AtomKind, CaseMatching, Normalization, Pattern},
 };
 use quick_cache::sync::Cache;
-use tabled::settings::Style;
 use vecdb::{AnyCollectableVec, AnyStoredVec};
 
 mod deser;
@@ -22,7 +21,6 @@ mod index;
 mod output;
 mod pagination;
 mod params;
-mod table;
 mod vecs;
 
 pub use format::Format;
@@ -30,7 +28,6 @@ pub use index::Index;
 pub use output::{Output, Value};
 pub use pagination::{PaginatedIndexParam, PaginationParam};
 pub use params::{IdParam, Params, ParamsOpt};
-pub use table::Tabled;
 use vecs::Vecs;
 
 use crate::vecs::{IdToVec, IndexToVec};
@@ -146,75 +143,66 @@ impl<'a> Interface<'a> {
                 .unwrap_or_default()
         });
 
-        let mut values = vecs
-            .iter()
-            .map(|(_, vec)| -> Result<Vec<serde_json::Value>> {
-                Ok(vec.collect_range_serde_json(from, to)?)
-            })
-            .collect::<Result<Vec<_>>>()?;
-
         let format = params.format();
 
-        if values.is_empty() {
-            return Ok(Output::default(format));
-        }
-
-        let ids_last_i = vecs.len() - 1;
-
         Ok(match format {
-            Some(Format::CSV) | Some(Format::TSV) => {
-                let delimiter = if format == Some(Format::CSV) {
-                    ','
-                } else {
-                    '\t'
-                };
-
-                let mut text = vecs
+            Format::CSV => {
+                let headers = vecs.iter().map(|(id, _)| id.as_str()).collect::<Vec<_>>();
+                let mut values = vecs
                     .iter()
-                    .map(|(id, _)| id.to_owned())
-                    .collect::<Vec<_>>()
-                    .join(&delimiter.to_string());
+                    .map(|(_, vec)| Ok(vec.collect_range_string(from, to)?))
+                    .collect::<Result<Vec<_>>>()?;
 
-                text.push('\n');
-
-                let values_len = values.first().unwrap().len();
-
-                (0..values_len).for_each(|i| {
-                    let mut line = "".to_string();
-                    values.iter().enumerate().for_each(|(id_i, v)| {
-                        line += &v.get(i).unwrap().to_string();
-                        if id_i == ids_last_i {
-                            line.push('\n');
-                        } else {
-                            line.push(delimiter);
-                        }
-                    });
-                    text += &line;
-                });
-
-                if format == Some(Format::CSV) {
-                    Output::CSV(text)
-                } else {
-                    Output::TSV(text)
+                if values.is_empty() {
+                    return Ok(Output::CSV(headers.join(",")));
                 }
-            }
-            Some(Format::MD) => {
-                let mut table =
-                    values.to_table(vecs.iter().map(|(s, _)| s.to_owned()).collect::<Vec<_>>());
 
-                table.with(Style::markdown());
+                let first_len = values[0].len();
+                let estimated_size = (headers.len() + values.len() * first_len) * 15;
+                let mut csv = String::with_capacity(estimated_size);
 
-                Output::MD(table.to_string())
-            }
-            Some(Format::JSON) | None => {
-                if values.len() == 1 {
-                    let mut values = values.pop().unwrap();
-                    if values.len() == 1 {
-                        let value = values.pop().unwrap();
-                        Output::Json(Value::Single(value))
-                    } else {
-                        Output::Json(Value::List(values))
+                csv.push_str(&headers.join(","));
+                csv.push('\n');
+
+                for col_index in 0..first_len {
+                    let mut first = true;
+                    for vec in &mut values {
+                        if col_index < vec.len() {
+                            if !first {
+                                csv.push(',');
+                            }
+                            first = false;
+
+                            let field = std::mem::take(&mut vec[col_index]);
+
+                            if field.contains(',') {
+                                csv.push('"');
+                                csv.push_str(&field);
+                                csv.push('"');
+                            } else {
+                                csv.push_str(&field);
+                            }
+                        }
                     }
+                    csv.push('\n');
+                }
+
+                Output::CSV(csv)
+            }
+            Format::JSON => {
+                let mut values = vecs
+                    .iter()
+                    .map(|(_, vec)| -> Result<Vec<u8>> {
+                        Ok(vec.collect_range_json_bytes(from, to)?)
+                    })
+                    .collect::<Result<Vec<_>>>()?;
+
+                if values.is_empty() {
+                    return Ok(Output::default(format));
+                }
+
+                if values.len() == 1 {
+                    Output::Json(Value::List(values.pop().unwrap()))
                 } else {
                     Output::Json(Value::Matrix(values))
                 }

crates/brk_interface/src/output.rs

@@ -1,44 +1,54 @@
-use std::fmt;
-
-use serde::Serialize;
-use tabled::Tabled as TabledTabled;
-
 use crate::Format;
 
-#[derive(Debug, Serialize)]
-#[serde(untagged, rename_all = "lowercase")]
+#[derive(Debug)]
 pub enum Output {
     Json(Value),
     CSV(String),
-    TSV(String),
-    MD(String),
-}
-
-#[derive(Debug, Serialize, TabledTabled)]
-#[serde(untagged)]
-pub enum Value {
-    Matrix(Vec<Vec<serde_json::Value>>),
-    List(Vec<serde_json::Value>),
-    Single(serde_json::Value),
 }
 
 impl Output {
-    pub fn default(format: Option<Format>) -> Self {
-        match format {
-            Some(Format::CSV) => Output::CSV("".to_string()),
-            Some(Format::TSV) => Output::TSV("".to_string()),
-            _ => Output::Json(Value::Single(serde_json::Value::Null)),
+    #[allow(clippy::inherent_to_string)]
+    pub fn to_string(self) -> String {
+        match self {
+            Output::CSV(s) => s,
+            Output::Json(v) => unsafe { String::from_utf8_unchecked(v.to_vec()) },
         }
     }
 }
 
-impl fmt::Display for Output {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+#[derive(Debug)]
+pub enum Value {
+    Matrix(Vec<Vec<u8>>),
+    List(Vec<u8>),
+}
+
+impl Value {
+    pub fn to_vec(self) -> Vec<u8> {
         match self {
-            Self::Json(value) => write!(f, "{}", serde_json::to_string_pretty(value).unwrap()),
-            Self::CSV(string) => write!(f, "{string}"),
-            Self::TSV(string) => write!(f, "{string}"),
-            Self::MD(string) => write!(f, "{string}"),
+            Value::List(v) => v,
+            Self::Matrix(m) => {
+                let total_size = m.iter().map(|v| v.len()).sum::<usize>() + m.len() - 1 + 2;
+                let mut matrix = Vec::with_capacity(total_size);
+                matrix.push(b'[');
+
+                for (i, vec) in m.into_iter().enumerate() {
+                    if i > 0 {
+                        matrix.push(b',');
+                    }
+                    matrix.extend(vec);
+                }
+                matrix.push(b']');
+                matrix
+            }
+        }
+    }
+}
+
+impl Output {
+    pub fn default(format: Format) -> Self {
+        match format {
+            Format::CSV => Output::CSV("".to_string()),
+            Format::JSON => Output::Json(Value::List(b"[]".to_vec())),
         }
     }
 }

crates/brk_interface/src/params.rs

@@ -63,7 +63,7 @@ pub struct ParamsOpt {
     #[serde(default)]
     /// Format of the output
     #[schemars(description = "Format of the output")]
-    format: Option<Format>,
+    format: Format,
 }
 
 impl ParamsOpt {
@@ -82,11 +82,6 @@ impl ParamsOpt {
         self
     }
 
-    pub fn set_format(mut self, format: Format) -> Self {
-        self.format.replace(format);
-        self
-    }
-
     pub fn from(&self) -> Option<i64> {
         self.from
     }
@@ -107,7 +102,7 @@ impl ParamsOpt {
         self.to
     }
 
-    pub fn format(&self) -> Option<Format> {
+    pub fn format(&self) -> Format {
         self.format
     }
 }

crates/brk_interface/src/table.rs

@@ -1,26 +0,0 @@
-use tabled::{Table, builder::Builder};
-
-pub trait Tabled {
-    fn to_table(&self, ids: Vec<String>) -> Table;
-}
-
-impl Tabled for Vec<Vec<serde_json::Value>> {
-    fn to_table(&self, ids: Vec<String>) -> Table {
-        let mut builder = Builder::default();
-
-        builder.push_record(ids);
-
-        if let Some(first) = self.first() {
-            let len = first.len();
-
-            (0..len).for_each(|index| {
-                builder.push_record(
-                    self.iter()
-                        .map(|vec| vec.get(index).unwrap().to_string().replace("\"", "")),
-                );
-            });
-        }
-
-        builder.build()
-    }
-}