computer: pools part 1 + fetcher: fix url + interface: more ddos protection

crates/brk_interface/src/ids.rs

@@ -0,0 +1,82 @@
+use std::fmt;
+
+use derive_deref::Deref;
+use schemars::JsonSchema;
+use serde::Deserialize;
+
+#[derive(Debug, Deref, JsonSchema)]
+pub struct MaybeIds(Vec<String>);
+
+const MAX_STRING_SIZE: usize = 10_000;
+const MAX_VECS: usize = 64;
+
+impl From<String> for MaybeIds {
+    fn from(value: String) -> Self {
+        Self(vec![value])
+    }
+}
+
+impl<'a> From<Vec<&'a str>> for MaybeIds {
+    fn from(value: Vec<&'a str>) -> Self {
+        Self(value.iter().map(|s| s.to_string()).collect::<Vec<_>>())
+    }
+}
+
+impl<'de> Deserialize<'de> for MaybeIds {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        match serde_json::Value::deserialize(deserializer)? {
+            serde_json::Value::String(str) => {
+                if str.len() > MAX_STRING_SIZE {
+                    Ok(MaybeIds(sanitize_ids(
+                        str.split(",").map(|s| s.to_string()),
+                    )))
+                } else {
+                    Err(serde::de::Error::custom("Given parameter is too long"))
+                }
+            }
+            serde_json::Value::Array(vec) => {
+                if vec.len() > MAX_VECS {
+                    Ok(MaybeIds(sanitize_ids(
+                        vec.into_iter().map(|s| s.as_str().unwrap().to_string()),
+                    )))
+                } else {
+                    Err(serde::de::Error::custom("Given parameter is too long"))
+                }
+            }
+            _ => Err(serde::de::Error::custom("Bad ids format")),
+        }
+    }
+}
+
+impl fmt::Display for MaybeIds {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let s = self.0.join(",");
+        write!(f, "{s}")
+    }
+}
+
+fn sanitize_ids(raw_ids: impl Iterator<Item = String>) -> Vec<String> {
+    let mut results = Vec::new();
+    raw_ids.for_each(|s| {
+        let mut current = String::new();
+        for c in s.to_lowercase().chars() {
+            match c {
+                ' ' | ',' | '+' => {
+                    if !current.is_empty() {
+                        results.push(std::mem::take(&mut current));
+                    }
+                }
+                '-' => current.push('_'),
+                c if c.is_alphanumeric() || c == '_' => current.push(c),
+                _ => {}
+            }
+        }
+        if !current.is_empty() {
+            results.push(current);
+        }
+    });
+    results
+}

crates/brk_interface/src/lib.rs

@@ -1,6 +1,6 @@
 #![doc = include_str!("../README.md")]
 
-use std::collections::BTreeMap;
+use std::{collections::BTreeMap, sync::OnceLock};
 
 use brk_computer::Computer;
 use brk_error::{Error, Result};
@@ -10,13 +10,14 @@ use nucleo_matcher::{
     Config, Matcher,
     pattern::{AtomKind, CaseMatching, Normalization, Pattern},
 };
+use quick_cache::sync::Cache;
 use tabled::settings::Style;
 use vecdb::{AnyCollectableVec, AnyStoredVec};
 
 mod deser;
 mod format;
+mod ids;
 mod index;
-mod maybe_ids;
 mod output;
 mod pagination;
 mod params;
@@ -33,6 +34,11 @@ use vecs::Vecs;
 
 use crate::vecs::{IdToVec, IndexToVec};
 
+pub fn cached_errors() -> &'static Cache<String, String> {
+    static CACHE: OnceLock<Cache<String, String>> = OnceLock::new();
+    CACHE.get_or_init(|| Cache::new(1000))
+}
+
 #[allow(dead_code)]
 pub struct Interface<'a> {
     vecs: Vecs<'a>,
@@ -58,34 +64,31 @@ impl<'a> Interface<'a> {
     }
 
     pub fn search(&self, params: &Params) -> Result<Vec<(String, &&dyn AnyCollectableVec)>> {
+        let ids = &params.ids;
+        let index = params.index;
+
         let ids_to_vec = self
             .vecs
             .index_to_id_to_vec
-            .get(&params.index)
+            .get(&index)
             .ok_or(Error::String(format!(
                 "Index \"{}\" isn't a valid index",
-                params.index
+                index
             )))?;
 
-        let maybe_ids = params.ids.iter().flat_map(|s| {
-            s.to_lowercase()
-                .replace("-", "_")
-                .split_whitespace()
-                .flat_map(|s| {
-                    s.split(',')
-                        .flat_map(|s| s.split('+').map(|s| s.to_string()))
-                })
-                .collect::<Vec<_>>()
-        });
-
-        maybe_ids
+        ids.iter()
             .map(|id| {
                 let vec = ids_to_vec.get(id.as_str()).ok_or_else(|| {
+                    let cached_errors = cached_errors();
+
+                    if let Some(message) = cached_errors.get(id) {
+                        return Error::String(message)
+                    }
+
                     let mut message = format!(
                         "No vec named \"{}\" indexed by \"{}\" found.\n",
-                        // tell if id found in another index
                         id,
-                        params.index
+                        index
                     );
 
                     let mut matcher = Matcher::new(Config::DEFAULT);
@@ -111,9 +114,11 @@ impl<'a> Interface<'a> {
                         message += &format!("\nBut there is a vec named {id} which supports the following indexes: {:#?}\n", index_to_vec.keys());
                     }
 
+                    cached_errors.insert(id.clone(), message.clone());
+
                     Error::String(message)
                 });
-                vec.map(|vec| (id, vec))
+                vec.map(|vec| (id.clone(), vec))
             })
             .collect::<Result<Vec<_>>>()
     }

crates/brk_interface/src/maybe_ids.rs

@@ -1,38 +0,0 @@
-use derive_deref::Deref;
-use schemars::JsonSchema;
-use serde::Deserialize;
-
-#[derive(Debug, Deref, JsonSchema)]
-pub struct MaybeIds(Vec<String>);
-
-impl From<String> for MaybeIds {
-    fn from(value: String) -> Self {
-        Self(vec![value])
-    }
-}
-
-impl<'a> From<Vec<&'a str>> for MaybeIds {
-    fn from(value: Vec<&'a str>) -> Self {
-        Self(value.iter().map(|s| s.to_string()).collect::<Vec<_>>())
-    }
-}
-
-impl<'de> Deserialize<'de> for MaybeIds {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        let maybe_ids = match serde_json::Value::deserialize(deserializer)? {
-            serde_json::Value::String(str) => {
-                str.split(",").map(|s| s.to_string()).collect::<Vec<_>>()
-            }
-            serde_json::Value::Array(vec) => vec
-                .into_iter()
-                .map(|s| s.as_str().unwrap().to_string())
-                .collect::<Vec<_>>(),
-            _ => return Err(serde::de::Error::custom("Bad ids format")),
-        };
-        // dbg!(&maybe_ids);
-        Ok(MaybeIds(maybe_ids))
-    }
-}

crates/brk_interface/src/params.rs

@@ -6,7 +6,7 @@ use serde::Deserialize;
 use crate::{
     Format, Index,
     deser::{de_unquote_i64, de_unquote_usize},
-    maybe_ids::MaybeIds,
+    ids::MaybeIds,
 };
 
 #[derive(Debug, Deserialize, JsonSchema)]