computer: pools part 1 + fetcher: fix url + interface: more ddos protection

crates/brk_computer/Cargo.toml

@@ -21,6 +21,7 @@ brk_parser = { workspace = true }
 vecdb = { workspace = true }
 derive_deref = { workspace = true }
 log = { workspace = true }
+num_enum = "0.7.4"
 pco = "0.4.6"
 rayon = { workspace = true }
 serde = { workspace = true }

crates/brk_computer/examples/pools.rs

@@ -0,0 +1,55 @@
+use std::{collections::BTreeMap, path::Path, thread};
+
+use brk_computer::{Computer, pools};
+use brk_error::Result;
+use brk_fetcher::Fetcher;
+use brk_indexer::Indexer;
+use vecdb::Exit;
+
+fn main() -> Result<()> {
+    brk_logger::init(Some(Path::new(".log")))?;
+
+    let exit = Exit::new();
+    exit.set_ctrlc_handler();
+
+    thread::Builder::new()
+        .stack_size(256 * 1024 * 1024)
+        .spawn(move || -> Result<()> {
+            let outputs_dir = Path::new(&std::env::var("HOME").unwrap()).join(".brk");
+
+            let indexer = Indexer::forced_import(&outputs_dir)?;
+
+            let fetcher = Fetcher::import(true, None)?;
+
+            let computer = Computer::forced_import(&outputs_dir, &indexer, Some(fetcher))?;
+
+            let pools = pools();
+
+            let mut res: BTreeMap<&'static str, usize> = BTreeMap::default();
+
+            let mut height_to_first_txindex_iter = indexer.vecs.height_to_first_txindex.iter();
+            // let mut i = indexer.vecs.txz
+
+            indexer
+                .stores
+                .height_to_coinbase_tag
+                .iter()
+                .for_each(|(_, coinbase_tag)| {
+                    let pool = pools.find_from_coinbase_tag(&coinbase_tag);
+                    if let Some(pool) = pool {
+                        *res.entry(pool.name).or_default() += 1;
+                    } else {
+                        *res.entry(pools.get_unknown().name).or_default() += 1;
+                    }
+                });
+
+            let mut v = res.into_iter().map(|(k, v)| (v, k)).collect::<Vec<_>>();
+            v.sort_unstable();
+            println!("{:#?}", v);
+            println!("{:#?}", v.len());
+
+            Ok(())
+        })?
+        .join()
+        .unwrap()
+}

crates/brk_computer/src/lib.rs

@@ -16,6 +16,7 @@ mod fetched;
 mod grouped;
 mod indexes;
 mod market;
+mod pools;
 mod price;
 mod stateful;
 mod states;
@@ -24,6 +25,7 @@ mod utils;
 
 use indexes::Indexes;
 
+pub use pools::*;
 pub use states::PriceToAmount;
 use states::*;
 

crates/brk_computer/src/pools/id.rs

@@ -0,0 +1,175 @@
+use num_enum::{FromPrimitive, IntoPrimitive};
+use serde::{Deserialize, Serialize};
+
+#[allow(clippy::upper_case_acronyms)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, FromPrimitive, IntoPrimitive)]
+#[repr(u16)]
+pub enum PoolId {
+    #[default]
+    Unknown,
+    BlockFills,
+    Ultimuspool,
+    TerraPool,
+    Luxor,
+    OneTHash,
+    BTCCom,
+    Bitfarms,
+    HuobiPool,
+    WayiCn,
+    CanoePool,
+    BTCTop,
+    BitcoinCom,
+    OneSevenFiveBtc,
+    GBMiners,
+    AXbt,
+    ASICMiner,
+    BitMinter,
+    BitcoinRussia,
+    BTCServ,
+    SimplecoinUs,
+    BTCGuild,
+    Eligius,
+    OzCoin,
+    EclipseMC,
+    MaxBTC,
+    TripleMining,
+    CoinLab,
+    FiftyBTC,
+    GHashIO,
+    STMiningCorp,
+    Bitparking,
+    MMPool,
+    Polmine,
+    KnCMiner,
+    Bitalo,
+    F2Pool,
+    HHTT,
+    MegaBigPower,
+    MtRed,
+    NMCbit,
+    YourbtcNet,
+    GiveMeCoins,
+    BraiinsPool,
+    AntPool,
+    MultiCoinCo,
+    BCPoolIo,
+    Cointerra,
+    KanoPool,
+    SoloCK,
+    CKPool,
+    NiceHash,
+    BitClub,
+    BitcoinAffiliateNetwork,
+    BTCC,
+    BWPool,
+    EXXAndBW,
+    Bitsolo,
+    BitFury,
+    TwentyOneInc,
+    DigitalBTC,
+    EightBaochi,
+    MyBTCcoinPool,
+    TBDice,
+    HASHPOOL,
+    Nexious,
+    BravoMining,
+    HotPool,
+    OKExPool,
+    BCMonster,
+    OneHash,
+    Bixin,
+    TATMASPool,
+    ViaBTC,
+    ConnectBTC,
+    BATPOOL,
+    Waterhole,
+    DCExploration,
+    DCEX,
+    BTPOOL,
+    FiftyEightCoin,
+    BitcoinIndiaLowercase,
+    ShawnP0wers,
+    PHashIO,
+    RigPool,
+    HAOZHUZHU,
+    SevenPool,
+    MiningKings,
+    HashBX,
+    DPOOL,
+    Rawpool,
+    Haominer,
+    Helix,
+    BitcoinUkraine,
+    Poolin,
+    SecretSuperstar,
+    TigerpoolNet,
+    SigmapoolCom,
+    OkpoolTop,
+    Hummerpool,
+    Tangpool,
+    BytePool,
+    SpiderPool,
+    NovaBlock,
+    MiningCity,
+    BinancePool,
+    Minerium,
+    LubianCom,
+    OKKONG,
+    AAOPool,
+    EMCDPool,
+    FoundryUSA,
+    SBICrypto,
+    ArkPool,
+    PureBTCCom,
+    MARAPool,
+    KuCoinPool,
+    EntrustCharityPool,
+    OKMINER,
+    Titan,
+    PEGAPool,
+    BTCNuggets,
+    CloudHashing,
+    DigitalXMintsy,
+    Telco214,
+    BTCPoolParty,
+    Multipool,
+    TransactionCoinMining,
+    BTCDig,
+    TrickysBTCPool,
+    BTCMP,
+    Eobot,
+    UNOMP,
+    Patels,
+    GoGreenLight,
+    BitcoinIndiaCamel, // duplicate-ish entry preserved with slight name change
+    EkanemBTC,
+    CanoeUppercase,
+    TigerLowercase,
+    OneM1X,
+    Zulupool,
+    SECPOOL,
+    OCEAN,
+    WhitePool,
+    Wiz,
+    Mononaut,
+    Rijndael,
+    Wk057,
+    FutureBitApolloSolo,
+    Emzy,
+    Knorrium,
+    CarbonNegative,
+    PortlandHODL,
+    Phoenix,
+    Neopool,
+    MaxiPool,
+    DrDetroit,
+    BitFuFuPool,
+    LuckyPool,
+    MiningDutch,
+    PublicPool,
+    MiningSquared,
+    InnopolisTech,
+    Nymkappa,
+    BTCLab,
+    Parasite,
+}

crates/brk_computer/src/pools/mod.rs

@@ -0,0 +1,7 @@
+mod id;
+mod pool;
+mod pools;
+
+pub use id::*;
+pub use pool::*;
+pub use pools::*;

crates/brk_computer/src/pools/pool.rs

@@ -0,0 +1,12 @@
+use serde::{Deserialize, Serialize};
+
+use crate::pools::PoolId;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Pool {
+    pub id: PoolId,
+    pub name: &'static str,
+    pub addresses: Box<[&'static str]>,
+    pub tags: Box<[&'static str]>,
+    pub link: &'static str,
+}

crates/brk_fetcher/src/brk.rs

@@ -14,7 +14,7 @@ pub struct BRK {
     dateindex_to_ohlc: BTreeMap<DateIndex, Vec<OHLCCents>>,
 }
 
-const API_URL: &str = "https://bitcoinresearchkit.org/api/vecs";
+const API_URL: &str = "https://bitview.space/api/vecs";
 const CHUNK_SIZE: usize = 10_000;
 
 impl BRK {
@@ -46,7 +46,7 @@ impl BRK {
 
         default_retry(|_| {
             let url = format!(
-                "{API_URL}/height-to-ohlc?from={}&to={}",
+                "{API_URL}/height-to-price-ohlc?from={}&to={}",
                 height,
                 height + CHUNK_SIZE
             );
@@ -91,7 +91,7 @@ impl BRK {
 
         default_retry(|_| {
             let url = format!(
-                "{API_URL}/dateindex-to-ohlc?from={}&to={}",
+                "{API_URL}/dateindex-to-price-ohlc?from={}&to={}",
                 dateindex,
                 dateindex + CHUNK_SIZE
             );

crates/brk_interface/Cargo.toml

@@ -16,6 +16,7 @@ brk_indexer = { workspace = true }
 brk_structs = { workspace = true }
 vecdb = { workspace = true }
 derive_deref = { workspace = true }
+quick_cache = { workspace = true }
 schemars = "1.0.4"
 serde = { workspace = true }
 serde_json = { workspace = true }

crates/brk_interface/src/ids.rs

@@ -0,0 +1,82 @@
+use std::fmt;
+
+use derive_deref::Deref;
+use schemars::JsonSchema;
+use serde::Deserialize;
+
+#[derive(Debug, Deref, JsonSchema)]
+pub struct MaybeIds(Vec<String>);
+
+const MAX_STRING_SIZE: usize = 10_000;
+const MAX_VECS: usize = 64;
+
+impl From<String> for MaybeIds {
+    fn from(value: String) -> Self {
+        Self(vec![value])
+    }
+}
+
+impl<'a> From<Vec<&'a str>> for MaybeIds {
+    fn from(value: Vec<&'a str>) -> Self {
+        Self(value.iter().map(|s| s.to_string()).collect::<Vec<_>>())
+    }
+}
+
+impl<'de> Deserialize<'de> for MaybeIds {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        match serde_json::Value::deserialize(deserializer)? {
+            serde_json::Value::String(str) => {
+                if str.len() > MAX_STRING_SIZE {
+                    Ok(MaybeIds(sanitize_ids(
+                        str.split(",").map(|s| s.to_string()),
+                    )))
+                } else {
+                    Err(serde::de::Error::custom("Given parameter is too long"))
+                }
+            }
+            serde_json::Value::Array(vec) => {
+                if vec.len() > MAX_VECS {
+                    Ok(MaybeIds(sanitize_ids(
+                        vec.into_iter().map(|s| s.as_str().unwrap().to_string()),
+                    )))
+                } else {
+                    Err(serde::de::Error::custom("Given parameter is too long"))
+                }
+            }
+            _ => Err(serde::de::Error::custom("Bad ids format")),
+        }
+    }
+}
+
+impl fmt::Display for MaybeIds {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let s = self.0.join(",");
+        write!(f, "{s}")
+    }
+}
+
+fn sanitize_ids(raw_ids: impl Iterator<Item = String>) -> Vec<String> {
+    let mut results = Vec::new();
+    raw_ids.for_each(|s| {
+        let mut current = String::new();
+        for c in s.to_lowercase().chars() {
+            match c {
+                ' ' | ',' | '+' => {
+                    if !current.is_empty() {
+                        results.push(std::mem::take(&mut current));
+                    }
+                }
+                '-' => current.push('_'),
+                c if c.is_alphanumeric() || c == '_' => current.push(c),
+                _ => {}
+            }
+        }
+        if !current.is_empty() {
+            results.push(current);
+        }
+    });
+    results
+}

crates/brk_interface/src/lib.rs

@@ -1,6 +1,6 @@
 #![doc = include_str!("../README.md")]
 
-use std::collections::BTreeMap;
+use std::{collections::BTreeMap, sync::OnceLock};
 
 use brk_computer::Computer;
 use brk_error::{Error, Result};
@@ -10,13 +10,14 @@ use nucleo_matcher::{
     Config, Matcher,
     pattern::{AtomKind, CaseMatching, Normalization, Pattern},
 };
+use quick_cache::sync::Cache;
 use tabled::settings::Style;
 use vecdb::{AnyCollectableVec, AnyStoredVec};
 
 mod deser;
 mod format;
+mod ids;
 mod index;
-mod maybe_ids;
 mod output;
 mod pagination;
 mod params;
@@ -33,6 +34,11 @@ use vecs::Vecs;
 
 use crate::vecs::{IdToVec, IndexToVec};
 
+pub fn cached_errors() -> &'static Cache<String, String> {
+    static CACHE: OnceLock<Cache<String, String>> = OnceLock::new();
+    CACHE.get_or_init(|| Cache::new(1000))
+}
+
 #[allow(dead_code)]
 pub struct Interface<'a> {
     vecs: Vecs<'a>,
@@ -58,34 +64,31 @@ impl<'a> Interface<'a> {
     }
 
     pub fn search(&self, params: &Params) -> Result<Vec<(String, &&dyn AnyCollectableVec)>> {
+        let ids = &params.ids;
+        let index = params.index;
+
         let ids_to_vec = self
             .vecs
             .index_to_id_to_vec
-            .get(&params.index)
+            .get(&index)
             .ok_or(Error::String(format!(
                 "Index \"{}\" isn't a valid index",
-                params.index
+                index
             )))?;
 
-        let maybe_ids = params.ids.iter().flat_map(|s| {
-            s.to_lowercase()
-                .replace("-", "_")
-                .split_whitespace()
-                .flat_map(|s| {
-                    s.split(',')
-                        .flat_map(|s| s.split('+').map(|s| s.to_string()))
-                })
-                .collect::<Vec<_>>()
-        });
-
-        maybe_ids
+        ids.iter()
             .map(|id| {
                 let vec = ids_to_vec.get(id.as_str()).ok_or_else(|| {
+                    let cached_errors = cached_errors();
+
+                    if let Some(message) = cached_errors.get(id) {
+                        return Error::String(message)
+                    }
+
                     let mut message = format!(
                         "No vec named \"{}\" indexed by \"{}\" found.\n",
-                        // tell if id found in another index
                         id,
-                        params.index
+                        index
                     );
 
                     let mut matcher = Matcher::new(Config::DEFAULT);
@@ -111,9 +114,11 @@ impl<'a> Interface<'a> {
                         message += &format!("\nBut there is a vec named {id} which supports the following indexes: {:#?}\n", index_to_vec.keys());
                     }
 
+                    cached_errors.insert(id.clone(), message.clone());
+
                     Error::String(message)
                 });
-                vec.map(|vec| (id, vec))
+                vec.map(|vec| (id.clone(), vec))
             })
             .collect::<Result<Vec<_>>>()
     }

crates/brk_interface/src/maybe_ids.rs

@@ -1,38 +0,0 @@
-use derive_deref::Deref;
-use schemars::JsonSchema;
-use serde::Deserialize;
-
-#[derive(Debug, Deref, JsonSchema)]
-pub struct MaybeIds(Vec<String>);
-
-impl From<String> for MaybeIds {
-    fn from(value: String) -> Self {
-        Self(vec![value])
-    }
-}
-
-impl<'a> From<Vec<&'a str>> for MaybeIds {
-    fn from(value: Vec<&'a str>) -> Self {
-        Self(value.iter().map(|s| s.to_string()).collect::<Vec<_>>())
-    }
-}
-
-impl<'de> Deserialize<'de> for MaybeIds {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        let maybe_ids = match serde_json::Value::deserialize(deserializer)? {
-            serde_json::Value::String(str) => {
-                str.split(",").map(|s| s.to_string()).collect::<Vec<_>>()
-            }
-            serde_json::Value::Array(vec) => vec
-                .into_iter()
-                .map(|s| s.as_str().unwrap().to_string())
-                .collect::<Vec<_>>(),
-            _ => return Err(serde::de::Error::custom("Bad ids format")),
-        };
-        // dbg!(&maybe_ids);
-        Ok(MaybeIds(maybe_ids))
-    }
-}

crates/brk_interface/src/params.rs

@@ -6,7 +6,7 @@ use serde::Deserialize;
 use crate::{
     Format, Index,
     deser::{de_unquote_i64, de_unquote_usize},
-    maybe_ids::MaybeIds,
+    ids::MaybeIds,
 };
 
 #[derive(Debug, Deserialize, JsonSchema)]

crates/brk_server/Cargo.toml

@@ -23,7 +23,7 @@ brk_parser = { workspace = true }
 vecdb = { workspace = true }
 jiff = { workspace = true }
 log = { workspace = true }
-quick_cache = "0.6.16"
+quick_cache = { workspace = true }
 serde_json = { workspace = true }
 tokio = { workspace = true }
 tower-http = { version = "0.6.6", features = ["compression-full", "trace"] }