check: crawl input dir for QMDL, update to new analysis format

Cargo.lock

@@ -2374,6 +2374,7 @@ dependencies = [
  "rayhunter",
  "simple_logger",
  "tokio",
+ "walkdir",
 ]
 
 [[package]]
@@ -2583,6 +2584,15 @@ version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
@@ -3233,6 +3243,16 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
 [[package]]
 name = "want"
 version = "0.3.1"

check/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "rayhunter-check"
+version = "0.4.0"
+edition = "2024"
+
+[dependencies]
+rayhunter = { path = "../lib" }
+futures = { version = "0.3.30", default-features = false }
+log = "0.4.20"
+tokio = { version = "1.44.2", default-features = false, features = ["fs", "signal", "process", "rt-multi-thread"] }
+clap = { version = "4.5.2", features = ["derive"] }
+simple_logger = "5.0.0"
+walkdir = "2.5.0"

check/src/main.rs

@@ -1,6 +1,6 @@
 use clap::Parser;
 use futures::TryStreamExt;
-use log::{info, warn};
+use log::{error, info, warn};
 use rayhunter::{
     analysis::analyzer::{AnalyzerConfig, EventType, Harness},
     diag::DataType,
@@ -9,34 +9,27 @@ use rayhunter::{
     qmdl::QmdlReader,
 };
 use std::{collections::HashMap, future, path::PathBuf, pin::pin};
-use tokio::fs::{File, metadata, read_dir};
+use tokio::fs::File;
-
+use walkdir::WalkDir;
-mod dummy_analyzer;
 
 #[derive(Parser, Debug)]
 #[command(version, about)]
 struct Args {
     #[arg(short = 'p', long)]
-    qmdl_path: PathBuf,
+    path: PathBuf,
 
-    #[arg(short = 'c', long)]
+    #[arg(long)]
     pcapify: bool,
 
     #[arg(long)]
     show_skipped: bool,
 
-    #[arg(long)]
-    enable_dummy_analyzer: bool,
-
     #[arg(short, long)]
     verbose: bool,
 }
 
-async fn analyze_file(enable_dummy_analyzer: bool, qmdl_path: &str, show_skipped: bool) {
+async fn analyze_file(qmdl_path: &str, show_skipped: bool) {
     let mut harness = Harness::new_with_config(&AnalyzerConfig::default());
-    if enable_dummy_analyzer {
-        harness.add_analyzer(Box::new(dummy_analyzer::TestAnalyzer { count: 0 }));
-    }
     let qmdl_file = &mut File::open(&qmdl_path).await.expect("failed to open file");
     let file_size = qmdl_file
         .metadata()
@@ -58,26 +51,27 @@ async fn analyze_file(enable_dummy_analyzer: bool, qmdl_path: &str, show_skipped
         .await
         .expect("failed getting QMDL container")
     {
-        let row = harness.analyze_qmdl_messages(container);
+        for row in harness.analyze_qmdl_messages(container) {
-        total_messages += 1;
+            total_messages += 1;
-        for reason in row.skipped_message_reasons {
+            if let Some(reason) = row.skipped_message_reason {
-            *skipped_reasons.entry(reason).or_insert(0) += 1;
+                *skipped_reasons.entry(reason).or_insert(0) += 1;
-            skipped += 1;
+                skipped += 1;
-        }
+                continue;
-        for analysis in row.analysis {
+            }
-            for maybe_event in analysis.events {
+            for maybe_event in row.events {
                 let Some(event) = maybe_event else { continue };
+                let Some(timestamp) = row.packet_timestamp else { continue };
                 match event.event_type {
                     EventType::Informational => {
                         info!(
                             "{}: INFO - {} {}",
-                            qmdl_path, analysis.timestamp, event.message,
+                            qmdl_path, timestamp, event.message,
                         );
                     }
                     EventType::QualitativeWarning { severity } => {
                         warn!(
                             "{}: WARNING (Severity: {:?}) - {} {}",
-                            qmdl_path, severity, analysis.timestamp, event.message,
+                            qmdl_path, severity, timestamp, event.message,
                         );
                         warnings += 1;
                     }
@@ -144,36 +138,27 @@ async fn main() {
         .unwrap();
     info!("Analyzers:");
 
-    let mut harness = Harness::new_with_config(&AnalyzerConfig::default());
+    let harness = Harness::new_with_config(&AnalyzerConfig::default());
-    if args.enable_dummy_analyzer {
-        harness.add_analyzer(Box::new(dummy_analyzer::TestAnalyzer { count: 0 }));
-    }
     for analyzer in harness.get_metadata().analyzers {
-        info!("    - {}: {}", analyzer.name, analyzer.description);
+        info!("    - {}: {} (v{})", analyzer.name, analyzer.description, analyzer.version);
     }
 
-    let metadata = metadata(&args.qmdl_path)
+    for maybe_entry in WalkDir::new(&args.path) {
-        .await
+        let Ok(entry) = maybe_entry else {
-        .expect("failed to get metadata");
+            error!("failed to open dir entry {:?}", maybe_entry);
-    if metadata.is_dir() {
+            continue;
-        let mut dir = read_dir(&args.qmdl_path).await.expect("failed to read dir");
+        };
-        while let Some(entry) = dir.next_entry().await.expect("failed to get entry") {
+        let name = entry.file_name();
-            let name = entry.file_name();
+        let name_str = name.to_str().unwrap();
-            let name_str = name.to_str().unwrap();
+        // instead of relying on the QMDL extension, can we check if a file is
-            if name_str.ends_with(".qmdl") {
+        // QMDL by inspecting the contents?
-                let path = entry.path();
+        if name_str.ends_with(".qmdl") {
-                let path_str = path.to_str().unwrap();
+            let path = entry.path();
-                analyze_file(args.enable_dummy_analyzer, path_str, args.show_skipped).await;
+            let path_str = path.to_str().unwrap();
-                if args.pcapify {
+            analyze_file(path_str, args.show_skipped).await;
-                    pcapify(&path).await;
+            if args.pcapify {
-                }
+                pcapify(&path.to_path_buf()).await;
             }
         }
-    } else {
-        let path = args.qmdl_path.to_str().unwrap();
-        analyze_file(args.enable_dummy_analyzer, path, args.show_skipped).await;
-        if args.pcapify {
-            pcapify(&args.qmdl_path).await;
-        }
     }
 }