diff --git a/Cargo.lock b/Cargo.lock index b43f6ff..c4907f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2711,7 +2711,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.57.0", + "windows-core 0.61.2", ] [[package]] @@ -3184,6 +3184,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" + [[package]] name = "json-patch" version = "3.0.1" @@ -4919,11 +4925,15 @@ dependencies = [ name = "rayhunter-check" version = "0.11.2" dependencies = [ + "chrono", "clap", "futures", + "json", "log", "pcap-file-tokio", "rayhunter", + "serde", + "serde_json", "tokio", "walkdir", ] @@ -5468,9 +5478,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -5489,22 +5499,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 3.0.3", ] [[package]] @@ -5927,6 +5937,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" diff --git a/check/Cargo.toml b/check/Cargo.toml index ec7f586..011071f 100644 --- a/check/Cargo.toml +++ b/check/Cargo.toml @@ -6,8 +6,12 @@ edition = "2024" [dependencies] rayhunter = { path = "../lib" } futures = { version = "0.3.30", default-features = false } +chrono = { version = "0.4.31", features = ["serde"] } log = "0.4.20" tokio = { version = "1.44.2", default-features = false, features = ["fs", "signal", "process", "rt-multi-thread"] } pcap-file-tokio = "0.1.0" clap = { version = "4.5.2", features = ["derive"] } walkdir = "2.5.0" +serde = "1.0.229" +serde_json = "1.0.151" +json = "0.12.4" diff --git a/check/src/json.rs b/check/src/json.rs new file mode 100644 index 0000000..d0968ad --- /dev/null +++ b/check/src/json.rs @@ -0,0 +1,113 @@ +use rayhunter::analysis::analyzer::ReportMetadata; +use tokio::{io::{AsyncWrite, AsyncWriteExt, BufWriter}}; + +use crate::Report; + +pub struct IncrementalJsonWriter { + writer: BufWriter, + reports_written: usize, +} + +impl IncrementalJsonWriter { + pub async fn new(file: T, check_path: &str, metadata: &ReportMetadata) -> std::io::Result { + let mut writer = BufWriter::new(file); + let path_str = serde_json::to_string(check_path)?; + let metadata_str = serde_json::to_string(metadata)?; + let s = format!(r#"{{ "path": {}, "metadata": {}, "reports": ["#, path_str, metadata_str); + writer.write_all(s.as_bytes()).await?; + Ok(Self { + writer, + reports_written: 0, + }) + } + + pub async fn write_report(&mut self, report: &Report) -> std::io::Result<()> { + if self.reports_written > 0 { + self.writer.write_u8(',' as u8).await?; + } + let report_str = serde_json::to_string(report)?; + self.writer.write_all(report_str.as_bytes()).await?; + self.reports_written += 1; + Ok(()) + } + + pub async fn finish(mut self) -> std::io::Result<()> { + self.writer.write_all("]}".as_bytes()).await?; + self.writer.flush().await?; + Ok(()) + } +} + +#[cfg(test)] +mod test { + use rayhunter::analysis::analyzer::{AnalyzerConfig, Event, EventType, Harness, ReportMetadata}; + use serde::Deserialize; + use tokio::io::{AsyncReadExt, DuplexStream, duplex}; + + use crate::{EventWithTimestamp, Report, json::IncrementalJsonWriter}; + + #[derive(Debug, Deserialize, PartialEq)] + struct ExpectedJsonLayout { + path: String, + metadata: ReportMetadata, + reports: Vec, + } + + fn create_report(path: &str) -> Report { + let mut report = Report::new(path); + report.skipped = 13; + report.warnings = 12; + report.events.push(EventWithTimestamp { + event: Event { event_type: EventType::High, message: "hi".into() }, + timestamp: chrono::DateTime::default(), + }); + report + } + + async fn create_test_writer(path: &str) -> (IncrementalJsonWriter, DuplexStream) { + let (writer, reader) = duplex(1_000_000); + let harness = Harness::new_with_config(&AnalyzerConfig::default()); + (IncrementalJsonWriter::new(writer, path, &harness.get_metadata()).await.unwrap(), reader) + } + + async fn parse_json(mut reader: DuplexStream) -> ExpectedJsonLayout { + let mut received = String::new(); + reader.read_to_string(&mut received).await.unwrap(); + serde_json::from_str(&received).unwrap() + } + + #[tokio::test] + async fn test_empty() { + let (writer, reader) = create_test_writer("test_path").await; + writer.finish().await.unwrap(); + let value = parse_json(reader).await; + let expected = ExpectedJsonLayout { + path: "test_path".to_string(), + metadata: Harness::new_with_config(&AnalyzerConfig::default()).get_metadata(), + reports: vec![], + }; + assert_eq!(value, expected); + } + + #[tokio::test] + async fn test_with_one_report() { + let (mut writer, reader) = create_test_writer("test_path").await; + let report = create_report("foo.qmdl"); + writer.write_report(&report).await.unwrap(); + writer.finish().await.unwrap(); + let value = parse_json(reader).await; + assert_eq!(value.reports, vec![report]); + } + + #[tokio::test] + async fn test_with_multiple_reports() { + let (mut writer, reader) = create_test_writer("test_path").await; + let report1 = create_report("foo.qmdl"); + writer.write_report(&report1).await.unwrap(); + let report2 = create_report("bar.qmdl"); + writer.write_report(&report2).await.unwrap(); + writer.finish().await.unwrap(); + let value = parse_json(reader).await; + assert_eq!(value.reports, vec![report1, report2]); + } +} diff --git a/check/src/main.rs b/check/src/main.rs index 869ecdd..09277d5 100644 --- a/check/src/main.rs +++ b/check/src/main.rs @@ -1,16 +1,19 @@ +use chrono::{DateTime, FixedOffset}; use clap::Parser; use log::{debug, error, info, warn}; use pcap_file_tokio::pcapng::{Block, PcapNgReader}; use rayhunter::{ - analysis::analyzer::{AnalysisRow, AnalyzerConfig, EventType, Harness}, - gsmtap::parser as gsmtap_parser, - pcap::GsmtapPcapWriter, - qmdl::QmdlMessageReader, + analysis::analyzer::{AnalysisRow, AnalyzerConfig, Event, EventType, Harness}, gsmtap::parser as gsmtap_parser, pcap::GsmtapPcapWriter, qmdl::QmdlMessageReader, }; +use serde::{Deserialize, Serialize}; use std::{collections::HashMap, path::PathBuf}; use tokio::fs::File; use walkdir::WalkDir; +use crate::json::IncrementalJsonWriter; + +mod json; + #[derive(Parser, Debug)] #[command(version, about)] struct Args { @@ -23,6 +26,9 @@ struct Args { #[arg(long, help = "Show why some packets were skipped during analysis")] show_skipped: bool, + #[arg(short = 'j', long, help = "Output report to a JSON file")] + json: Option, + #[arg(short, long, help = "Only print warnings/errors to stdout")] quiet: bool, @@ -30,9 +36,16 @@ struct Args { debug: bool, } -#[derive(Default)] +#[derive(Debug, Serialize, Deserialize, PartialEq)] +struct EventWithTimestamp { + timestamp: DateTime, + event: Event, +} + +#[derive(Debug, Default, Serialize, Deserialize, PartialEq)] struct Report { skipped_reasons: HashMap, + events: Vec, total_messages: u32, warnings: u32, skipped: u32, @@ -71,6 +84,7 @@ impl Report { self.warnings += 1; } } + self.events.push(EventWithTimestamp { timestamp, event }); } } @@ -88,7 +102,7 @@ impl Report { } } -async fn analyze_pcap(pcap_path: &str, show_skipped: bool) { +async fn analyze_pcap(pcap_path: &str, show_skipped: bool, json_writer: Option<&mut IncrementalJsonWriter>) { let mut harness = Harness::new_with_config(&AnalyzerConfig::default()); let pcap_file = &mut File::open(&pcap_path).await.expect("failed to open file"); let mut pcap_reader = PcapNgReader::new(pcap_file) @@ -106,9 +120,12 @@ async fn analyze_pcap(pcap_path: &str, show_skipped: bool) { report.process_row(row); } report.print_summary(show_skipped); + if let Some(writer) = json_writer { + writer.write_report(&report).await.expect("failed to write report to JSON"); + } } -async fn analyze_qmdl(qmdl_path: &str, show_skipped: bool) { +async fn analyze_qmdl(qmdl_path: &str, show_skipped: bool, json_writer: Option<&mut IncrementalJsonWriter>) { let mut harness = Harness::new_with_config(&AnalyzerConfig::default()); let qmdl_file = &mut File::open(&qmdl_path).await.expect("failed to open file"); let mut qmdl_reader = QmdlMessageReader::new(qmdl_file) @@ -123,6 +140,9 @@ async fn analyze_qmdl(qmdl_path: &str, show_skipped: bool) { report.process_row(harness.analyze_qmdl_message(maybe_message)); } report.print_summary(show_skipped); + if let Some(writer) = json_writer { + writer.write_report(&report).await.expect("failed to write report to JSON"); + } } async fn pcapify(qmdl_path: &PathBuf) { @@ -169,14 +189,30 @@ async fn main() { rayhunter::init_logging(level); let harness = Harness::new_with_config(&AnalyzerConfig::default()); + let metadata = harness.get_metadata(); info!("Analyzers:"); - for analyzer in harness.get_metadata().analyzers { + for analyzer in &metadata.analyzers { info!( " - {} (v{}): {}", analyzer.name, analyzer.version, analyzer.description ); } + let mut json_writer = None; + if let Some(json_path) = &args.json { + let json_file = File::options() + .write(true) + .create(true) + .truncate(true) + .open(json_path) + .await + .expect("failed to create JSON output file"); + let check_path_str = args.path.to_string_lossy(); + json_writer = Some(IncrementalJsonWriter::new(json_file, &check_path_str, &metadata) + .await + .expect("failed to create JSON writer")); + } + for maybe_entry in WalkDir::new(&args.path) { let Ok(entry) = maybe_entry else { error!("failed to open dir entry {maybe_entry:?}"); @@ -188,14 +224,20 @@ async fn main() { let path_str = path.to_str().unwrap(); if name_str.ends_with(".qmdl") || name_str.ends_with(".qmdl.gz") { info!("**** Beginning analysis of {name_str}"); - analyze_qmdl(path_str, args.show_skipped).await; + analyze_qmdl(path_str, args.show_skipped, json_writer.as_mut()).await; if args.pcapify { pcapify(&path.to_path_buf()).await; } } else if name_str.ends_with(".pcap") || name_str.ends_with(".pcapng") { // TODO: if we've already analyzed a QMDL, skip its corresponding pcap info!("**** Beginning analysis of {name_str}"); - analyze_pcap(path_str, args.show_skipped).await; + analyze_pcap(path_str, args.show_skipped, json_writer.as_mut()).await; } } + + if let Some(writer) = json_writer { + // if we have a json_writer, we also have args.json + info!("Writing report to {:?}", args.json.unwrap()); + writer.finish().await.expect("failed to finish writing to JSON file"); + } } diff --git a/lib/src/analysis/analyzer.rs b/lib/src/analysis/analyzer.rs index ebb6823..82315e9 100644 --- a/lib/src/analysis/analyzer.rs +++ b/lib/src/analysis/analyzer.rs @@ -106,7 +106,7 @@ impl<'de> Deserialize<'de> for EventType { /// Events are user-facing signals that can be emitted by an [Analyzer] upon a /// message being received. They can be used to signifiy an IC detection /// warning, or just to display some relevant information to the user. -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Event { pub event_type: EventType, pub message: String, @@ -144,7 +144,7 @@ pub trait Analyzer { } /// Specific information on a given analyzer -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, PartialEq)] #[cfg_attr(feature = "apidocs", derive(utoipa::ToSchema))] pub struct AnalyzerMetadata { /// The analyzer name @@ -156,7 +156,7 @@ pub struct AnalyzerMetadata { } /// The metadata for an analyzed report -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, PartialEq)] #[serde(default)] #[derive(Default)] #[cfg_attr(feature = "apidocs", derive(utoipa::ToSchema))] diff --git a/lib/src/util.rs b/lib/src/util.rs index b313269..9c22b13 100644 --- a/lib/src/util.rs +++ b/lib/src/util.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use nix::sys::utsname::uname; /// Expose binary and system information. -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, PartialEq)] #[cfg_attr(feature = "apidocs", derive(utoipa::ToSchema))] pub struct RuntimeMetadata { /// The cargo package version from this library's cargo.toml, e.g., "1.2.3".