Make WiFi tools optional to build, refactor installer file handling

PR #888 introduced more files that the installer needs to bundle. Those
files in particular are annoying to deal with because now every
developer needs a working C crosscompiler to get the installer working.
This prompted me to do some other refactoring.

Refactor install-dev to not build the wifi tools if there is no
crosscompiler, and refactor the installer so that these files are loaded
at runtime when built in debug mode.

The build script only ever warns if files are missing, and depending on
debug/release mode, the get_file!() macro either panics at runtime or
fails compiling.

Now the installer can be built again without any files, clippy can be
run directly without any envvars, and the installer runs atleast for
devices that don't need those files. The orbic installer will panic at
runtime if the wifi tools haven't been built. Building the installer in
release mode still requires all files.

Another nicety of loading these files on runtime is that the installer
does not need to be recompiled when the daemon has been rebuilt. This
should make things like make.sh really obsolete, which bypass the
installer for speed.

.github/workflows/main.yml

@@ -141,13 +141,13 @@ jobs:
         npm install
         npm run build
         popd
-        NO_FIRMWARE_BIN=true cargo check --verbose
+        cargo check --verbose
     - name: Run tests
       run: |
-        NO_FIRMWARE_BIN=true cargo test --verbose
+        cargo test --verbose
     - name: Run clippy
       run: |
-        NO_FIRMWARE_BIN=true cargo clippy --verbose
+        cargo clippy --verbose
 
   installer_gui_check:
     # we test the GUI installer separately to:
@@ -173,9 +173,9 @@ jobs:
     # fmt --all runs on all workspace packages so this is handled by
     # check_and_test above
     - name: Check
-      run: NO_FIRMWARE_BIN=true cargo check --package installer-gui --verbose
+      run: cargo check --package installer-gui --verbose
     - name: Run clippy
-      run: NO_FIRMWARE_BIN=true cargo clippy --package installer-gui --verbose
+      run: cargo clippy --package installer-gui --verbose
 
   test_daemon_frontend:
     needs: files_changed
@@ -227,12 +227,12 @@ jobs:
         shell: bash
         run: |
           cd installer
-          NO_FIRMWARE_BIN=true cargo check --verbose
+          cargo check --verbose
       - name: cargo test
         shell: bash
         run: |
           cd installer
-          NO_FIRMWARE_BIN=true cargo test --verbose --no-default-features
+          cargo test --verbose --no-default-features
 
   build_rayhunter_check:
     if: needs.files_changed.outputs.daemon_changed == 'true'

installer/build.rs

@@ -1,8 +1,6 @@
 use std::path::Path;
-use std::process::exit;
 
 fn main() {
-    println!("cargo::rerun-if-env-changed=NO_FIRMWARE_BIN");
     println!("cargo::rerun-if-env-changed=FIRMWARE_PROFILE");
     let profile = std::env::var("FIRMWARE_PROFILE").unwrap_or_else(|_| {
         // Default to firmware-devel for debug builds, firmware for release builds
@@ -26,24 +24,19 @@ fn main() {
 
 fn set_binary_var(include_dir: &Path, var: &str, file: &str) {
     println!("cargo::rerun-if-env-changed={var}");
-    if std::env::var_os("NO_FIRMWARE_BIN").is_some() {
+    if std::env::var_os(var).is_some() {
-        let out_dir = std::env::var("OUT_DIR").unwrap();
-        std::fs::create_dir_all(&out_dir).unwrap();
-        let blank = Path::new(&out_dir).join("blank");
-        std::fs::write(&blank, []).unwrap();
-        println!("cargo::rustc-env={var}={}", blank.display());
         return;
     }
-    if std::env::var_os(var).is_none() {
+    let binary = include_dir.join(file);
-        let binary = include_dir.join(file);
+    println!("cargo::rerun-if-changed={}", binary.display());
-        println!("cargo::rerun-if-changed={}", binary.display());
+    if binary.exists() {
-        if !binary.exists() {
-            println!(
-                "cargo::error=Firmware binary {file} not present at {}",
-                binary.display()
-            );
-            exit(0);
-        }
         println!("cargo::rustc-env={var}={}", binary.display());
+    } else {
+        println!(
+            "cargo::warning=Firmware binary {file} not present at {}; \
+             installers that need it will fail",
+            binary.display()
+        );
+        println!("cargo::rustc-env={var}=");
     }
 }

installer/src/files.rs

@@ -0,0 +1,23 @@
+#[cfg(debug_assertions)]
+macro_rules! get_file {
+    ($var:literal) => {{
+        let path = env!($var);
+        match ::std::fs::read(path) {
+            Ok(bytes) => bytes.leak() as &'static [u8],
+            Err(e) => panic!("Failed to read file for {}: {}", $var, e),
+        }
+    }};
+}
+
+#[cfg(not(debug_assertions))]
+macro_rules! get_file {
+    ($var:literal) => {{
+        const _: () = assert!(
+            !env!($var).is_empty(),
+            concat!($var, " was not bundled at build time"),
+        );
+        include_bytes!(env!($var)) as &'static [u8]
+    }};
+}
+
+pub(crate) use get_file;

installer/src/lib.rs

@@ -6,6 +6,9 @@ use env_logger::Env;
 use anyhow::bail;
 
 mod connection;
+mod files;
+pub(crate) use files::*;
+
 mod moxee;
 #[cfg(not(target_os = "android"))]
 mod orbic;

installer/src/orbic.rs

@@ -136,7 +136,7 @@ async fn force_debug_mode() -> Result<ADBUSBDevice> {
 }
 
 async fn setup_rootshell(adb_device: &mut ADBUSBDevice) -> Result<()> {
-    let rootshell_bin = include_bytes!(env!("FILE_ROOTSHELL"));
+    let rootshell_bin = crate::get_file!("FILE_ROOTSHELL");
 
     install_file(adb_device, "/bin/rootshell", rootshell_bin).await?;
     tokio::time::sleep(Duration::from_secs(1)).await;
@@ -151,7 +151,7 @@ async fn setup_rootshell(adb_device: &mut ADBUSBDevice) -> Result<()> {
 }
 
 async fn setup_rayhunter(mut adb_device: ADBUSBDevice, reset_config: bool) -> Result<ADBUSBDevice> {
-    let rayhunter_daemon_bin = include_bytes!(env!("FILE_RAYHUNTER_DAEMON"));
+    let rayhunter_daemon_bin = crate::get_file!("FILE_RAYHUNTER_DAEMON");
 
     adb_at_syscmd(
         &mut adb_device,
@@ -172,9 +172,9 @@ async fn setup_rayhunter(mut adb_device: ADBUSBDevice, reset_config: bool) -> Re
         install_config(&mut conn, "orbic", reset_config).await?;
         install_wifi_tools(
             &mut conn,
-            include_bytes!(env!("FILE_WPA_SUPPLICANT")),
+            crate::get_file!("FILE_WPA_SUPPLICANT"),
-            include_bytes!(env!("FILE_WPA_CLI")),
+            crate::get_file!("FILE_WPA_CLI"),
-            include_bytes!(env!("FILE_IW")),
+            crate::get_file!("FILE_IW"),
         )
         .await?;
     }

installer/src/orbic_network.rs

@@ -216,7 +216,7 @@ async fn wait_for_telnet(admin_ip: &str) -> Result<()> {
 
 async fn setup_rayhunter(admin_ip: &str, reset_config: bool, data_dir: &str) -> Result<()> {
     let addr = SocketAddr::from_str(&format!("{admin_ip}:{TELNET_PORT}"))?;
-    let rayhunter_daemon_bin = include_bytes!(env!("FILE_RAYHUNTER_DAEMON"));
+    let rayhunter_daemon_bin = crate::get_file!("FILE_RAYHUNTER_DAEMON");
 
     // Remount filesystem as read-write to allow modifications
     // This is really only necessary for the Moxee Hotspot
@@ -250,9 +250,9 @@ async fn setup_rayhunter(admin_ip: &str, reset_config: bool, data_dir: &str) ->
 
     install_wifi_tools(
         &mut conn,
-        include_bytes!(env!("FILE_WPA_SUPPLICANT")),
+        crate::get_file!("FILE_WPA_SUPPLICANT"),
-        include_bytes!(env!("FILE_WPA_CLI")),
+        crate::get_file!("FILE_WPA_CLI"),
-        include_bytes!(env!("FILE_IW")),
+        crate::get_file!("FILE_IW"),
     )
     .await?;
 

installer/src/pinephone.rs

@@ -29,7 +29,7 @@ pub async fn install() -> Result<()> {
     run_command_expect(&mut adb, "mount -o remount,rw /", "exit code 0").await?;
     run_command_expect(&mut adb, "mkdir -p /data/rayhunter", "exit code 0").await?;
 
-    let rayhunter_daemon_bin = include_bytes!(env!("FILE_RAYHUNTER_DAEMON"));
+    let rayhunter_daemon_bin = crate::get_file!("FILE_RAYHUNTER_DAEMON");
     adb.write_file("/data/rayhunter/rayhunter-daemon", rayhunter_daemon_bin)
         .await?;
     adb.write_file(

installer/src/tmobile.rs

@@ -48,7 +48,7 @@ async fn run_install(admin_ip: String, admin_password: String) -> Result<()> {
     )
     .await?;
 
-    let rayhunter_daemon_bin = include_bytes!(env!("FILE_RAYHUNTER_DAEMON"));
+    let rayhunter_daemon_bin = crate::get_file!("FILE_RAYHUNTER_DAEMON");
     telnet_send_file(
         addr,
         "/data/rayhunter/rayhunter-daemon",

installer/src/tplink.rs

@@ -188,7 +188,7 @@ async fn tplink_run_install(
 
     install_config(&mut conn, "tplink", reset_config).await?;
 
-    let rayhunter_daemon_bin = include_bytes!(env!("FILE_RAYHUNTER_DAEMON"));
+    let rayhunter_daemon_bin = crate::get_file!("FILE_RAYHUNTER_DAEMON");
 
     telnet_send_file(
         addr,

installer/src/uz801.rs

@@ -148,7 +148,7 @@ async fn install_rayhunter_files(adb_device: &mut ADBUSBDevice) -> Result<()> {
     adb_device.shell_command(&["mount", "-o", "remount,rw", "/system"], &mut buf)?;
 
     // Install rayhunter daemon binary with verification
-    let rayhunter_daemon_bin = include_bytes!(env!("FILE_RAYHUNTER_DAEMON"));
+    let rayhunter_daemon_bin = crate::get_file!("FILE_RAYHUNTER_DAEMON");
     install_file(
         adb_device,
         "/data/rayhunter/rayhunter-daemon",

installer/src/wingtech.rs

@@ -26,7 +26,6 @@ pub async fn install(
     Args {
         admin_ip,
         admin_password,
-        ..
     }: Args,
 ) -> Result<()> {
     wingtech_run_install(admin_ip, admin_password).await
@@ -105,7 +104,7 @@ async fn wingtech_run_install(admin_ip: String, admin_password: String) -> Resul
     )
     .await?;
 
-    let rayhunter_daemon_bin = include_bytes!(env!("FILE_RAYHUNTER_DAEMON"));
+    let rayhunter_daemon_bin = crate::get_file!("FILE_RAYHUNTER_DAEMON");
     telnet_send_file(
         addr,
         "/data/rayhunter/rayhunter-daemon",

scripts/build-dev.sh

@@ -52,10 +52,10 @@ build_wifi_tools() {
     fi
 
     if ! command -v arm-linux-musleabihf-gcc &> /dev/null; then
-        echo "Error: arm-linux-musleabihf-gcc not found."
+        echo "Warning: Skipping building WiFi tools due to missing C crosscompiler."
+        echo "arm-linux-musleabihf-gcc not found."
         echo "Install with: brew install FiloSottile/musl-cross/musl-cross"
-        echo "(Required because the installer bundles wpa_supplicant, wpa_cli, and iw for orbic-family devices.)"
+        return
-        exit 1
     fi
 
     echo "Building WiFi tools..."