Merge pull request #114 from Next-Flip/gzipped-resources

Updater: Gzipped resources (220% faster FW upload) + Updater improvements

.gitmodules

@@ -47,3 +47,6 @@
 [submodule "lib/tlsf"]
 	path = lib/tlsf
 	url = https://github.com/espressif/tlsf
+[submodule "lib/uzlib"]
+	path = lib/uzlib
+	url = https://github.com/pfalcon/uzlib.git

applications/system/updater/util/update_task.c

@@ -238,7 +238,7 @@ static const UpdateTaskStageGroupMap update_task_stage_progress[] = {
     [UpdateTaskStageOBValidation] = STAGE_DEF(UpdateTaskStageGroupOptionBytes, 2),
 
     [UpdateTaskStageValidateDFUImage] = STAGE_DEF(UpdateTaskStageGroupFirmware, 30),
-    [UpdateTaskStageFlashWrite] = STAGE_DEF(UpdateTaskStageGroupFirmware, 150),
+    [UpdateTaskStageFlashWrite] = STAGE_DEF(UpdateTaskStageGroupFirmware, 75),
     [UpdateTaskStageFlashValidate] = STAGE_DEF(UpdateTaskStageGroupFirmware, 15),
 
     [UpdateTaskStageLfsRestore] = STAGE_DEF(UpdateTaskStageGroupPostUpdate, 5),
@@ -334,11 +334,23 @@ void update_task_set_progress(UpdateTask* update_task, UpdateTaskStage stage, ui
     update_task->state.overall_progress = adapted_progress;
 
     if(update_task->status_change_cb) {
-        (update_task->status_change_cb)(
-            furi_string_get_cstr(update_task->state.status),
-            adapted_progress,
-            update_stage_is_error(update_task->state.stage),
-            update_task->status_change_cb_state);
+        if(update_stage_is_error(update_task->state.stage)) {
+            (update_task->status_change_cb)(
+                furi_string_get_cstr(update_task->state.status),
+                adapted_progress,
+                update_stage_is_error(update_task->state.stage),
+                update_task->status_change_cb_state);
+        } else {
+            size_t len = furi_string_size(update_task->state.status) + strlen(" 100%") + 1;
+            char* s = malloc(len);
+            snprintf(s, len, "%s %d%%", furi_string_get_cstr(update_task->state.status), progress);
+            (update_task->status_change_cb)(
+                s,
+                adapted_progress,
+                update_stage_is_error(update_task->state.stage),
+                update_task->status_change_cb_state);
+            free(s);
+        }
     }
 }
 

applications/system/updater/util/update_task_worker_backup.c

@@ -10,6 +10,8 @@
 #include <update_util/lfs_backup.h>
 #include <update_util/update_operation.h>
 #include <update_util/resources/manifest.h>
+#include <update_util/resources/manifest_i.h>
+#include <toolbox/stream/stream.h>
 #include <toolbox/tar/tar_archive.h>
 #include <toolbox/crc32_calc.h>
 
@@ -39,34 +41,24 @@ static bool update_task_pre_update(UpdateTask* update_task) {
 }
 
 typedef enum {
-    UpdateTaskResourcesWeightsFileCleanup = 20,
-    UpdateTaskResourcesWeightsDirCleanup = 20,
-    UpdateTaskResourcesWeightsFileUnpack = 60,
+    UpdateTaskResourcesWeightsFileCleanup = 10,
+    UpdateTaskResourcesWeightsDirCleanup = 10,
+    UpdateTaskResourcesWeightsFileUnpack = 80,
 } UpdateTaskResourcesWeights;
 
 #define UPDATE_TASK_RESOURCES_FILE_TO_TOTAL_PERCENT 90
 
-typedef struct {
-    UpdateTask* update_task;
-    int32_t total_files, processed_files;
-} TarUnpackProgress;
-
-static bool update_task_resource_unpack_cb(const char* name, bool is_directory, void* context) {
-    UNUSED(name);
-    UNUSED(is_directory);
-    TarUnpackProgress* unpack_progress = context;
-    unpack_progress->processed_files++;
+static void update_task_resource_progress_cb(size_t progress, size_t total, void* context) {
+    UpdateTask* update_task = context;
     update_task_set_progress(
-        unpack_progress->update_task,
+        update_task,
         UpdateTaskStageProgress,
         /* For this stage, last progress segment = extraction */
         (UpdateTaskResourcesWeightsFileCleanup + UpdateTaskResourcesWeightsDirCleanup) +
-            (unpack_progress->processed_files * UpdateTaskResourcesWeightsFileUnpack) /
-                (unpack_progress->total_files + 1));
-    return true;
+            (progress * UpdateTaskResourcesWeightsFileUnpack) / total);
 }
 
-static void update_task_cleanup_resources(UpdateTask* update_task, const uint32_t n_tar_entries) {
+static void update_task_cleanup_resources(UpdateTask* update_task) {
     ResourceManifestReader* manifest_reader = resource_manifest_reader_alloc(update_task->storage);
     do {
         FURI_LOG_D(TAG, "Cleaning up old manifest");
@@ -75,8 +67,7 @@ static void update_task_cleanup_resources(UpdateTask* update_task, const uint32_
             break;
         }
 
-        const uint32_t n_approx_file_entries =
-            n_tar_entries * UPDATE_TASK_RESOURCES_FILE_TO_TOTAL_PERCENT / 100 + 1;
+        size_t manifest_size = stream_size(manifest_reader->stream);
         uint32_t n_dir_entries = 1;
 
         ResourceManifestEntry* entry_ptr = NULL;
@@ -87,8 +78,9 @@ static void update_task_cleanup_resources(UpdateTask* update_task, const uint32_
                     update_task,
                     UpdateTaskStageProgress,
                     /* For this stage, first pass = old manifest's file cleanup */
-                    (n_processed_entries++ * UpdateTaskResourcesWeightsFileCleanup) /
-                        n_approx_file_entries);
+                    (stream_tell(manifest_reader->stream) *
+                     UpdateTaskResourcesWeightsFileCleanup) /
+                        manifest_size);
 
                 FuriString* file_path = furi_string_alloc();
                 path_concat(
@@ -177,11 +169,6 @@ static bool update_task_post_update(UpdateTask* update_task) {
 #endif
 
         if(update_task->state.groups & UpdateTaskStageGroupResources) {
-            TarUnpackProgress progress = {
-                .update_task = update_task,
-                .total_files = 0,
-                .processed_files = 0,
-            };
             update_task_set_progress(update_task, UpdateTaskStageResourcesUpdate, 0);
 
             path_concat(
@@ -189,16 +176,13 @@ static bool update_task_post_update(UpdateTask* update_task) {
                 furi_string_get_cstr(update_task->manifest->resource_bundle),
                 file_path);
 
-            tar_archive_set_file_callback(archive, update_task_resource_unpack_cb, &progress);
+            tar_archive_set_read_callback(archive, update_task_resource_progress_cb, update_task);
             CHECK_RESULT(
                 tar_archive_open(archive, furi_string_get_cstr(file_path), TAR_OPEN_MODE_READ));
 
-            progress.total_files = tar_archive_get_entries_count(archive);
-            if(progress.total_files > 0) {
-                update_task_cleanup_resources(update_task, progress.total_files);
+            update_task_cleanup_resources(update_task);
 
-                CHECK_RESULT(tar_archive_unpack_to(archive, STORAGE_EXT_PATH_PREFIX, NULL));
-            }
+            CHECK_RESULT(tar_archive_unpack_to(archive, STORAGE_EXT_PATH_PREFIX, NULL));
         }
 
         if(update_task->state.groups & UpdateTaskStageGroupSplashscreen) {

lib/SConscript

@@ -40,6 +40,7 @@ libs = env.BuildModules(
         "lfrfid",
         "flipper_application",
         "music_worker",
+        "uzlib",
         "mjs",
         "nanopb",
         "update_util",

lib/toolbox/tar/tar_archive.c

@@ -5,6 +5,110 @@
 #include <furi.h>
 #include <toolbox/path.h>
 
+#include <lib/uzlib/src/uzlib.h>
+
+typedef struct {
+    File* file;
+    uint8_t* buffer;
+    size_t buffer_size;
+    uint8_t* dict;
+    size_t dict_size;
+    struct uzlib_uncomp uzlib;
+
+    uint32_t source_pos;
+    uint32_t dest_pos;
+    bool eof;
+} Gunzip;
+
+int gunzip_read_cb(struct uzlib_uncomp* uncomp) {
+    void* data_p = uncomp;
+    data_p -= offsetof(Gunzip, uzlib);
+    Gunzip* gunzip = data_p;
+
+    uint16_t read_size = storage_file_read(gunzip->file, gunzip->buffer, gunzip->buffer_size);
+    gunzip->uzlib.source = &gunzip->buffer[1]; // we will return buffer[0] at exit
+    gunzip->uzlib.source_limit = gunzip->buffer + read_size;
+
+    if(read_size == 0) {
+        return -1;
+    }
+    gunzip->source_pos += read_size;
+
+    return gunzip->buffer[0];
+}
+
+Gunzip* gunzip_alloc(File* file, size_t dict_size, size_t buffer_size) {
+    Gunzip* gunzip = malloc(sizeof(Gunzip));
+    gunzip->file = file;
+    gunzip->buffer_size = buffer_size;
+    gunzip->buffer = malloc(buffer_size);
+    gunzip->dict_size = dict_size;
+    gunzip->dict = malloc(dict_size);
+
+    uzlib_uncompress_init(&gunzip->uzlib, gunzip->dict, gunzip->dict_size);
+
+    gunzip->uzlib.source = 0;
+    gunzip->uzlib.source_limit = 0;
+    gunzip->uzlib.source_read_cb = gunzip_read_cb;
+    gunzip->source_pos = 0;
+    gunzip->dest_pos = 0;
+    gunzip->eof = false;
+
+    return gunzip;
+}
+
+void gunzip_free(Gunzip* gunzip) {
+    free(gunzip->buffer);
+    free(gunzip->dict);
+    free(gunzip);
+}
+
+int32_t gunzip_uncompress(Gunzip* gunzip, void* out, size_t out_len) {
+    if(gunzip->eof) {
+        return 0;
+    }
+
+    gunzip->uzlib.dest = out;
+    gunzip->uzlib.dest_limit = (uint8_t*)out + out_len;
+
+    int res = uzlib_uncompress_chksum(&gunzip->uzlib);
+
+    if(res == TINF_DONE) {
+        gunzip->eof = true;
+    }
+    if(res < 0) {
+        return res;
+    }
+    int32_t read = gunzip->uzlib.dest - (uint8_t*)out;
+    gunzip->dest_pos += read;
+    return read;
+}
+
+int32_t gunzip_seek(Gunzip* gunzip, size_t pos) {
+    if(pos == gunzip->dest_pos) {
+        return 0;
+    }
+
+    if(pos < gunzip->dest_pos) {
+        // TODO: rewind to start if this causes issues
+        furi_crash("Gunzip rewind");
+        return -1;
+    }
+
+    size_t void_size = MIN(4096U, pos - gunzip->dest_pos);
+    void* void_buf = malloc(void_size);
+    while(!gunzip->eof && gunzip->dest_pos < pos) {
+        size_t uncompress_size = MIN(void_size, pos - gunzip->dest_pos);
+        int res = gunzip_uncompress(gunzip, void_buf, uncompress_size);
+        if(res < 0) {
+            free(void_buf);
+            return res;
+        }
+    }
+    free(void_buf);
+    return (pos == gunzip->dest_pos) ? 0 : -1;
+}
+
 #define TAG "TarArch"
 #define MAX_NAME_LEN 254
 #define FILE_BLOCK_SIZE 512
@@ -17,6 +121,11 @@ typedef struct TarArchive {
     mtar_t tar;
     tar_unpack_file_cb unpack_cb;
     void* unpack_cb_context;
+
+    tar_unpack_read_cb read_cb;
+    void* read_cb_context;
+    size_t total_size;
+    Gunzip* gunzip;
 } TarArchive;
 
 /* API WRAPPER */
@@ -50,6 +159,49 @@ const struct mtar_ops filesystem_ops = {
     .close = mtar_storage_file_close,
 };
 
+static int mtar_storage_gunzip_write(void* gunzip, const void* data, unsigned size) {
+    UNUSED(gunzip);
+    UNUSED(data);
+    UNUSED(size);
+    furi_crash("Write to gzipped tar");
+    return MTAR_EWRITEFAIL;
+}
+
+static int mtar_storage_gunzip_read(void* gunzip, void* data, unsigned size) {
+    int32_t res = gunzip_uncompress(gunzip, data, size);
+    if(res < 0) {
+        FURI_LOG_E(TAG, "Error uncompressing gzip: %ld\n", res);
+    }
+    return (res == (int32_t)size) ? res : MTAR_EREADFAIL;
+}
+
+static int mtar_storage_gunzip_seek(void* gunzip, unsigned offset) {
+    int32_t res = gunzip_seek(gunzip, offset);
+    if(res < 0) {
+        FURI_LOG_E(TAG, "Error seeking gzip: %ld\n", res);
+    }
+    return (res == 0) ? MTAR_ESUCCESS : MTAR_ESEEKFAIL;
+}
+
+static int mtar_storage_gunzip_close(void* _gunzip) {
+    Gunzip* gunzip = _gunzip;
+    if(gunzip) {
+        if(gunzip->file) {
+            storage_file_close(gunzip->file);
+            storage_file_free(gunzip->file);
+        }
+        gunzip_free(gunzip);
+    }
+    return MTAR_ESUCCESS;
+}
+
+const struct mtar_ops gunzip_ops = {
+    .read = mtar_storage_gunzip_read,
+    .write = mtar_storage_gunzip_write,
+    .seek = mtar_storage_gunzip_seek,
+    .close = mtar_storage_gunzip_close,
+};
+
 TarArchive* tar_archive_alloc(Storage* storage) {
     furi_check(storage);
     TarArchive* archive = malloc(sizeof(TarArchive));
@@ -84,7 +236,26 @@ bool tar_archive_open(TarArchive* archive, const char* path, TarOpenMode mode) {
         storage_file_free(stream);
         return false;
     }
-    mtar_init(&archive->tar, mtar_access, &filesystem_ops, stream);
+    archive->total_size = storage_file_size(stream);
+
+    char* dot = strrchr(path, '.');
+    if(dot == NULL || strcmp(dot, ".gz") != 0 || mode != TAR_OPEN_MODE_READ) {
+        mtar_init(&archive->tar, mtar_access, &filesystem_ops, stream);
+    } else {
+        archive->gunzip = gunzip_alloc(stream, 32 * 1024, 10 * 1024);
+
+        int res = uzlib_gzip_parse_header(&archive->gunzip->uzlib);
+        if(res != TINF_OK) {
+            FURI_LOG_E(TAG, "Error parsing gzip header: %d\n", res);
+            storage_file_close(stream);
+            storage_file_free(stream);
+            gunzip_free(archive->gunzip);
+            archive->gunzip = NULL;
+            return false;
+        }
+
+        mtar_init(&archive->tar, mtar_access, &gunzip_ops, archive->gunzip);
+    }
 
     return true;
 }
@@ -103,6 +274,12 @@ void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callb
     archive->unpack_cb_context = context;
 }
 
+void tar_archive_set_read_callback(TarArchive* archive, tar_unpack_read_cb callback, void* context) {
+    furi_check(archive);
+    archive->read_cb = callback;
+    archive->read_cb_context = context;
+}
+
 static int tar_archive_entry_counter(mtar_t* tar, const mtar_header_t* header, void* param) {
     UNUSED(tar);
     UNUSED(header);
@@ -198,6 +375,13 @@ static bool archive_extract_current_file(TarArchive* archive, const char* dst_pa
                 success = false;
                 break;
             }
+
+            if(archive->read_cb) {
+                archive->read_cb(
+                    archive->gunzip ? archive->gunzip->source_pos : archive->tar.pos,
+                    archive->total_size,
+                    archive->read_cb_context);
+            }
         }
     } while(false);
     storage_file_free(out_file);

lib/toolbox/tar/tar_archive.h

@@ -50,6 +50,11 @@ typedef bool (*tar_unpack_file_cb)(const char* name, bool is_directory, void* co
 
 void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context);
 
+/* Optional read progress callback on unpacking */
+typedef void (*tar_unpack_read_cb)(size_t progress, size_t total, void* context);
+
+void tar_archive_set_read_callback(TarArchive* archive, tar_unpack_read_cb callback, void* context);
+
 /* Low-level API */
 bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath);
 

lib/update_util/resources/manifest.c

@@ -3,12 +3,7 @@
 #include <toolbox/stream/buffered_file_stream.h>
 #include <toolbox/hex.h>
 
-struct ResourceManifestReader {
-    Storage* storage;
-    Stream* stream;
-    FuriString* linebuf;
-    ResourceManifestEntry entry;
-};
+#include "manifest_i.h"
 
 ResourceManifestReader* resource_manifest_reader_alloc(Storage* storage) {
     ResourceManifestReader* resource_manifest =

lib/update_util/resources/manifest_i.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <toolbox/stream/buffered_file_stream.h>
+
+struct ResourceManifestReader {
+    Storage* storage;
+    Stream* stream;
+    FuriString* linebuf;
+    ResourceManifestEntry entry;
+};

lib/uzlib.scons

@@ -0,0 +1,28 @@
+Import("env")
+
+env.Append(
+    CPPPATH=[
+        "#/lib/uzlib/src",
+    ],
+)
+
+libenv = env.Clone(FW_LIB_NAME="uzlib")
+libenv.ApplyLibFlags()
+
+libenv.AppendUnique(
+    CCFLAGS=[
+        "-Wno-redundant-decls",
+        "-Wno-sign-compare",
+    ],
+)
+
+sources = [
+    File("uzlib/src/adler32.c"),
+    File("uzlib/src/crc32.c"),
+    File("uzlib/src/tinfgzip.c"),
+    File("uzlib/src/tinflate.c"),
+]
+
+lib = libenv.StaticLibrary("${FW_LIB_NAME}", sources)
+libenv.Install("${LIB_DIST_DIR}", lib)
+Return("lib")

scripts/update.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import math
+import gzip
 import os
 import shutil
 import tarfile
@@ -158,6 +159,14 @@ class Main(App):
                 self.args.resources, join(self.args.directory, resources_basename)
             ):
                 return 3
+            resources_path = join(self.args.directory, resources_basename)
+            with open(resources_path, "rb") as f_raw:
+                resources_raw = f_raw.read()
+            os.unlink(resources_path)
+            resources_basename += ".gz"
+            resources_path += ".gz"
+            with gzip.open(resources_path, "wb", compresslevel=9) as f_zip:
+                f_zip.write(resources_raw)
 
         if not self.layout_check(dfu_size, radio_addr):
             self.logger.warn("Memory layout looks suspicious")

targets/f7/api_symbols.csv

@@ -3564,6 +3564,7 @@ Function,+,tar_archive_free,void,TarArchive*
 Function,+,tar_archive_get_entries_count,int32_t,TarArchive*
 Function,+,tar_archive_open,_Bool,"TarArchive*, const char*, TarOpenMode"
 Function,+,tar_archive_set_file_callback,void,"TarArchive*, tar_unpack_file_cb, void*"
+Function,+,tar_archive_set_read_callback,void,"TarArchive*, tar_unpack_read_cb, void*"
 Function,+,tar_archive_store_data,_Bool,"TarArchive*, const char*, const uint8_t*, const int32_t"
 Function,+,tar_archive_unpack_file,_Bool,"TarArchive*, const char*, const char*"
 Function,+,tar_archive_unpack_to,_Bool,"TarArchive*, const char*, Storage_name_converter"

targets/f7/target.json

@@ -32,6 +32,7 @@
         "toolbox",
         "nfc",
         "digital_signal",
+        "uzlib",
         "pulse_reader",
         "signal_reader",
         "microtar",