diff --git a/.gitmodules b/.gitmodules index f6bb3b18a..cf41ef5cc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -47,3 +47,6 @@ [submodule "lib/tlsf"] path = lib/tlsf url = https://github.com/espressif/tlsf +[submodule "lib/uzlib"] + path = lib/uzlib + url = https://github.com/pfalcon/uzlib.git diff --git a/applications/system/updater/util/update_task.c b/applications/system/updater/util/update_task.c index c0ea6421c..47b2746ce 100644 --- a/applications/system/updater/util/update_task.c +++ b/applications/system/updater/util/update_task.c @@ -238,7 +238,7 @@ static const UpdateTaskStageGroupMap update_task_stage_progress[] = { [UpdateTaskStageOBValidation] = STAGE_DEF(UpdateTaskStageGroupOptionBytes, 2), [UpdateTaskStageValidateDFUImage] = STAGE_DEF(UpdateTaskStageGroupFirmware, 30), - [UpdateTaskStageFlashWrite] = STAGE_DEF(UpdateTaskStageGroupFirmware, 150), + [UpdateTaskStageFlashWrite] = STAGE_DEF(UpdateTaskStageGroupFirmware, 75), [UpdateTaskStageFlashValidate] = STAGE_DEF(UpdateTaskStageGroupFirmware, 15), [UpdateTaskStageLfsRestore] = STAGE_DEF(UpdateTaskStageGroupPostUpdate, 5), @@ -334,11 +334,23 @@ void update_task_set_progress(UpdateTask* update_task, UpdateTaskStage stage, ui update_task->state.overall_progress = adapted_progress; if(update_task->status_change_cb) { - (update_task->status_change_cb)( - furi_string_get_cstr(update_task->state.status), - adapted_progress, - update_stage_is_error(update_task->state.stage), - update_task->status_change_cb_state); + if(update_stage_is_error(update_task->state.stage)) { + (update_task->status_change_cb)( + furi_string_get_cstr(update_task->state.status), + adapted_progress, + update_stage_is_error(update_task->state.stage), + update_task->status_change_cb_state); + } else { + size_t len = furi_string_size(update_task->state.status) + strlen(" 100%") + 1; + char* s = malloc(len); + snprintf(s, len, "%s %d%%", furi_string_get_cstr(update_task->state.status), progress); + (update_task->status_change_cb)( + s, + adapted_progress, + update_stage_is_error(update_task->state.stage), + update_task->status_change_cb_state); + free(s); + } } } diff --git a/applications/system/updater/util/update_task_worker_backup.c b/applications/system/updater/util/update_task_worker_backup.c index f817ea013..ac052ded8 100644 --- a/applications/system/updater/util/update_task_worker_backup.c +++ b/applications/system/updater/util/update_task_worker_backup.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include @@ -39,34 +41,24 @@ static bool update_task_pre_update(UpdateTask* update_task) { } typedef enum { - UpdateTaskResourcesWeightsFileCleanup = 20, - UpdateTaskResourcesWeightsDirCleanup = 20, - UpdateTaskResourcesWeightsFileUnpack = 60, + UpdateTaskResourcesWeightsFileCleanup = 10, + UpdateTaskResourcesWeightsDirCleanup = 10, + UpdateTaskResourcesWeightsFileUnpack = 80, } UpdateTaskResourcesWeights; #define UPDATE_TASK_RESOURCES_FILE_TO_TOTAL_PERCENT 90 -typedef struct { - UpdateTask* update_task; - int32_t total_files, processed_files; -} TarUnpackProgress; - -static bool update_task_resource_unpack_cb(const char* name, bool is_directory, void* context) { - UNUSED(name); - UNUSED(is_directory); - TarUnpackProgress* unpack_progress = context; - unpack_progress->processed_files++; +static void update_task_resource_progress_cb(size_t progress, size_t total, void* context) { + UpdateTask* update_task = context; update_task_set_progress( - unpack_progress->update_task, + update_task, UpdateTaskStageProgress, /* For this stage, last progress segment = extraction */ (UpdateTaskResourcesWeightsFileCleanup + UpdateTaskResourcesWeightsDirCleanup) + - (unpack_progress->processed_files * UpdateTaskResourcesWeightsFileUnpack) / - (unpack_progress->total_files + 1)); - return true; + (progress * UpdateTaskResourcesWeightsFileUnpack) / total); } -static void update_task_cleanup_resources(UpdateTask* update_task, const uint32_t n_tar_entries) { +static void update_task_cleanup_resources(UpdateTask* update_task) { ResourceManifestReader* manifest_reader = resource_manifest_reader_alloc(update_task->storage); do { FURI_LOG_D(TAG, "Cleaning up old manifest"); @@ -75,8 +67,7 @@ static void update_task_cleanup_resources(UpdateTask* update_task, const uint32_ break; } - const uint32_t n_approx_file_entries = - n_tar_entries * UPDATE_TASK_RESOURCES_FILE_TO_TOTAL_PERCENT / 100 + 1; + size_t manifest_size = stream_size(manifest_reader->stream); uint32_t n_dir_entries = 1; ResourceManifestEntry* entry_ptr = NULL; @@ -87,8 +78,9 @@ static void update_task_cleanup_resources(UpdateTask* update_task, const uint32_ update_task, UpdateTaskStageProgress, /* For this stage, first pass = old manifest's file cleanup */ - (n_processed_entries++ * UpdateTaskResourcesWeightsFileCleanup) / - n_approx_file_entries); + (stream_tell(manifest_reader->stream) * + UpdateTaskResourcesWeightsFileCleanup) / + manifest_size); FuriString* file_path = furi_string_alloc(); path_concat( @@ -177,11 +169,6 @@ static bool update_task_post_update(UpdateTask* update_task) { #endif if(update_task->state.groups & UpdateTaskStageGroupResources) { - TarUnpackProgress progress = { - .update_task = update_task, - .total_files = 0, - .processed_files = 0, - }; update_task_set_progress(update_task, UpdateTaskStageResourcesUpdate, 0); path_concat( @@ -189,16 +176,13 @@ static bool update_task_post_update(UpdateTask* update_task) { furi_string_get_cstr(update_task->manifest->resource_bundle), file_path); - tar_archive_set_file_callback(archive, update_task_resource_unpack_cb, &progress); + tar_archive_set_read_callback(archive, update_task_resource_progress_cb, update_task); CHECK_RESULT( tar_archive_open(archive, furi_string_get_cstr(file_path), TAR_OPEN_MODE_READ)); - progress.total_files = tar_archive_get_entries_count(archive); - if(progress.total_files > 0) { - update_task_cleanup_resources(update_task, progress.total_files); + update_task_cleanup_resources(update_task); - CHECK_RESULT(tar_archive_unpack_to(archive, STORAGE_EXT_PATH_PREFIX, NULL)); - } + CHECK_RESULT(tar_archive_unpack_to(archive, STORAGE_EXT_PATH_PREFIX, NULL)); } if(update_task->state.groups & UpdateTaskStageGroupSplashscreen) { diff --git a/lib/SConscript b/lib/SConscript index 61221d87a..e5dbe048b 100644 --- a/lib/SConscript +++ b/lib/SConscript @@ -40,6 +40,7 @@ libs = env.BuildModules( "lfrfid", "flipper_application", "music_worker", + "uzlib", "mjs", "nanopb", "update_util", diff --git a/lib/toolbox/tar/tar_archive.c b/lib/toolbox/tar/tar_archive.c index b469a4d9d..01244b0f5 100644 --- a/lib/toolbox/tar/tar_archive.c +++ b/lib/toolbox/tar/tar_archive.c @@ -5,6 +5,110 @@ #include #include +#include + +typedef struct { + File* file; + uint8_t* buffer; + size_t buffer_size; + uint8_t* dict; + size_t dict_size; + struct uzlib_uncomp uzlib; + + uint32_t source_pos; + uint32_t dest_pos; + bool eof; +} Gunzip; + +int gunzip_read_cb(struct uzlib_uncomp* uncomp) { + void* data_p = uncomp; + data_p -= offsetof(Gunzip, uzlib); + Gunzip* gunzip = data_p; + + uint16_t read_size = storage_file_read(gunzip->file, gunzip->buffer, gunzip->buffer_size); + gunzip->uzlib.source = &gunzip->buffer[1]; // we will return buffer[0] at exit + gunzip->uzlib.source_limit = gunzip->buffer + read_size; + + if(read_size == 0) { + return -1; + } + gunzip->source_pos += read_size; + + return gunzip->buffer[0]; +} + +Gunzip* gunzip_alloc(File* file, size_t dict_size, size_t buffer_size) { + Gunzip* gunzip = malloc(sizeof(Gunzip)); + gunzip->file = file; + gunzip->buffer_size = buffer_size; + gunzip->buffer = malloc(buffer_size); + gunzip->dict_size = dict_size; + gunzip->dict = malloc(dict_size); + + uzlib_uncompress_init(&gunzip->uzlib, gunzip->dict, gunzip->dict_size); + + gunzip->uzlib.source = 0; + gunzip->uzlib.source_limit = 0; + gunzip->uzlib.source_read_cb = gunzip_read_cb; + gunzip->source_pos = 0; + gunzip->dest_pos = 0; + gunzip->eof = false; + + return gunzip; +} + +void gunzip_free(Gunzip* gunzip) { + free(gunzip->buffer); + free(gunzip->dict); + free(gunzip); +} + +int32_t gunzip_uncompress(Gunzip* gunzip, void* out, size_t out_len) { + if(gunzip->eof) { + return 0; + } + + gunzip->uzlib.dest = out; + gunzip->uzlib.dest_limit = (uint8_t*)out + out_len; + + int res = uzlib_uncompress_chksum(&gunzip->uzlib); + + if(res == TINF_DONE) { + gunzip->eof = true; + } + if(res < 0) { + return res; + } + int32_t read = gunzip->uzlib.dest - (uint8_t*)out; + gunzip->dest_pos += read; + return read; +} + +int32_t gunzip_seek(Gunzip* gunzip, size_t pos) { + if(pos == gunzip->dest_pos) { + return 0; + } + + if(pos < gunzip->dest_pos) { + // TODO: rewind to start if this causes issues + furi_crash("Gunzip rewind"); + return -1; + } + + size_t void_size = MIN(4096U, pos - gunzip->dest_pos); + void* void_buf = malloc(void_size); + while(!gunzip->eof && gunzip->dest_pos < pos) { + size_t uncompress_size = MIN(void_size, pos - gunzip->dest_pos); + int res = gunzip_uncompress(gunzip, void_buf, uncompress_size); + if(res < 0) { + free(void_buf); + return res; + } + } + free(void_buf); + return (pos == gunzip->dest_pos) ? 0 : -1; +} + #define TAG "TarArch" #define MAX_NAME_LEN 254 #define FILE_BLOCK_SIZE 512 @@ -17,6 +121,11 @@ typedef struct TarArchive { mtar_t tar; tar_unpack_file_cb unpack_cb; void* unpack_cb_context; + + tar_unpack_read_cb read_cb; + void* read_cb_context; + size_t total_size; + Gunzip* gunzip; } TarArchive; /* API WRAPPER */ @@ -50,6 +159,49 @@ const struct mtar_ops filesystem_ops = { .close = mtar_storage_file_close, }; +static int mtar_storage_gunzip_write(void* gunzip, const void* data, unsigned size) { + UNUSED(gunzip); + UNUSED(data); + UNUSED(size); + furi_crash("Write to gzipped tar"); + return MTAR_EWRITEFAIL; +} + +static int mtar_storage_gunzip_read(void* gunzip, void* data, unsigned size) { + int32_t res = gunzip_uncompress(gunzip, data, size); + if(res < 0) { + FURI_LOG_E(TAG, "Error uncompressing gzip: %ld\n", res); + } + return (res == (int32_t)size) ? res : MTAR_EREADFAIL; +} + +static int mtar_storage_gunzip_seek(void* gunzip, unsigned offset) { + int32_t res = gunzip_seek(gunzip, offset); + if(res < 0) { + FURI_LOG_E(TAG, "Error seeking gzip: %ld\n", res); + } + return (res == 0) ? MTAR_ESUCCESS : MTAR_ESEEKFAIL; +} + +static int mtar_storage_gunzip_close(void* _gunzip) { + Gunzip* gunzip = _gunzip; + if(gunzip) { + if(gunzip->file) { + storage_file_close(gunzip->file); + storage_file_free(gunzip->file); + } + gunzip_free(gunzip); + } + return MTAR_ESUCCESS; +} + +const struct mtar_ops gunzip_ops = { + .read = mtar_storage_gunzip_read, + .write = mtar_storage_gunzip_write, + .seek = mtar_storage_gunzip_seek, + .close = mtar_storage_gunzip_close, +}; + TarArchive* tar_archive_alloc(Storage* storage) { furi_check(storage); TarArchive* archive = malloc(sizeof(TarArchive)); @@ -84,7 +236,26 @@ bool tar_archive_open(TarArchive* archive, const char* path, TarOpenMode mode) { storage_file_free(stream); return false; } - mtar_init(&archive->tar, mtar_access, &filesystem_ops, stream); + archive->total_size = storage_file_size(stream); + + char* dot = strrchr(path, '.'); + if(dot == NULL || strcmp(dot, ".gz") != 0 || mode != TAR_OPEN_MODE_READ) { + mtar_init(&archive->tar, mtar_access, &filesystem_ops, stream); + } else { + archive->gunzip = gunzip_alloc(stream, 32 * 1024, 10 * 1024); + + int res = uzlib_gzip_parse_header(&archive->gunzip->uzlib); + if(res != TINF_OK) { + FURI_LOG_E(TAG, "Error parsing gzip header: %d\n", res); + storage_file_close(stream); + storage_file_free(stream); + gunzip_free(archive->gunzip); + archive->gunzip = NULL; + return false; + } + + mtar_init(&archive->tar, mtar_access, &gunzip_ops, archive->gunzip); + } return true; } @@ -103,6 +274,12 @@ void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callb archive->unpack_cb_context = context; } +void tar_archive_set_read_callback(TarArchive* archive, tar_unpack_read_cb callback, void* context) { + furi_check(archive); + archive->read_cb = callback; + archive->read_cb_context = context; +} + static int tar_archive_entry_counter(mtar_t* tar, const mtar_header_t* header, void* param) { UNUSED(tar); UNUSED(header); @@ -198,6 +375,13 @@ static bool archive_extract_current_file(TarArchive* archive, const char* dst_pa success = false; break; } + + if(archive->read_cb) { + archive->read_cb( + archive->gunzip ? archive->gunzip->source_pos : archive->tar.pos, + archive->total_size, + archive->read_cb_context); + } } } while(false); storage_file_free(out_file); diff --git a/lib/toolbox/tar/tar_archive.h b/lib/toolbox/tar/tar_archive.h index ba2f7749f..22f09c2a4 100644 --- a/lib/toolbox/tar/tar_archive.h +++ b/lib/toolbox/tar/tar_archive.h @@ -50,6 +50,11 @@ typedef bool (*tar_unpack_file_cb)(const char* name, bool is_directory, void* co void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context); +/* Optional read progress callback on unpacking */ +typedef void (*tar_unpack_read_cb)(size_t progress, size_t total, void* context); + +void tar_archive_set_read_callback(TarArchive* archive, tar_unpack_read_cb callback, void* context); + /* Low-level API */ bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath); diff --git a/lib/update_util/resources/manifest.c b/lib/update_util/resources/manifest.c index 5a818a0a4..bf6311ae8 100644 --- a/lib/update_util/resources/manifest.c +++ b/lib/update_util/resources/manifest.c @@ -3,12 +3,7 @@ #include #include -struct ResourceManifestReader { - Storage* storage; - Stream* stream; - FuriString* linebuf; - ResourceManifestEntry entry; -}; +#include "manifest_i.h" ResourceManifestReader* resource_manifest_reader_alloc(Storage* storage) { ResourceManifestReader* resource_manifest = diff --git a/lib/update_util/resources/manifest_i.h b/lib/update_util/resources/manifest_i.h new file mode 100644 index 000000000..39114d023 --- /dev/null +++ b/lib/update_util/resources/manifest_i.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +struct ResourceManifestReader { + Storage* storage; + Stream* stream; + FuriString* linebuf; + ResourceManifestEntry entry; +}; diff --git a/lib/uzlib b/lib/uzlib new file mode 160000 index 000000000..6d60d651a --- /dev/null +++ b/lib/uzlib @@ -0,0 +1 @@ +Subproject commit 6d60d651a4499a64f2e5b21b4cc08d98cb84b5c1 diff --git a/lib/uzlib.scons b/lib/uzlib.scons new file mode 100644 index 000000000..d7a3c5340 --- /dev/null +++ b/lib/uzlib.scons @@ -0,0 +1,28 @@ +Import("env") + +env.Append( + CPPPATH=[ + "#/lib/uzlib/src", + ], +) + +libenv = env.Clone(FW_LIB_NAME="uzlib") +libenv.ApplyLibFlags() + +libenv.AppendUnique( + CCFLAGS=[ + "-Wno-redundant-decls", + "-Wno-sign-compare", + ], +) + +sources = [ + File("uzlib/src/adler32.c"), + File("uzlib/src/crc32.c"), + File("uzlib/src/tinfgzip.c"), + File("uzlib/src/tinflate.c"), +] + +lib = libenv.StaticLibrary("${FW_LIB_NAME}", sources) +libenv.Install("${LIB_DIST_DIR}", lib) +Return("lib") diff --git a/scripts/update.py b/scripts/update.py index 0cdf3459d..fd933126f 100755 --- a/scripts/update.py +++ b/scripts/update.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import math +import gzip import os import shutil import tarfile @@ -158,6 +159,14 @@ class Main(App): self.args.resources, join(self.args.directory, resources_basename) ): return 3 + resources_path = join(self.args.directory, resources_basename) + with open(resources_path, "rb") as f_raw: + resources_raw = f_raw.read() + os.unlink(resources_path) + resources_basename += ".gz" + resources_path += ".gz" + with gzip.open(resources_path, "wb", compresslevel=9) as f_zip: + f_zip.write(resources_raw) if not self.layout_check(dfu_size, radio_addr): self.logger.warn("Memory layout looks suspicious") diff --git a/targets/f7/api_symbols.csv b/targets/f7/api_symbols.csv index da2a8fc0a..313bf34bf 100644 --- a/targets/f7/api_symbols.csv +++ b/targets/f7/api_symbols.csv @@ -3564,6 +3564,7 @@ Function,+,tar_archive_free,void,TarArchive* Function,+,tar_archive_get_entries_count,int32_t,TarArchive* Function,+,tar_archive_open,_Bool,"TarArchive*, const char*, TarOpenMode" Function,+,tar_archive_set_file_callback,void,"TarArchive*, tar_unpack_file_cb, void*" +Function,+,tar_archive_set_read_callback,void,"TarArchive*, tar_unpack_read_cb, void*" Function,+,tar_archive_store_data,_Bool,"TarArchive*, const char*, const uint8_t*, const int32_t" Function,+,tar_archive_unpack_file,_Bool,"TarArchive*, const char*, const char*" Function,+,tar_archive_unpack_to,_Bool,"TarArchive*, const char*, Storage_name_converter" diff --git a/targets/f7/target.json b/targets/f7/target.json index 4c79b2e66..70d8587c0 100644 --- a/targets/f7/target.json +++ b/targets/f7/target.json @@ -32,6 +32,7 @@ "toolbox", "nfc", "digital_signal", + "uzlib", "pulse_reader", "signal_reader", "microtar",