FBT/SDK: New app flag UnloadAssetPacks to free RAM (#260)

* Store app flags in FAP header

* Add app flag to UnloadAssetPacks to free RAM

* Unload asset packs in NFC and MFKey

* Clearer size units

* Update changelog

* Future proof logic

* Sync apps

.github/workflows/build.yml

@@ -128,8 +128,8 @@ jobs:
           dfu_size_new=$(du --apparent-size -B 1 artifacts/flipper-z-${TARGET}-full-*.dfu | cut -f1)
           dfu_size_dev=$(du --apparent-size -B 1 dev.dfu | cut -f1)
           dfu_size_diff=$((dfu_size_new - dfu_size_dev))
-          DFU_SIZE=$(echo $dfu_size_new | numfmt --to=iec --format=%.2f)
-          DFU_DIFF=$(echo $dfu_size_diff | numfmt --to=iec --format=%.2f | sed -r 's/^([^-])/+\1/')
+          DFU_SIZE=$(echo $dfu_size_new | numfmt --to=iec-i --format=%.2fB | sed 's/.00B$/B/')
+          DFU_DIFF=$(echo $dfu_size_diff | numfmt --to=iec-i --format=%.2fB | sed 's/.00B$/B/' | sed -r 's/^([^-])/+\1/')
           echo "DFU_SIZE=$DFU_SIZE" >> $GITHUB_ENV
           echo "DFU_DIFF=$DFU_DIFF" >> $GITHUB_ENV
 
@@ -139,8 +139,8 @@ jobs:
           min_gap=$((2 * 4 * 1024))
           flash_free_total=$((radio_addr - flash_base - dfu_size_new))
           flash_free_usable=$((flash_free_total - min_gap))
-          FLASH_FREE=$(echo $flash_free_total | numfmt --to=iec --format=%.2f)
-          FLASH_USABLE=$(echo $flash_free_usable | numfmt --to=iec --format=%.2f)
+          FLASH_FREE=$(echo $flash_free_total | numfmt --to=iec-i --format=%.2fB | sed 's/.00B$/B/')
+          FLASH_USABLE=$(echo $flash_free_usable | numfmt --to=iec-i --format=%.2fB | sed 's/.00B$/B/')
           echo "FLASH_FREE=$FLASH_FREE" >> $GITHUB_ENV
           echo "FLASH_USABLE=$FLASH_USABLE" >> $GITHUB_ENV
 

CHANGELOG.md

@@ -102,6 +102,7 @@
   - Added `illegalSymbols` prop for `gui/text_input` view (#290 by @Willy-JL)
   - Added typedocs for all extra JS modules in Momentum (by @Willy-JL)
 - RPC: Added ASCII event support (#284 by @Willy-JL)
+- FBT/SDK: New app flag UnloadAssetPacks to free RAM in heavy apps like NFC, MFKey, uPython (#260 by @Willy-JL)
 - OFW: Settings: Clock editing & Alarm function (目覚め時計) (by @skotopes)
 - BadKB:
   - OFW: Add linux/gnome badusb demo files (by @thomasnemer)

applications/main/nfc/application.fam

@@ -16,6 +16,7 @@ App(
     fap_libs=["assets", "mbedtls"],
     fap_icon="icon.png",
     fap_category="NFC",
+    flags=["UnloadAssetPacks"],
 )
 
 # Parser plugins

applications/services/applications.h

@@ -3,9 +3,11 @@
 #include <furi.h>
 #include <gui/icon.h>
 
-typedef enum {
+typedef enum FURI_PACKED {
     FlipperApplicationFlagDefault = 0,
     FlipperApplicationFlagInsomniaSafe = (1 << 0),
+
+    FlipperApplicationFlagUnloadAssetPacks = (1 << 7),
 } FlipperApplicationFlag;
 
 typedef struct {
@@ -21,7 +23,6 @@ typedef struct {
     const char* name;
     const Icon* icon;
     const char* path;
-    const FlipperApplicationFlag flags;
 } FlipperExternalApplication;
 
 typedef void (*FlipperInternalOnStartHook)(void);

applications/services/loader/loader.c

@@ -10,24 +10,23 @@
 #include <flipper_application/flipper_application.h>
 #include <loader/firmware_api/firmware_api.h>
 
+#include <momentum/asset_packs.h>
+
 #define TAG "Loader"
 
 #define LOADER_MAGIC_THREAD_VALUE 0xDEADBEEF
 
 // helpers
 
-static const char*
-    loader_find_external_application_by_name(const char* app_name, FlipperApplicationFlag* flags) {
+static const char* loader_find_external_application_by_name(const char* app_name) {
     for(size_t i = 0; i < FLIPPER_EXTERNAL_APPS_COUNT; i++) {
         if(strcmp(FLIPPER_EXTERNAL_APPS[i].name, app_name) == 0) {
-            if(flags) *flags = FLIPPER_EXTERNAL_APPS[i].flags;
             return FLIPPER_EXTERNAL_APPS[i].path;
         }
     }
 
     for(size_t i = 0; i < FLIPPER_SETTINGS_APPS_COUNT; i++) {
         if(strcmp(FLIPPER_SETTINGS_APPS[i].name, app_name) == 0) {
-            if(flags) *flags = FLIPPER_SETTINGS_APPS[i].flags;
             return FLIPPER_SETTINGS_APPS[i].path;
         }
     }
@@ -101,7 +100,7 @@ static void loader_show_gui_error(
     DialogMessage* message = dialog_message_alloc();
 
     if(status.value == LoaderStatusErrorUnknownApp &&
-       loader_find_external_application_by_name(name, NULL) != NULL) {
+       loader_find_external_application_by_name(name) != NULL) {
         // Special case for external apps
         const char* header = NULL;
         const char* text = NULL;
@@ -421,6 +420,12 @@ static void loader_start_internal_app(
     LoaderEvent event;
     event.type = LoaderEventTypeApplicationBeforeLoad;
     furi_pubsub_publish(loader->pubsub, &event);
+    if(app->flags & FlipperApplicationFlagUnloadAssetPacks) {
+        loader->app.unloaded_asset_packs = true;
+        asset_packs_free();
+    } else {
+        loader->app.unloaded_asset_packs = false;
+    }
 
     // store args
     furi_assert(loader->app.args == NULL);
@@ -503,8 +508,7 @@ static LoaderMessageLoaderStatusResult loader_start_external_app(
     Storage* storage,
     const char* path,
     const char* args,
-    FuriString* error_message,
-    FlipperApplicationFlag flags) {
+    FuriString* error_message) {
     LoaderMessageLoaderStatusResult result;
     result.value = loader_make_success_status(error_message);
     result.error = LoaderStatusErrorUnknown;
@@ -519,8 +523,22 @@ static LoaderMessageLoaderStatusResult loader_start_external_app(
 
         FURI_LOG_I(TAG, "Loading %s", path);
 
+        // Calling preload will load whole FAP file, so need to preload manifest first to
+        // get flags value, unload asset packs if requested by flags, then preload whole FAP
+        FlipperApplicationFlag flags = FlipperApplicationFlagDefault;
         FlipperApplicationPreloadStatus preload_res =
-            flipper_application_preload(loader->app.fap, path);
+            flipper_application_preload_manifest(loader->app.fap, path);
+        loader->app.unloaded_asset_packs = false;
+        if(preload_res != FlipperApplicationPreloadStatusInvalidFile &&
+           preload_res != FlipperApplicationPreloadStatusInvalidManifest) {
+            flags = flipper_application_get_manifest(loader->app.fap)->flags;
+            if(flags & FlipperApplicationFlagUnloadAssetPacks) {
+                loader->app.unloaded_asset_packs = true;
+                asset_packs_free();
+            }
+            preload_res = flipper_application_preload(loader->app.fap, path);
+        }
+
         bool api_mismatch = false;
         if(preload_res == FlipperApplicationPreloadStatusApiTooOld ||
            preload_res == FlipperApplicationPreloadStatusApiTooNew) {
@@ -612,6 +630,9 @@ static LoaderMessageLoaderStatusResult loader_start_external_app(
         loader->app.fap = NULL;
         event.type = LoaderEventTypeApplicationLoadFailed;
         furi_pubsub_publish(loader->pubsub, &event);
+        if(loader->app.unloaded_asset_packs) {
+            asset_packs_init();
+        }
     }
 
     return result;
@@ -702,9 +723,8 @@ static LoaderMessageLoaderStatusResult loader_do_start_by_name(
         }
 
         // check External Applications
-        FlipperApplicationFlag flags = FlipperApplicationFlagDefault;
         {
-            const char* path = loader_find_external_application_by_name(name, &flags);
+            const char* path = loader_find_external_application_by_name(name);
             if(path) {
                 name = path;
             }
@@ -714,8 +734,7 @@ static LoaderMessageLoaderStatusResult loader_do_start_by_name(
         {
             Storage* storage = furi_record_open(RECORD_STORAGE);
             if(storage_file_exists(storage, name)) {
-                status =
-                    loader_start_external_app(loader, storage, name, args, error_message, flags);
+                status = loader_start_external_app(loader, storage, name, args, error_message);
                 furi_record_close(RECORD_STORAGE);
                 break;
             }
@@ -772,6 +791,9 @@ static void loader_do_app_closed(Loader* loader) {
     LoaderEvent event;
     event.type = LoaderEventTypeApplicationStopped;
     furi_pubsub_publish(loader->pubsub, &event);
+    if(loader->app.unloaded_asset_packs) {
+        asset_packs_init();
+    }
 }
 
 static bool loader_is_application_running(Loader* loader) {

applications/services/loader/loader_i.h

@@ -11,6 +11,8 @@ typedef struct {
     FuriThread* thread;
     bool insomniac;
     FlipperApplication* fap;
+
+    bool unloaded_asset_packs;
 } LoaderAppData;
 
 struct Loader {

lib/flipper_application/application_manifest.h

@@ -8,6 +8,8 @@
 #include <stdbool.h>
 #include "elf/elf_api_interface.h"
 
+#include <applications.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -42,7 +44,22 @@ typedef struct {
     char icon[FAP_MANIFEST_MAX_ICON_SIZE];
 } FlipperApplicationManifestV1;
 
-typedef FlipperApplicationManifestV1 FlipperApplicationManifest;
+typedef FlipperApplicationManifestV1 FlipperApplicationManifestOfw;
+
+typedef struct {
+    FlipperApplicationManifestBase base;
+    uint16_t stack_size;
+    uint32_t app_version;
+    char name[FAP_MANIFEST_MAX_APP_NAME_LENGTH];
+    char has_icon;
+    char icon[FAP_MANIFEST_MAX_ICON_SIZE];
+
+    FlipperApplicationFlag flags;
+} FlipperApplicationManifestV1Ex;
+
+typedef FlipperApplicationManifestV1Ex FlipperApplicationManifestEx;
+
+typedef FlipperApplicationManifestEx FlipperApplicationManifest;
 
 #pragma pack(pop)
 

lib/flipper_application/flipper_application.c

@@ -15,6 +15,8 @@ struct FlipperApplication {
     ELFFile* elf;
     FuriThread* thread;
     void* ep_thread_args;
+
+    bool preloaded_manifest;
 };
 
 /********************** Debugger access to loader state **********************/
@@ -127,16 +129,25 @@ static bool flipper_application_process_manifest_section(
     void* context) {
     FlipperApplicationManifest* manifest = context;
 
-    if(size < sizeof(FlipperApplicationManifest)) {
+    // Support both OFW manifest and extended manifest with flags
+    if(size < sizeof(FlipperApplicationManifestOfw) ||
+       size > sizeof(FlipperApplicationManifestEx)) {
         return false;
     }
 
     if(manifest == NULL) {
         return true;
     }
 
-    return storage_file_seek(file, offset, true) &&
-           storage_file_read(file, manifest, size) == size;
+    bool result = storage_file_seek(file, offset, true) &&
+                  storage_file_read(file, manifest, size) == size;
+
+    // Default flags when loading OFW manifests that don't include flags
+    if(result && size < sizeof(FlipperApplicationManifestEx)) {
+        manifest->flags = FlipperApplicationFlagDefault;
+    }
+
+    return result;
 }
 
 // we can't use const char* as context because we will lose the const qualifier
@@ -155,7 +166,7 @@ static bool flipper_application_process_assets_section(
 
 static FlipperApplicationPreloadStatus
     flipper_application_load(FlipperApplication* app, const char* path, bool load_full) {
-    if(!elf_file_open(app->elf, path)) {
+    if(!app->preloaded_manifest && !elf_file_open(app->elf, path)) {
         return FlipperApplicationPreloadStatusInvalidFile;
     }
 
@@ -181,12 +192,18 @@ static FlipperApplicationPreloadStatus
     }
 
     // load manifest section
-    if(elf_process_section(
+    if(!app->preloaded_manifest &&
+       elf_process_section(
            app->elf, ".fapmeta", flipper_application_process_manifest_section, &app->manifest) !=
-       ElfProcessSectionResultSuccess) {
+           ElfProcessSectionResultSuccess) {
         return FlipperApplicationPreloadStatusInvalidFile;
     }
 
+    // Avoid preloading manifest twice, when user calls both preload_manifest() and preload()
+    if(!load_full) {
+        app->preloaded_manifest = true;
+    }
+
     return flipper_application_validate_manifest(app);
 }
 

scripts/fbt/elfmanifest.py

@@ -1,5 +1,6 @@
 import os
 import struct
+from enum import IntFlag
 from dataclasses import dataclass, field
 
 from flipper.assets.icon import file2image
@@ -9,6 +10,13 @@
 _MANIFEST_MAGIC = 0x52474448
 
 
+class ElfManifestFlag(IntFlag):
+    Default = 0
+    InsomniaSafe = 1 << 0
+
+    UnloadAssetPacks = 1 << 7
+
+
 @dataclass
 class ElfManifestBaseHeader:
     manifest_version: int
@@ -45,6 +53,26 @@ def as_bytes(self):
         )
 
 
+@dataclass
+class ElfManifestV1Ext:
+    stack_size: int
+    app_version: int
+    name: str = ""
+    icon: bytes = field(default=b"")
+    flags: int = ElfManifestFlag.Default
+
+    def as_bytes(self):
+        return struct.pack(
+            "<hI32s?32sB",
+            self.stack_size,
+            self.app_version,
+            bytes(self.name.encode("ascii")),
+            bool(self.icon),
+            self.icon,
+            self.flags,
+        )
+
+
 def assemble_manifest_data(
     app_manifest: FlipperApplication,
     hardware_target: int,
@@ -67,16 +95,21 @@ def assemble_manifest_data(
         app_manifest.fap_version[1] & 0xFFFF
     )
 
+    flags_as_int = ElfManifestFlag.Default
+    for flag in app_manifest.flags:
+        flags_as_int |= ElfManifestFlag[flag]
+
     data = ElfManifestBaseHeader(
         manifest_version=1,
         api_version=sdk_version,
         hardware_target_id=hardware_target,
     ).as_bytes()
-    data += ElfManifestV1(
+    data += ElfManifestV1Ext(
         stack_size=app_manifest.stack_size,
         app_version=app_version_as_int,
         name=app_manifest.name,
         icon=image_data,
+        flags=flags_as_int,
     ).as_bytes()
 
     return data

scripts/fbt_tools/fbt_apps.py

@@ -69,8 +69,7 @@ def get_external_app_descr(self, app: FlipperApplication):
     {{
      .name = "{app.name}",
      .icon = {f"&{app.icon}" if app.icon else "NULL"},
-     .path = "{app_path}",
-     .flags = {'|'.join(f"FlipperApplicationFlag{flag}" for flag in app.flags)} }}"""
+     .path = "{app_path}" }}"""
 
     def generate(self):
         contents = [