FBT: Optimize icons blob -4KB DFU, scrub unused icons

.github/workflows/build.yml

@@ -128,11 +128,22 @@ jobs:
           dfu_size_new=$(du --apparent-size -B 1 artifacts/flipper-z-${TARGET}-full-*.dfu | cut -f1)
           dfu_size_dev=$(du --apparent-size -B 1 dev.dfu | cut -f1)
           dfu_size_diff=$((dfu_size_new - dfu_size_dev))
-          DFU_SIZE=$(echo ${dfu_size_new}B | sed -r 's/^([0-9]+)([0-9]{2})([0-9])B/\1.\2K/')
-          DFU_DIFF=$(echo ${dfu_size_diff}B | sed -r 's/^(-?[0-9]+)([0-9]{2})([0-9])B/\1.\2K/' | sed -r 's/^([^-])/+\1/')
+          DFU_SIZE=$(echo $dfu_size_new | numfmt --to=iec --format=%.2f)
+          DFU_DIFF=$(echo $dfu_size_diff | numfmt --to=iec --format=%.2f | sed -r 's/^([^-])/+\1/')
           echo "DFU_SIZE=$DFU_SIZE" >> $GITHUB_ENV
           echo "DFU_DIFF=$DFU_DIFF" >> $GITHUB_ENV
 
+          tar xzf build/core2_firmware.tgz
+          radio_addr=$(jq -r '.copro.radio.files[0].address' core2_firmware/Manifest.json)
+          flash_base=0x8000000
+          min_gap=$((2 * 4 * 1024))
+          flash_free_total=$((radio_addr - flash_base - dfu_size_new))
+          flash_free_usable=$((flash_free_total - min_gap))
+          FLASH_FREE=$(echo $flash_free_total | numfmt --to=iec --format=%.2f)
+          FLASH_USABLE=$(echo $flash_free_usable | numfmt --to=iec --format=%.2f)
+          echo "FLASH_FREE=$FLASH_FREE" >> $GITHUB_ENV
+          echo "FLASH_USABLE=$FLASH_USABLE" >> $GITHUB_ENV
+
       - name: "Upload artifacts to update server"
         env:
           INDEXER_URL: ${{ secrets.INDEXER_URL }}
@@ -170,6 +181,7 @@ jobs:
             - [☁️ Flipper Lab/App](https://lab.flipper.net/?url=${{secrets.INDEXER_URL}}/builds/firmware/${{steps.names.outputs.branch_name}}/flipper-z-${{steps.names.outputs.default_target}}-update-${{steps.names.outputs.suffix}}.tgz&channel=mntm-${{steps.names.outputs.branch_name}}&version=${{steps.names.outputs.commit_sha}})
             - [📦 qFlipper Package](${{secrets.INDEXER_URL}}/builds/firmware/${{steps.names.outputs.branch_name}}/flipper-z-${{steps.names.outputs.default_target}}-update-${{steps.names.outputs.suffix}}.tgz)
             - DFU Size: `${{ env.DFU_SIZE }}` (`${{ env.DFU_DIFF }}` from dev)
+            - Free Flash: `${{ env.FLASH_FREE }}` (`${{ env.FLASH_USABLE }}` usable)
           edit-mode: replace
 
       - name: Send devbuild webhook

CHANGELOG.md

@@ -133,9 +133,6 @@
   - UL: W5500 Ethernet: Various fixes and improvements (by @xMasterX)
   - OFW: SPI Mem Manager: Fixed UI rendering bug related to line breaks (by @portasynthinca3)
   - OFW: USB/BT Remote: Mouse clicker option to click as fast as possible (by @sumukhj1219)
-- CLI:
-  - Print plugin name on load fail (by @Willy-JL)
-  - Move more commands as plugins on SD, refactor plugin wrapper (#276 by @Willy-JL)
 - NFC:
   - NDEF Parser:
     - Mifare Classic support (#265 by @luu176), protocol-agnostic rewrite and more improvements (#265 by @Willy-JL)
@@ -166,6 +163,10 @@
   - OFW: Add TCL 75S451 to TV universal remote (by @christhetech131)
   - OFW: Universal remote additions (by @jaylikesbunda)
   - OFW: Heavily Expand Universal Remotes (by @jaylikesbunda)
+- CLI:
+  - Print plugin name on load fail (by @Willy-JL)
+  - Move more commands as plugins on SD, refactor plugin wrapper (#276 by @Willy-JL)
+- FBT: Optimize icons blob, scrub unused icons (#291 by @Willy-JL)
 - OFW: BadKB: Improve ChromeOS and GNOME demo scripts (by @kowalski7cc)
 - OFW: GUI: Change dialog_ex text ownership model (by @skotopes)
 - OFW: CCID: App changes and improvements (by @kidbomb)

applications/main/nfc/application.fam

@@ -13,7 +13,7 @@ App(
         "!plugins",
         "!nfc_cli.c",
     ],
-    fap_libs=["mbedtls"],
+    fap_libs=["assets", "mbedtls"],
     fap_icon="icon.png",
     fap_category="NFC",
 )

applications/settings/about/application.fam

@@ -12,4 +12,5 @@ App(
     stack_size=1 * 1024,
     order=1000,
     fap_category="assets",
+    fap_libs=["assets"],
 )

applications/settings/power_settings_app/application.fam

@@ -12,4 +12,5 @@ App(
     stack_size=1 * 1024,
     order=40,
     fap_category="assets",
+    fap_libs=["assets"],
 )

applications/settings/storage_settings/application.fam

@@ -8,4 +8,5 @@ App(
     stack_size=2 * 1024,
     order=30,
     fap_category="assets",
+    fap_libs=["assets"],
 )

applications/system/js_app/application.fam

@@ -5,6 +5,7 @@ App(
     cdefines=["JS_RUNNER_FAP"],
     fap_icon="icon.png",
     fap_category="assets",
+    fap_libs=["assets"],
     entry_point="js_app",
     stack_size=2 * 1024,
     resources="examples",
@@ -35,6 +36,7 @@ App(
         "modules/js_flipper.c",
         "modules/js_tests.c",
     ],
+    fap_libs=["assets"],
 )
 
 App(

assets/SConscript

@@ -14,9 +14,26 @@ icons_path = (
     if assetsenv["IS_BASE_FIRMWARE"]
     else assetsenv["ASSETS_WORK_DIR"]
 )
-icons = assetsenv.CompileIcons(icons_path, assetsenv["ASSETS_SRC_DIR"].Dir("icons"))
+icons = assetsenv.CompileIcons(
+    icons_path,
+    assetsenv["ASSETS_SRC_DIR"].Dir("icons"),
+    fw_bundle=True,
+)
 assetsenv.Alias("icons", icons)
 
+# Extra icons lib so apps don't override asset packs to use fw icons not exposed to api
+# Not used anywhere in code, apps still reference assets_icons.h, but if one of the
+# icons they need is not exposed in api, they can link with fap_libs=["assets"] which
+# will use this lib purely for linking as a drop-in for missing symbols.
+# Should mean that apps made for OFW with fap_libs=["assets"], when compiled with
+# this FBT, will still use asset packs at runtime and only link non-exposed icons.
+if assetsenv["IS_BASE_FIRMWARE"]:
+    dropin_icons = assetsenv.CompileIcons(
+        assetsenv["ASSETS_WORK_DIR"],
+        assetsenv["ASSETS_SRC_DIR"].Dir("icons"),
+        icon_bundle_name="_assets_icons_dropin",
+    )
+
 
 # Protobuf .proto -> .c + .h
 proto_src = assetsenv.Glob("protobuf/*.proto", source=True)
@@ -59,9 +76,14 @@ assetsenv.Alias("proto_ver", proto_ver)
 assets_parts = (icons, proto, dolphin_blocking, dolphin_internal, proto_ver)
 env.Replace(FW_ASSETS_HEADERS=assets_parts)
 
-assetslib = assetsenv.Library("${FW_LIB_NAME}", assets_parts)
+assetslib = assetsenv.Library("fw${FW_LIB_NAME}", assets_parts)
 assetsenv.Install("${LIB_DIST_DIR}", assetslib)
 
+if assetsenv["IS_BASE_FIRMWARE"]:
+    dropin_parts = (dropin_icons, proto, dolphin_blocking, dolphin_internal, proto_ver)
+    dropin_assetslib = assetsenv.Library("${FW_LIB_NAME}", dropin_parts)
+    assetsenv.Install("${LIB_DIST_DIR}", dropin_assetslib)
+
 
 # Resources for SD card
 if assetsenv["IS_BASE_FIRMWARE"]:

scripts/assets.py

@@ -11,11 +11,20 @@
 
 ICONS_TEMPLATE_H_HEADER = """#pragma once
 
-#include <furi.h>
+#include <stddef.h>
 #include <gui/icon.h>
 
 """
 ICONS_TEMPLATE_H_ICON_NAME = "extern const Icon {name};\n"
+ICONS_TEMPLATE_H_ICON_PATHS = """
+typedef struct {
+    const Icon* icon;
+    const char* path;
+} IconPath;
+
+extern const IconPath ICON_PATHS[];
+extern const size_t ICON_PATHS_COUNT;
+"""
 
 ICONS_TEMPLATE_C_HEADER = """#include "{assets_filename}.h"
 
@@ -25,6 +34,15 @@
 ICONS_TEMPLATE_C_FRAME = "const uint8_t {name}[] = {data};\n"
 ICONS_TEMPLATE_C_DATA = "const uint8_t* const {name}[] = {data};\n"
 ICONS_TEMPLATE_C_ICONS = "const Icon {name} = {{.width={width},.height={height},.frame_count={frame_count},.frame_rate={frame_rate},.frames=_{name}}};\n"
+ICONS_TEMPLATE_C_ICON_PATH = '    {{&{name}, "{path}"}},\n'
+ICONS_TEMPLATE_C_ICON_PATHS = """
+const IconPath ICON_PATHS[] = {{
+#ifndef FURI_RAM_EXEC
+{icon_paths}
+#endif
+}};
+const size_t ICON_PATHS_COUNT = COUNT_OF(ICON_PATHS);
+"""
 
 MAX_IMAGE_WIDTH = 2**16 - 1
 MAX_IMAGE_HEIGHT = 2**16 - 1
@@ -45,6 +63,22 @@ def init(self):
             required=False,
             default="assets_icons",
         )
+        self.parser_icons.add_argument(
+            "--fw-bundle",
+            dest="fw_bundle",
+            help="Bundle all icons and path info, only for use in firmware blob",
+            default=0,
+            type=int,
+            required=False,
+        )
+        self.parser_icons.add_argument(
+            "--add-include",
+            dest="add_include",
+            help="Add assets_icons.h include drop-in for apps",
+            default=0,
+            type=int,
+            required=False,
+        )
 
         self.parser_icons.set_defaults(func=self.icons)
 
@@ -138,7 +172,6 @@ def icons(self):
         )
         icons = []
         paths = []
-        is_main_assets = self.args.filename == "assets_icons"
         symbols = pathlib.Path(__file__).parent.parent
         if "UFBT_HOME" in os.environ:
             symbols /= "sdk_headers/f7_sdk"
@@ -154,7 +187,8 @@ def icons(self):
             if "frame_rate" in filenames:
                 self.logger.debug("Folder contains animation")
                 icon_name = "A_" + os.path.split(dirpath)[1].replace("-", "_")
-                if not is_main_assets and api_has_icon(icon_name):
+                icon_in_api = api_has_icon(icon_name)
+                if not self.args.fw_bundle and icon_in_api:
                     self.logger.info(
                         f"{self.args.filename}: ignoring duplicate icon {icon_name}"
                     )
@@ -193,8 +227,9 @@ def icons(self):
                 )
                 icons_c.write("\n")
                 icons.append((icon_name, width, height, frame_rate, frame_count))
-                p = dirpath.removeprefix(self.args.input_directory)[1:]
-                paths.append((icon_name, p.replace("\\", "/")))
+                if self.args.fw_bundle and icon_in_api:
+                    path = dirpath.removeprefix(self.args.input_directory)[1:]
+                    paths.append((icon_name, path.replace("\\", "/")))
             else:
                 # process icons
                 for filename in filenames:
@@ -204,7 +239,8 @@ def icons(self):
                     icon_name = "I_" + "_".join(filename.split(".")[:-1]).replace(
                         "-", "_"
                     )
-                    if not is_main_assets and api_has_icon(icon_name):
+                    icon_in_api = api_has_icon(icon_name)
+                    if not self.args.fw_bundle and icon_in_api:
                         self.logger.info(
                             f"{self.args.filename}: ignoring duplicate icon {icon_name}"
                         )
@@ -222,8 +258,11 @@ def icons(self):
                     )
                     icons_c.write("\n")
                     icons.append((icon_name, width, height, 0, 1))
-                    p = fullfilename.removeprefix(self.args.input_directory)[1:]
-                    paths.append((icon_name, p.replace("\\", "/").rsplit(".", 1)[0]))
+                    if self.args.fw_bundle and icon_in_api:
+                        path = fullfilename.removeprefix(self.args.input_directory)[1:]
+                        paths.append(
+                            (icon_name, path.replace("\\", "/").rsplit(".", 1)[0])
+                        )
         # Create array of images:
         self.logger.debug("Finalizing source file")
         for name, width, height, frame_rate, frame_count in icons:
@@ -236,21 +275,14 @@ def icons(self):
                     frame_count=frame_count,
                 )
             )
-        if is_main_assets:
-            icons_c.write(
-                """
-const IconPath ICON_PATHS[] = {
-#ifndef FURI_RAM_EXEC
-"""
-            )
-            for name, path in paths:
-                icons_c.write(f'    {{&{name}, "{path}"}},\n')
-            icons_c.write(
-                """#endif
-};
-const size_t ICON_PATHS_COUNT = COUNT_OF(ICON_PATHS);
-"""
+        if not self.args.fw_bundle:
+            icons_c.write("\n")
+        else:
+            icon_paths = "\n".join(
+                ICONS_TEMPLATE_C_ICON_PATH.format(name=name, path=path)
+                for name, path in paths
             )
+            icons_c.write(ICONS_TEMPLATE_C_ICON_PATHS.format(icon_paths=icon_paths))
         icons_c.close()
 
         # Create Public Header
@@ -263,19 +295,9 @@ def icons(self):
         icons_h.write(ICONS_TEMPLATE_H_HEADER)
         for name, width, height, frame_rate, frame_count in icons:
             icons_h.write(ICONS_TEMPLATE_H_ICON_NAME.format(name=name))
-        if is_main_assets:
-            icons_h.write(
-                """
-typedef struct {
-    const Icon* icon;
-    const char* path;
-} IconPath;
-
-extern const IconPath ICON_PATHS[];
-extern const size_t ICON_PATHS_COUNT;
-"""
-            )
-        else:
+        if self.args.fw_bundle:
+            icons_h.write(ICONS_TEMPLATE_H_ICON_PATHS)
+        if self.args.add_include:
             icons_h.write("#include <assets_icons.h>\n")
         icons_h.close()
         self.logger.debug("Done")

scripts/check_unused_icons.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+import pathlib
+
+root = pathlib.Path(__file__).parent / ".."
+icons = root / "assets/icons"
+
+
+def count_icon_usages(name: str):
+    count = 0
+    name = name.encode()
+    # EXTREMELY wasteful, but who cares
+    for dir in ("applications", "furi", "lib", "targets"):
+        for filetype in (".c", ".cpp", ".h", ".fam"):
+            for file in (root / dir).glob(f"**/*{filetype}"):
+                try:
+                    if name in file.read_bytes():
+                        count += 1
+                except Exception:
+                    print(f"Failed to read {file}")
+    return count
+
+
+if __name__ == "__main__":
+    counts = {}
+
+    for category in icons.iterdir():
+        if not category.is_dir():
+            continue
+        for icon in category.iterdir():
+            if icon.is_dir() and (icon / "frame_rate").is_file():
+                name = "A_" + icon.name.replace("-", "_")
+            elif icon.is_file() and icon.suffix == ".png":
+                name = "I_" + "_".join(icon.name.split(".")[:-1]).replace("-", "_")
+            else:
+                continue
+            counts[name[2:]] = count_icon_usages(name)
+
+    for name, count in sorted(counts.items(), key=lambda x: x[1], reverse=True):
+        print(f"{name} used {count} times")

scripts/fbt_tools/fbt_assets.py

@@ -181,7 +181,15 @@ def describe():
         file.write("\n".join(version_file_data))
 
 
-def CompileIcons(env, target_dir, source_dir, *, icon_bundle_name="assets_icons"):
+def CompileIcons(
+    env,
+    target_dir,
+    source_dir,
+    *,
+    icon_bundle_name="assets_icons",
+    fw_bundle=False,
+    add_include=False,
+):
     try:
         os.mkdir(str(source_dir))
     except FileExistsError:
@@ -191,6 +199,8 @@ def CompileIcons(env, target_dir, source_dir, *, icon_bundle_name="assets_icons"
         None,
         ICON_SRC_DIR=source_dir,
         ICON_FILE_NAME=icon_bundle_name,
+        ICON_FW_BUNDLE=int(fw_bundle),
+        ICON_ADD_INCLUDE=int(add_include),
     )
 
 
@@ -223,6 +233,10 @@ def generate(env):
                             "${TARGET.dir}",
                             "--filename",
                             "${ICON_FILE_NAME}",
+                            "--fw-bundle",
+                            "${ICON_FW_BUNDLE}",
+                            "--add-include",
+                            "${ICON_ADD_INCLUDE}",
                         ],
                     ],
                     "${ICONSCOMSTR}",

scripts/fbt_tools/fbt_extapps.py

@@ -90,6 +90,7 @@ def _compile_assets(self):
             self.app_work_dir,
             self.app._appdir.Dir(self.app.fap_icon_assets),
             icon_bundle_name=f"{self.app.fap_icon_assets_symbol or self.app.appid }_icons",
+            add_include=True,
         )
         self.app_env.Alias("_fap_icons", fap_icons)
         self.fw_env.Append(_APP_ICONS=[fap_icons])