Prune mobile benchmark suites (iree-org#14923)

Remove the convolution models from mobile benchmarks to save limited mobile device resources for new focus targets The longest Pixel 6 Pro benchmark time is reduced from 1h to 22mins. We still track the convolution models on RISC-V and other platforms
nod-ai · Oct 11, 2023 · 02e1625 · 02e1625
1 parent dec2949
commit 02e1625
Show file tree

Hide file tree

Showing 4 changed files with 991 additions and 2,472 deletions.
diff --git a/build_tools/python/benchmark_suites/iree/adreno_benchmarks.py b/build_tools/python/benchmark_suites/iree/adreno_benchmarks.py
@@ -45,12 +45,7 @@ def generate(
         self,
     ) -> List[iree_definitions.E2EModelRunConfig]:
         default_models = [
-            tflite_models.DEEPLABV3_FP32,
-            tflite_models.MOBILESSD_FP32,
-            tflite_models.POSENET_FP32,
             tflite_models.MOBILEBERT_FP32,
-            tflite_models.MOBILENET_V2,
-            tflite_models.MOBILENET_V3SMALL,
         ]
         default_gen_configs = [
             iree_definitions.ModuleGenerationConfig.build(
@@ -66,18 +61,6 @@ def generate(
             )
             for model in default_models
         ]
-        fuse_padding_repeated_kernel_gen_configs = [
-            iree_definitions.ModuleGenerationConfig.build(
-                compile_config=self.FUSE_PADDING_REPEATED_KERNEL_COMPILE_CONFIG,
-                imported_model=iree_definitions.ImportedModel.from_model(model),
-            )
-            for model in [
-                tflite_models.MOBILESSD_FP32,
-                tflite_models.POSENET_FP32,
-                tflite_models.MOBILENET_V2,
-                tflite_models.MOBILENET_V3SMALL,
-            ]
-        ]
 
         adreno_devices = (
             device_collections.DEFAULT_DEVICE_COLLECTION.query_device_specs(
@@ -97,13 +80,5 @@ def generate(
             device_specs=adreno_devices,
             presets=[benchmark_presets.ANDROID_GPU],
         )
-        run_configs += utils.generate_e2e_model_run_configs(
-            module_generation_configs=fuse_padding_repeated_kernel_gen_configs,
-            module_execution_configs=[
-                module_execution_configs.VULKAN_BATCH_SIZE_16_CONFIG
-            ],
-            device_specs=adreno_devices,
-            presets=[benchmark_presets.ANDROID_GPU],
-        )
 
         return run_configs
diff --git a/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py b/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py
@@ -19,11 +19,7 @@ class Android_ARMv8_A_Benchmarks(object):
 
     NONQUANT_MODELS = [
         tflite_models.DEEPLABV3_FP32,
-        tflite_models.MOBILESSD_FP32,
-        tflite_models.POSENET_FP32,
         tflite_models.MOBILEBERT_FP32,
-        tflite_models.MOBILENET_V2,
-        tflite_models.MOBILENET_V3SMALL,
     ]
     QUANT_MODELS = [tflite_models.MOBILEBERT_INT8]
 

diff --git a/build_tools/python/benchmark_suites/iree/mali_benchmarks.py b/build_tools/python/benchmark_suites/iree/mali_benchmarks.py
@@ -59,18 +59,11 @@ class Android_Mali_Benchmarks(object):
     EXPERIMENTAL_REPEATED_KERNEL_RUN_FLAGS = ["--batch_size=32"]
 
     FP32_MODELS = [
-        tflite_models.DEEPLABV3_FP32,
-        tflite_models.MOBILESSD_FP32,
-        tflite_models.POSENET_FP32,
         tflite_models.MOBILEBERT_FP32,
-        tflite_models.MOBILENET_V2,
-        tflite_models.MOBILENET_V3SMALL,
     ]
     FP16_MODELS = [tflite_models.MOBILEBERT_FP16]
     QUANT_MODELS = [
         tflite_models.MOBILEBERT_INT8,
-        tflite_models.EFFICIENTNET_INT8,
-        tflite_models.PERSON_DETECT_INT8,
     ]
 
     def generate(