[infra] Run parameterized ONNX model tests across CPU, Vulkan, and HIP.

iree-org · Dec 18, 2024 · fea2a82 · fea2a82
1 parent e553425
commit fea2a82
Show file tree

Hide file tree

Showing 4 changed files with 103 additions and 5 deletions.
diff --git a/.github/workflows/pkgci_test_onnx.yml b/.github/workflows/pkgci_test_onnx.yml
@@ -130,11 +130,22 @@ jobs:
         include:
           # CPU
           - name: cpu_llvm_task
+            config-file: onnx_models_cpu_llvm_task.json
             runs-on: ubuntu-24.04
 
-          # TODO(scotttodd): test other backends (parameterize the test suite)
+          # AMD GPU
+          - name: amdgpu_rocm_rdna3
+            config-file: onnx_models_gpu_rocm_rdna3.json
+            runs-on: nodai-amdgpu-w7900-x86-64
+          - name: amdgpu_vulkan
+            config-file: onnx_models_gpu_vulkan.json
+            runs-on: nodai-amdgpu-w7900-x86-64
+
+          # NVIDIA GPU
+          # TODO(#18238): migrate to new runner cluster
     env:
       VENV_DIR: ${{ github.workspace }}/venv
+      CONFIG_FILE_PATH: tests/external/iree-test-suites/onnx_models/${{ matrix.config-file }}
     steps:
       - name: Checking out IREE repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -157,8 +168,8 @@ jobs:
       - name: Checkout test suites repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          repository: iree-org/iree-test-suites
-          ref: 8e6af9e75d874ef8c9f8ff55f12cb38157dd55eb
+          repository: ScottTodd/iree-test-suites  # DO NOT SUBMIT - merge https://github.com/iree-org/iree-test-suites/pull/65
+          ref: f0aa54520573bc0627a92676fce329c15966d093
           path: iree-test-suites
       - name: Install ONNX models test suite requirements
         run: |
@@ -170,6 +181,6 @@ jobs:
           pytest iree-test-suites/onnx_models/ \
               -rA \
               --log-cli-level=info \
-              --override-ini=xfail_strict=false \
               --timeout=120 \
-              --durations=0
+              --durations=0 \
+              --test-config-file=${CONFIG_FILE_PATH}
diff --git a/tests/external/iree-test-suites/onnx_models/onnx_models_cpu_llvm_task.json b/tests/external/iree-test-suites/onnx_models/onnx_models_cpu_llvm_task.json
@@ -0,0 +1,35 @@
+{
+  "config_name": "cpu_llvm_task",
+  "iree_compile_flags": [
+    "--iree-hal-target-backends=llvm-cpu",
+    "--iree-llvmcpu-target-cpu=host"
+  ],
+  "iree_run_module_flags": [
+    "--device=local-task"
+  ],
+  "tests_and_expected_outcomes": {
+    "default": "skip",
+    "tests/model_zoo/validated/vision/body_analysis_models_test.py::test_age_gender_age_googlenet": "pass",
+    "tests/model_zoo/validated/vision/body_analysis_models_test.py::test_age_gender_gender_googlenet": "pass",
+    "tests/model_zoo/validated/vision/body_analysis_models_test.py::test_emotion_ferplus": "pass",
+    "tests/model_zoo/validated/vision/body_analysis_models_test.py::test_ultraface": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_alexnet": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_caffenet": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_densenet_121": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_googlenet": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_inception_v2": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_mnist": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_mobilenet": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_resnet50_v1": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_resnet50_v2": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_shufflenet": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_shufflenet_v2": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_squeezenet": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_ssd_mobilenetv1": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_tiny_yolov2": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_tiny_yolov3": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_yolov2_coco": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_yolov3": "pass",
+    "tests/model_zoo/validated/vision/super_resolution_models_test.py::test_fast_neural_style": "pass"
+  }
+}
diff --git a/tests/external/iree-test-suites/onnx_models/onnx_models_gpu_rocm_rdna3.json b/tests/external/iree-test-suites/onnx_models/onnx_models_gpu_rocm_rdna3.json
@@ -0,0 +1,25 @@
+{
+  "config_name": "gpu_rocm_rdna3",
+  "iree_compile_flags": [
+    "--iree-hal-target-backends=rocm",
+    "--iree-hip-target=gfx1100"
+  ],
+  "iree_run_module_flags": [
+    "--device=hip"
+  ],
+  "tests_and_expected_outcomes": {
+    "default": "skip",
+    "tests/model_zoo/validated/vision/body_analysis_models_test.py::test_emotion_ferplus": "pass",
+    "tests/model_zoo/validated/vision/body_analysis_models_test.py::test_ultraface": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_alexnet": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_caffenet": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_mnist": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_shufflenet": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_shufflenet_v2": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_squeezenet": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_ssd_mobilenetv1": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_tiny_yolov3": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_yolov3": "pass",
+    "tests/model_zoo/validated/vision/super_resolution_models_test.py::test_fast_neural_style": "pass"
+  }
+}
diff --git a/tests/external/iree-test-suites/onnx_models/onnx_models_gpu_vulkan.json b/tests/external/iree-test-suites/onnx_models/onnx_models_gpu_vulkan.json
@@ -0,0 +1,27 @@
+{
+  "config_name": "gpu_vulkan",
+  "iree_compile_flags": [
+    "--iree-hal-target-backends=vulkan-spirv"
+  ],
+  "iree_run_module_flags": [
+    "--device=vulkan"
+  ],
+  "tests_and_expected_outcomes": {
+    "default": "skip",
+    "tests/model_zoo/validated/vision/body_analysis_models_test.py::test_emotion_ferplus": "pass",
+    "tests/model_zoo/validated/vision/body_analysis_models_test.py::test_ultraface": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_alexnet": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_caffenet": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_mnist": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_mobilenet": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_resnet50_v1": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_resnet50_v2": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_shufflenet_v2": "pass",
+    "tests/model_zoo/validated/vision/classification_models_test.py::test_squeezenet": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_ssd_mobilenetv1": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_tiny_yolov2": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_tiny_yolov3": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_yolov2_coco": "pass",
+    "tests/model_zoo/validated/vision/object_detection_segmentation_models_test.py::test_yolov3": "pass"
+  }
+}