Rename matmul/ to linalg_ops/, rebase IREE, add test workflow. (#21)

Progress on #2. * Renamed top level `matmul/` folder to `linalg_ops/`. We can put `convolution` (linalg) and `attention` (linalg_ext) tests under here too. Maybe other named ops or generics too. * Rebased on the latest IREE code, including flag renames (though we should drop usage of some flags and prefer to test the main user-facing options) * Added a workflow that goes through the instructions in `linalg_ops/README.md` and configured both workflows to run on `push` events in addition to the current triggers.
iree-org · Sep 4, 2024 · 03f10e9 · 03f10e9
1 parent cc9b026
commit 03f10e9
Show file tree

Hide file tree

Showing 12 changed files with 118 additions and 31 deletions.
diff --git a/.github/workflows/test_linalg_ops.yml b/.github/workflows/test_linalg_ops.yml
@@ -0,0 +1,77 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+name: Test Linalg Ops
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/test_linalg_ops.yml"
+      - "linalg_ops/**"
+  pull_request:
+    paths:
+      - ".github/workflows/test_linalg_ops.yml"
+      - "linalg_ops/**"
+  workflow_dispatch:
+  schedule:
+    # Runs at 3:00 PM UTC, which is 8:00 AM PST
+    - cron: "0 15 * * *"
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  test-linalg-ops:
+    runs-on: ubuntu-24.04
+    env:
+      BUILD_DIR: build
+      VENV_DIR: ${{ github.workspace }}/.venv
+      CC: clang
+      CXX: clang++
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Install Python packages.
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Setup Python venv
+        run: python3 -m venv ${VENV_DIR}
+      - name: Install IREE nightly release Python packages
+        run: |
+          source ${VENV_DIR}/bin/activate
+          python3 -m pip install -r linalg_ops/requirements-iree.txt
+
+      # Build and run tests.
+      - name: Install build dependencies
+        run: sudo apt update && sudo apt install -y ninja-build
+      - name: CMake - Configure
+        working-directory: linalg_ops
+        run: |
+          source ${VENV_DIR}/bin/activate
+          cmake -G Ninja -B ${BUILD_DIR} . \
+            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+            -DIREE_HOST_BIN_DIR=${VENV_DIR}/bin
+      - name: CMake - build
+        working-directory: linalg_ops
+        run: |
+          cmake --build ${BUILD_DIR}
+          cmake --build ${BUILD_DIR} --target iree-test-suites-linalg-ops-deps
+      - name: CTest
+        working-directory: linalg_ops
+        run: |
+          ctest \
+            --test-dir build/ \
+            -R iree-test-suites \
+            --label-exclude "^driver=vulkan$|^driver=metal$|^driver=cuda$|^driver=hip$"
diff --git a/.github/workflows/test_onnx_ops.yml b/.github/workflows/test_onnx_ops.yml
@@ -6,9 +6,14 @@
 
 name: Test ONNX Ops
 on:
+  push:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/test_onnx_ops.yml"
+      - "onnx_ops/**"
   pull_request:
     paths:
-      # This file itself.
       - ".github/workflows/test_onnx_ops.yml"
       - "onnx_ops/**"
   workflow_dispatch:
@@ -31,23 +36,23 @@ jobs:
       VENV_DIR: ${{ github.workspace }}/.venv
       CONFIG_FILE_PATH: onnx_ops/configs/onnx_ops_cpu_llvm_sync.json
     steps:
-      - name: "Checking out repository"
+      - name: Checkout repository
         uses: actions/checkout@v4
 
       # Install Python packages.
-      - name: "Setting up Python"
+      - name: Setup Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-      - name: "Setup Python venv"
+      - name: Setup Python venv
         run: python3 -m venv ${VENV_DIR}
-      - name: "Installing IREE nightly release Python packages"
+      - name: Install IREE nightly release Python packages
         run: |
           source ${VENV_DIR}/bin/activate
           python3 -m pip install -r onnx_ops/requirements-iree.txt
 
       # Run tests and output new config files as needed.
-      - name: "Running the ONNX ops test suite"
+      - name: Run ONNX ops test suite
         run: |
           source ${VENV_DIR}/bin/activate
           pytest onnx_ops/ \
@@ -57,15 +62,15 @@ jobs:
             --durations=10 \
             --report-log=/tmp/onnx_ops_cpu_logs.json \
             --config-files=${CONFIG_FILE_PATH}
-      - name: "Updating config file with latest XFAIL lists"
+      - name: Update config file with latest XFAIL lists
         if: failure()
         run: |
           source ${VENV_DIR}/bin/activate
           python onnx_ops/update_config_xfails.py \
             --log-file=/tmp/onnx_ops_cpu_logs.json \
             --config-file=${CONFIG_FILE_PATH}
           cat ${CONFIG_FILE_PATH}
-      - name: "Uploading new config file"
+      - name: Upload new config file
         if: failure()
         uses: actions/upload-artifact@v4
         with:

diff --git a/README.md b/README.md
@@ -8,10 +8,13 @@ See https://groups.google.com/g/iree-discuss/c/GIWyj8hmP0k/ for context.
 
 ## Test suites
 
-### [matmul/](matmul/) : matrix multiplication
+### [linalg_ops/](linalg_ops/) : 'linalg' and related ops
+
+[![Test Linalg Ops](https://github.com/iree-org/iree-test-suites/actions/workflows/test_linalg_ops.yml/badge.svg?branch=main)](https://github.com/iree-org/iree-test-suites/actions/workflows/test_linalg_ops.yml?query=branch%3Amain)
 
 * Generated tests for matrix multiplication using the
-  [MLIR 'linalg' dialect](https://mlir.llvm.org/docs/Dialects/Linalg/).
+  [MLIR 'linalg' dialect](https://mlir.llvm.org/docs/Dialects/Linalg/)
+  (tests for other ops like 'attention' and 'convolution' are planned).
 * Built with [cmake](https://cmake.org/) and run via
   [ctest](https://cmake.org/cmake/help/latest/manual/ctest.1.html) (for now?).
 

diff --git a/matmul/CMakeLists.txt → linalg_ops/CMakeLists.txt b/matmul/CMakeLists.txt → linalg_ops/CMakeLists.txt
@@ -10,7 +10,7 @@ cmake_minimum_required(VERSION 3.21...3.24)
 # Project configuration
 #-------------------------------------------------------------------------------
 
-project(iree-test-suites-matmul C CXX)
+project(iree-test-suites-linalg C CXX)
 set(CMAKE_C_STANDARD 11)
 set(CMAKE_CXX_STANDARD 17)
 set_property(GLOBAL PROPERTY USE_FOLDERS ON)
@@ -47,11 +47,10 @@ else()
   # see https://gitlab.kitware.com/cmake/cmake/-/issues/18238#note_440475
   include(FetchContent)
 
-  # TODO(scotttodd): pin to a version from a stable release?
   FetchContent_Declare(
     iree
     GIT_REPOSITORY https://github.com/iree-org/iree.git
-    GIT_TAG f1319fc3404a2c97bfc76c6ee2268a96954a1b2c # 2024-08-16
+    GIT_TAG candidate-20240828.999
     GIT_SUBMODULES_RECURSE OFF
     GIT_SHALLOW OFF
     GIT_PROGRESS ON
@@ -69,11 +68,11 @@ endif()
 # Test code
 #-------------------------------------------------------------------------------
 
-enable_testing(iree-test-suites-matmul)
+enable_testing(iree-test-suites-linalg-ops)
 
-add_custom_target(iree-test-suites-matmul-deps
+add_custom_target(iree-test-suites-linalg-ops-deps
   COMMENT
-    "Building matmul test suite deps"
+    "Building linalg operator test suite deps"
 )
 
 iree_cc_library(
@@ -125,4 +124,4 @@ iree_cc_binary(
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR})
 include(iree_e2e_generated_runner_test)
 
-add_subdirectory(tests)
+add_subdirectory(matmul)
diff --git a/matmul/README.md → linalg_ops/README.md b/matmul/README.md → linalg_ops/README.md
@@ -1,16 +1,19 @@
-# Matrix Multiplication (Matmul) Tests
+# Linalg Operator Tests
 
 These are being migrated from
-https://github.com/iree-org/iree/tree/main/tests/e2e/matmul.
 
-## Prerequisites
+* https://github.com/iree-org/iree/tree/main/tests/e2e/attention
+* https://github.com/iree-org/iree/tree/main/tests/e2e/convolution
+* https://github.com/iree-org/iree/tree/main/tests/e2e/matmul
 
-## Quickstart
+## Prerequisites
 
 First ensure you have the prerequisites from
 https://iree.dev/building-from-source/getting-started/, including CMake, a
 compiler like clang, and Python.
 
+## Quickstart
+
 1. Get the IREE compiler tools, either from release packages or a source build:
 
     * To use Python packages:
@@ -60,7 +63,7 @@ compiler like clang, and Python.
 
     ```bash
     cmake --build build/
-    cmake --build build/ --target iree-test-suites-matmul-deps
+    cmake --build build/ --target iree-test-suites-linalg-ops-deps
     ```
 
 4. Run tests:

diff --git a/matmul/iree-e2e-matmul-test.cc → linalg_ops/iree-e2e-matmul-test.cc b/matmul/iree-e2e-matmul-test.cc → linalg_ops/iree-e2e-matmul-test.cc
diff --git a/matmul/iree_e2e_generated_runner_test.cmake → ..._ops/iree_e2e_generated_runner_test.cmake b/matmul/iree_e2e_generated_runner_test.cmake → ..._ops/iree_e2e_generated_runner_test.cmake
@@ -105,7 +105,7 @@ function(iree_e2e_runner_test)
     "${_RULE_TEST_RUNNER}"
   )
 
-  add_dependencies(iree-test-suites-matmul-deps "${_NAME}${_RULE_VARIANT_NAME}")
+  add_dependencies(iree-test-suites-linalg-ops-deps "${_NAME}${_RULE_VARIANT_NAME}")
 
   if(_RULE_TEST_DEFINED)
     iree_native_test(

diff --git a/matmul/tests/CMakeLists.txt → linalg_ops/matmul/CMakeLists.txt b/matmul/tests/CMakeLists.txt → linalg_ops/matmul/CMakeLists.txt
@@ -1888,7 +1888,7 @@ iree_generated_e2e_runner_test(
   DRIVERS
     "cuda"
   COMPILER_FLAGS
-    "--iree-hal-cuda-llvm-target-arch=sm_80"
+    "--iree-cuda-target=sm_80"
   LABELS
     "noasan"
     "nomsan"
@@ -1915,7 +1915,7 @@ iree_generated_e2e_runner_test(
   DRIVERS
     "cuda"
   COMPILER_FLAGS
-    "--iree-hal-cuda-llvm-target-arch=sm_80"
+    "--iree-cuda-target=sm_80"
   LABELS
     "noasan"
     "nomsan"
@@ -1942,7 +1942,7 @@ iree_generated_e2e_runner_test(
   DRIVERS
     "cuda"
   COMPILER_FLAGS
-    "--iree-flow-split-matmul-reduction=4"
+    "--iree-dispatch-creation-split-matmul-reduction=4"
   LABELS
     "noasan"
     "nomsan"

diff --git a/matmul/tests/generate_e2e_matmul_tests.py → ...g_ops/matmul/generate_e2e_matmul_tests.py b/matmul/tests/generate_e2e_matmul_tests.py → ...g_ops/matmul/generate_e2e_matmul_tests.py
diff --git a/matmul/requirements-iree.txt → linalg_ops/requirements-iree.txt b/matmul/requirements-iree.txt → linalg_ops/requirements-iree.txt
diff --git a/matmul/test_utils.c → linalg_ops/test_utils.c b/matmul/test_utils.c → linalg_ops/test_utils.c
@@ -130,8 +130,8 @@ iree_test_utils_e2e_value_t iree_test_utils_read_buffer_element(
   } else if (result_type == IREE_HAL_ELEMENT_TYPE_FLOAT_32) {
     return iree_test_utils_value_make_f32(((float*)data)[index]);
   }
-  iree_status_abort(iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
-                                     "unhandled matmul result type"));
+  iree_status_abort(
+      iree_make_status(IREE_STATUS_INVALID_ARGUMENT, "unhandled result type"));
   return iree_test_utils_value_make_none();
 }
 

diff --git a/matmul/test_utils.h → linalg_ops/test_utils.h b/matmul/test_utils.h → linalg_ops/test_utils.h
@@ -4,8 +4,8 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#ifndef IREE_TEST_SUITES_MATMUL_TEST_UTILS_H_
-#define IREE_TEST_SUITES_MATMUL_TEST_UTILS_H_
+#ifndef IREE_TEST_SUITES_LINALG_OPS_TEST_UTILS_H_
+#define IREE_TEST_SUITES_LINALG_OPS_TEST_UTILS_H_
 
 #include <stdio.h>
 
@@ -146,4 +146,4 @@ iree_status_t iree_test_utils_load_and_run_e2e_tests(
 }  // extern "C"
 #endif  // __cplusplus
 
-#endif  // IREE_TEST_SUITES_MATMUL_TEST_UTILS_H_
+#endif  // IREE_TEST_SUITES_LINALG_OPS_TEST_UTILS_H_