Move 'gcc', 'byo_llvm', and 'debug' jobs to their own workflows. (ire…

…e-org#17981) Progress on iree-org#17957 - bringing back jobs that were disabled, with changes to the `runs-on` and triggering. For now these are the configurations: * `gcc` runs on a nightly `schedule` on standard GitHub-hosted runners, taking ~3h30m * `byo_llvm` runs on a nightly `schedule` on standard GitHub-hosted runners, taking ~3h20m * `debug` runs on a nightly `schedule` on self-hosted runners, taking ~20m * This build runs out of disk space for standard GitHub-hosted runners We can adjust the triggers over time, such as by adding ways to trigger with labels / git trailers, or by triggering on `pull_request` when file paths like `third_party/llvm-project` are changed. skip-ci: adding new workflows
nod-ai · Jul 24, 2024 · 80be52b · 80be52b
1 parent 52eff2d
commit 80be52b
Show file tree

Hide file tree

Showing 4 changed files with 149 additions and 94 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -456,40 +456,6 @@ jobs:
             ./build_tools/cmake/ctest_all.sh \
             "${BUILD_DIR}"
 
-  # Disabled to reduce self-hosted runners needed. See #17957
-  # gcc:
-  #   needs: setup
-  #   if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'gcc')
-  #   runs-on:
-  #     - self-hosted # must come first
-  #     - runner-group=${{ needs.setup.outputs.runner-group }}
-  #     - environment=${{ needs.setup.outputs.runner-env }}
-  #     - cpu
-  #     - os-family=Linux
-  #   env:
-  #     BUILD_DIR: build-gcc
-  #   steps:
-  #     - name: "Checking out repository"
-  #       uses: actions/[email protected]
-  #       with:
-  #         submodules: true
-  #     - name: "Building IREE with gcc"
-  #       env:
-  #         IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
-  #       run: |
-  #         ./build_tools/github_actions/docker_run.sh \
-  #           --env CC=/usr/bin/gcc-9 \
-  #           --env CXX=/usr/bin/g++-9 \
-  #           --env CMAKE_BUILD_TYPE=Release \
-  #           --env "IREE_TARGET_BACKEND_WEBGPU_SPIRV=OFF" \
-  #           --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
-  #           --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
-  #           --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \
-  #           --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \
-  #           gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
-  #           ./build_tools/cmake/build_all.sh \
-  #           "${BUILD_DIR}"
-
   tracing:
     needs: setup
     if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'tracing')
@@ -516,63 +482,6 @@ jobs:
             ./build_tools/cmake/build_runtime_tracing.sh \
             "${BUILD_DIR}"
 
-  # Disabled to reduce self-hosted runners needed. See #17957
-  # debug:
-  #   needs: setup
-  #   if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'debug')
-  #   runs-on:
-  #     - self-hosted # must come first
-  #     - runner-group=${{ needs.setup.outputs.runner-group }}
-  #     - environment=${{ needs.setup.outputs.runner-env }}
-  #     - cpu
-  #     - os-family=Linux
-  #   env:
-  #     BUILD_DIR: build-debug
-  #   steps:
-  #     - name: "Checking out repository"
-  #       uses: actions/[email protected]
-  #       with:
-  #         submodules: true
-  #     - name: "Building IREE in Debug configuration"
-  #       env:
-  #         IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
-  #       run: |
-  #         ./build_tools/github_actions/docker_run.sh \
-  #           --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
-  #           --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
-  #           --env "CMAKE_BUILD_TYPE=Debug" \
-  #           --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \
-  #           --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \
-  #           gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
-  #           ./build_tools/cmake/build_all.sh \
-  #           "${BUILD_DIR}"
-
-  # Disabled to reduce self-hosted runners needed. See #17957
-  # byo_llvm:
-  #   needs: setup
-  #   if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'byo_llvm')
-  #   runs-on:
-  #     - self-hosted # must come first
-  #     - runner-group=${{ needs.setup.outputs.runner-group }}
-  #     - environment=${{ needs.setup.outputs.runner-env }}
-  #     - cpu
-  #     - os-family=Linux
-  #   steps:
-  #     - name: "Checking out repository"
-  #       uses: actions/[email protected]
-  #       with:
-  #         submodules: true
-  #     - name: "Building and testing with bring-your-own-LLVM"
-  #       env:
-  #         IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
-  #       run: |
-  #         ./build_tools/github_actions/docker_run.sh \
-  #           --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
-  #           --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
-  #           --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \
-  #           gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
-  #           ./build_tools/cmake/build_and_test_byo_llvm.sh
-
   ############################## Crosscompilation ##############################
   # Jobs that cross-compile IREE for other platforms
   ##############################################################################
@@ -706,10 +615,7 @@ jobs:
       # Configurations
       - build_test_runtime
       - small_runtime
-      # - gcc
       - tracing
-      # - debug
-      # - byo_llvm
 
       # Crosscompilation
       # - cross_compile_and_test

diff --git a/.github/workflows/ci_linux_x64_clang_byollvm.yml b/.github/workflows/ci_linux_x64_clang_byollvm.yml
@@ -0,0 +1,40 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+name: CI - Linux x64 clang BYO LLVM
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/ci_linux_x64_clang_byollvm.yml"
+  schedule:
+    # Weekday mornings at 09:15 UTC = 01:15 PST (UTC - 8).
+    - cron: "15 9 * * 1-5"
+  workflow_dispatch:
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  linux_x64_clang_byollvm:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: "Checking out repository"
+        uses: actions/[email protected]
+        with:
+          submodules: true
+      - name: "Building and testing with bring-your-own-LLVM"
+        run: |
+          ./build_tools/github_actions/docker_run.sh \
+            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
+            ./build_tools/cmake/build_and_test_byo_llvm.sh
diff --git a/.github/workflows/ci_linux_x64_clang_debug.yml b/.github/workflows/ci_linux_x64_clang_debug.yml
@@ -0,0 +1,61 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+name: CI - Linux x64 clang debug
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/ci_linux_x64_clang_debug.yml"
+  schedule:
+    # Weekday mornings at 09:15 UTC = 01:15 PST (UTC - 8).
+    - cron: "15 9 * * 1-5"
+  workflow_dispatch:
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  setup:
+    uses: ./.github/workflows/setup.yml
+
+  # This may run out of memory / disk space on standard GitHub-hosted runners,
+  # so run on self-hosted CPU build runners instead.
+  linux_x64_clang_debug:
+    needs: setup
+    runs-on:
+      - self-hosted # must come first
+      - runner-group=${{ needs.setup.outputs.runner-group }}
+      - environment=${{ needs.setup.outputs.runner-env }}
+      - cpu
+      - os-family=Linux
+    env:
+      BUILD_DIR: build-debug
+    steps:
+      - name: "Checking out repository"
+        uses: actions/[email protected]
+        with:
+          submodules: true
+      - name: "Building IREE in Debug configuration"
+        env:
+          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
+        run: |
+          ./build_tools/github_actions/docker_run.sh \
+            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
+            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
+            --env "CMAKE_BUILD_TYPE=Debug" \
+            --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \
+            --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \
+            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
+            ./build_tools/cmake/build_all.sh \
+            "${BUILD_DIR}"
diff --git a/.github/workflows/ci_linux_x64_gcc.yml b/.github/workflows/ci_linux_x64_gcc.yml
@@ -0,0 +1,48 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+name: CI - Linux x64 gcc
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/ci_linux_x64_gcc.yml"
+  schedule:
+    # Weekday mornings at 09:15 UTC = 01:15 PST (UTC - 8).
+    - cron: "15 9 * * 1-5"
+  workflow_dispatch:
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  linux_x64_gcc:
+    runs-on: ubuntu-20.04
+    env:
+      BUILD_DIR: build-gcc
+    steps:
+      - name: "Checking out repository"
+        uses: actions/[email protected]
+        with:
+          submodules: true
+      - name: "Building IREE with gcc"
+        run: |
+          ./build_tools/github_actions/docker_run.sh \
+            --env CC=/usr/bin/gcc-9 \
+            --env CXX=/usr/bin/g++-9 \
+            --env CMAKE_BUILD_TYPE=Release \
+            --env "IREE_TARGET_BACKEND_WEBGPU_SPIRV=OFF" \
+            --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \
+            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
+            ./build_tools/cmake/build_all.sh \
+            "${BUILD_DIR}"