From 80be52bdc2cdc63f3d45c1413ac62d62a635a8ed Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Wed, 24 Jul 2024 16:08:31 -0700 Subject: [PATCH] Move 'gcc', 'byo_llvm', and 'debug' jobs to their own workflows. (#17981) Progress on https://github.com/iree-org/iree/issues/17957 - bringing back jobs that were disabled, with changes to the `runs-on` and triggering. For now these are the configurations: * `gcc` runs on a nightly `schedule` on standard GitHub-hosted runners, taking ~3h30m * `byo_llvm` runs on a nightly `schedule` on standard GitHub-hosted runners, taking ~3h20m * `debug` runs on a nightly `schedule` on self-hosted runners, taking ~20m * This build runs out of disk space for standard GitHub-hosted runners We can adjust the triggers over time, such as by adding ways to trigger with labels / git trailers, or by triggering on `pull_request` when file paths like `third_party/llvm-project` are changed. skip-ci: adding new workflows --- .github/workflows/ci.yml | 94 ------------------- .../workflows/ci_linux_x64_clang_byollvm.yml | 40 ++++++++ .../workflows/ci_linux_x64_clang_debug.yml | 61 ++++++++++++ .github/workflows/ci_linux_x64_gcc.yml | 48 ++++++++++ 4 files changed, 149 insertions(+), 94 deletions(-) create mode 100644 .github/workflows/ci_linux_x64_clang_byollvm.yml create mode 100644 .github/workflows/ci_linux_x64_clang_debug.yml create mode 100644 .github/workflows/ci_linux_x64_gcc.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89961435c48e..8be9d44f5c03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -456,40 +456,6 @@ jobs: ./build_tools/cmake/ctest_all.sh \ "${BUILD_DIR}" - # Disabled to reduce self-hosted runners needed. See #17957 - # gcc: - # needs: setup - # if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'gcc') - # runs-on: - # - self-hosted # must come first - # - runner-group=${{ needs.setup.outputs.runner-group }} - # - environment=${{ needs.setup.outputs.runner-env }} - # - cpu - # - os-family=Linux - # env: - # BUILD_DIR: build-gcc - # steps: - # - name: "Checking out repository" - # uses: actions/checkout@v4.1.7 - # with: - # submodules: true - # - name: "Building IREE with gcc" - # env: - # IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }} - # run: | - # ./build_tools/github_actions/docker_run.sh \ - # --env CC=/usr/bin/gcc-9 \ - # --env CXX=/usr/bin/g++-9 \ - # --env CMAKE_BUILD_TYPE=Release \ - # --env "IREE_TARGET_BACKEND_WEBGPU_SPIRV=OFF" \ - # --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \ - # --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \ - # --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \ - # --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \ - # gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \ - # ./build_tools/cmake/build_all.sh \ - # "${BUILD_DIR}" - tracing: needs: setup if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'tracing') @@ -516,63 +482,6 @@ jobs: ./build_tools/cmake/build_runtime_tracing.sh \ "${BUILD_DIR}" - # Disabled to reduce self-hosted runners needed. See #17957 - # debug: - # needs: setup - # if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'debug') - # runs-on: - # - self-hosted # must come first - # - runner-group=${{ needs.setup.outputs.runner-group }} - # - environment=${{ needs.setup.outputs.runner-env }} - # - cpu - # - os-family=Linux - # env: - # BUILD_DIR: build-debug - # steps: - # - name: "Checking out repository" - # uses: actions/checkout@v4.1.7 - # with: - # submodules: true - # - name: "Building IREE in Debug configuration" - # env: - # IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }} - # run: | - # ./build_tools/github_actions/docker_run.sh \ - # --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \ - # --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \ - # --env "CMAKE_BUILD_TYPE=Debug" \ - # --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \ - # --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \ - # gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \ - # ./build_tools/cmake/build_all.sh \ - # "${BUILD_DIR}" - - # Disabled to reduce self-hosted runners needed. See #17957 - # byo_llvm: - # needs: setup - # if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'byo_llvm') - # runs-on: - # - self-hosted # must come first - # - runner-group=${{ needs.setup.outputs.runner-group }} - # - environment=${{ needs.setup.outputs.runner-env }} - # - cpu - # - os-family=Linux - # steps: - # - name: "Checking out repository" - # uses: actions/checkout@v4.1.7 - # with: - # submodules: true - # - name: "Building and testing with bring-your-own-LLVM" - # env: - # IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }} - # run: | - # ./build_tools/github_actions/docker_run.sh \ - # --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \ - # --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \ - # --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \ - # gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \ - # ./build_tools/cmake/build_and_test_byo_llvm.sh - ############################## Crosscompilation ############################## # Jobs that cross-compile IREE for other platforms ############################################################################## @@ -706,10 +615,7 @@ jobs: # Configurations - build_test_runtime - small_runtime - # - gcc - tracing - # - debug - # - byo_llvm # Crosscompilation # - cross_compile_and_test diff --git a/.github/workflows/ci_linux_x64_clang_byollvm.yml b/.github/workflows/ci_linux_x64_clang_byollvm.yml new file mode 100644 index 000000000000..06954adc3855 --- /dev/null +++ b/.github/workflows/ci_linux_x64_clang_byollvm.yml @@ -0,0 +1,40 @@ +# Copyright 2024 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: CI - Linux x64 clang BYO LLVM + +on: + pull_request: + branches: + - main + paths: + - ".github/workflows/ci_linux_x64_clang_byollvm.yml" + schedule: + # Weekday mornings at 09:15 UTC = 01:15 PST (UTC - 8). + - cron: "15 9 * * 1-5" + workflow_dispatch: + +concurrency: + # A PR number if a pull request and otherwise the commit hash. This cancels + # queued and in-progress runs for the same PR (presubmit) or commit + # (postsubmit). The workflow name is prepended to avoid conflicts between + # different workflows. + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + linux_x64_clang_byollvm: + runs-on: ubuntu-20.04 + steps: + - name: "Checking out repository" + uses: actions/checkout@v4.1.7 + with: + submodules: true + - name: "Building and testing with bring-your-own-LLVM" + run: | + ./build_tools/github_actions/docker_run.sh \ + gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \ + ./build_tools/cmake/build_and_test_byo_llvm.sh diff --git a/.github/workflows/ci_linux_x64_clang_debug.yml b/.github/workflows/ci_linux_x64_clang_debug.yml new file mode 100644 index 000000000000..1c82415aac78 --- /dev/null +++ b/.github/workflows/ci_linux_x64_clang_debug.yml @@ -0,0 +1,61 @@ +# Copyright 2024 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: CI - Linux x64 clang debug + +on: + pull_request: + branches: + - main + paths: + - ".github/workflows/ci_linux_x64_clang_debug.yml" + schedule: + # Weekday mornings at 09:15 UTC = 01:15 PST (UTC - 8). + - cron: "15 9 * * 1-5" + workflow_dispatch: + +concurrency: + # A PR number if a pull request and otherwise the commit hash. This cancels + # queued and in-progress runs for the same PR (presubmit) or commit + # (postsubmit). The workflow name is prepended to avoid conflicts between + # different workflows. + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + setup: + uses: ./.github/workflows/setup.yml + + # This may run out of memory / disk space on standard GitHub-hosted runners, + # so run on self-hosted CPU build runners instead. + linux_x64_clang_debug: + needs: setup + runs-on: + - self-hosted # must come first + - runner-group=${{ needs.setup.outputs.runner-group }} + - environment=${{ needs.setup.outputs.runner-env }} + - cpu + - os-family=Linux + env: + BUILD_DIR: build-debug + steps: + - name: "Checking out repository" + uses: actions/checkout@v4.1.7 + with: + submodules: true + - name: "Building IREE in Debug configuration" + env: + IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }} + run: | + ./build_tools/github_actions/docker_run.sh \ + --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \ + --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \ + --env "CMAKE_BUILD_TYPE=Debug" \ + --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \ + --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \ + gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \ + ./build_tools/cmake/build_all.sh \ + "${BUILD_DIR}" diff --git a/.github/workflows/ci_linux_x64_gcc.yml b/.github/workflows/ci_linux_x64_gcc.yml new file mode 100644 index 000000000000..b2ccd801354d --- /dev/null +++ b/.github/workflows/ci_linux_x64_gcc.yml @@ -0,0 +1,48 @@ +# Copyright 2024 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: CI - Linux x64 gcc + +on: + pull_request: + branches: + - main + paths: + - ".github/workflows/ci_linux_x64_gcc.yml" + schedule: + # Weekday mornings at 09:15 UTC = 01:15 PST (UTC - 8). + - cron: "15 9 * * 1-5" + workflow_dispatch: + +concurrency: + # A PR number if a pull request and otherwise the commit hash. This cancels + # queued and in-progress runs for the same PR (presubmit) or commit + # (postsubmit). The workflow name is prepended to avoid conflicts between + # different workflows. + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + linux_x64_gcc: + runs-on: ubuntu-20.04 + env: + BUILD_DIR: build-gcc + steps: + - name: "Checking out repository" + uses: actions/checkout@v4.1.7 + with: + submodules: true + - name: "Building IREE with gcc" + run: | + ./build_tools/github_actions/docker_run.sh \ + --env CC=/usr/bin/gcc-9 \ + --env CXX=/usr/bin/g++-9 \ + --env CMAKE_BUILD_TYPE=Release \ + --env "IREE_TARGET_BACKEND_WEBGPU_SPIRV=OFF" \ + --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \ + gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \ + ./build_tools/cmake/build_all.sh \ + "${BUILD_DIR}"