From 32f009d45939d48a190cbbe246d3ce1d5e4c7846 Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Wed, 19 Apr 2023 17:30:42 -0700 Subject: [PATCH 1/8] Use GCS for Windows ccache --- .github/workflows/ci.yml | 43 ++++++++++------------------------------ 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c549b4a410b..1ccc33cc7855 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,19 +83,14 @@ jobs: BUILD_DIR: build-windows IREE_VULKAN_DISABLE: 1 steps: + - id: 'gcp-auth' + name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v1' + with: + token_format: 'access_token' + credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}' - name: "Checking out repository" uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - # Attempt to restore from cache unconditionally. - # Note: this will first try to grab a cache entry for this exact commit - # then it will fall back to the latest for any commit. - - name: "Fetching cache (CMake/ccache)" - uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 - with: - path: ${{ github.workspace }}/.ccache - key: ccache_all_windows_${{ github.sha }} - restore-keys: ccache_all_windows - # Fetch dependencies. - # TODO(scotttodd): Move some of these into a Docker image / add to PATH. - name: "Updating git submodules" run: git submodule update --init --jobs 8 --depth 1 - name: "Setting up Python" @@ -114,30 +109,12 @@ jobs: # Finally: build and run tests. - name: "Building IREE" env: - IREE_READ_REMOTE_CCACHE: 0 - IREE_WRITE_REMOTE_CCACHE: 0 - IREE_READ_LOCAL_CCACHE: 1 - IREE_WRITE_LOCAL_CCACHE: ${{ needs.setup.outputs.write-caches }} - CCACHE_DIR: ${{ github.workspace }}/.ccache - # Cache size and compression level settings are a delicate balance. - # * A full build cache is around 2-5GB depending on compression level - # * Upload/download is slow (double compression may or may not help) - # * We have a limit of 10GB across all cached files per repository - # * Cache misses are quite costly: - # * 99% cache hits -> ~5 minutes to build - # * 20% cache hits -> ~15-20 minutes to build - CCACHE_MAXSIZE: 4G - CCACHE_COMPRESSLEVEL: 5 - run: ./build_tools/cmake/build_all.sh "${BUILD_DIR}" + IREE_WRITE_REMOTE_CCACHE: ${{ inputs.write-caches }} + IREE_CCACHE_GCP_TOKEN: ${{ steps.gcp-auth.outputs.access_token }} + run: | + ./build_tools/cmake/build_all.sh "${BUILD_DIR}" - name: "Testing IREE" run: ./build_tools/cmake/ctest_all.sh "${BUILD_DIR}" - # Write cache (if configured to) after all other steps are finished. - - name: "Saving cache (CMake/ccache)" - if: needs.setup.outputs.write-caches == '1' - uses: actions/cache/save@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 - with: - path: ${{ github.workspace }}/.ccache - key: ccache_all_windows_${{ github.sha }} build_test_all_macos_arm64: needs: setup From afe58a89db33bcbc52a14bd7ebef1a62c55656eb Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Wed, 19 Apr 2023 17:58:45 -0700 Subject: [PATCH 2/8] REVERT ME: Enable testing on my branch, skipping other jobs --- .github/workflows/ci.yml | 3 ++- .github/workflows/setup.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ccc33cc7855..4100de9f0793 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,7 @@ on: push: branches: - main + - gcmn-gcs-cache concurrency: # A PR number if a pull request and otherwise the commit hash. This cancels @@ -74,7 +75,7 @@ jobs: build_test_all_windows: needs: setup - if: fromJson(needs.setup.outputs.should-run) && ! fromJson(needs.setup.outputs.is-pr) + # if: fromJson(needs.setup.outputs.should-run) && ! fromJson(needs.setup.outputs.is-pr) runs-on: windows-2022-64core defaults: run: diff --git a/.github/workflows/setup.yml b/.github/workflows/setup.yml index 9a3ab91fbbd0..388b327bb91e 100644 --- a/.github/workflows/setup.yml +++ b/.github/workflows/setup.yml @@ -55,7 +55,7 @@ jobs: # parent will be the tip of main. BASE_REF: HEAD^ outputs: - should-run: ${{ steps.configure.outputs.should-run }} + should-run: "false" is-pr: ${{ steps.configure.outputs.is-pr }} runner-env: ${{ steps.configure.outputs.runner-env }} runner-group: ${{ steps.configure.outputs.runner-group }} From 98d900f9183a4b7ac9dd0b369982a4f2e5067376 Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Wed, 19 Apr 2023 18:00:00 -0700 Subject: [PATCH 3/8] Be consistent about quotes --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4100de9f0793..2511be8c640e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,12 +84,12 @@ jobs: BUILD_DIR: build-windows IREE_VULKAN_DISABLE: 1 steps: - - id: 'gcp-auth' - name: 'Authenticate to Google Cloud' - uses: 'google-github-actions/auth@v1' + - id: "gcp-auth" + name: "Authenticating to Google Cloud" + uses: "google-github-actions/auth@v1" with: - token_format: 'access_token' - credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}' + token_format: "access_token" + credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}" - name: "Checking out repository" uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - name: "Updating git submodules" From fda431a5f2d06de7b2221d3ea2babe8a70ff2fe8 Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Wed, 19 Apr 2023 18:01:28 -0700 Subject: [PATCH 4/8] Use correct secret name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2511be8c640e..7df3c060d5c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,7 @@ jobs: uses: "google-github-actions/auth@v1" with: token_format: "access_token" - credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}" + credentials_json: "${{ secrets.IREE_OSS_GITHUB_RUNNER_BASIC_TRUST_SERVICE_ACCOUNT_KEY }}" - name: "Checking out repository" uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - name: "Updating git submodules" From 36109d456ed3b4fc71d55ce3d8a891b0066be76c Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Wed, 19 Apr 2023 18:03:37 -0700 Subject: [PATCH 5/8] Don't try to create credentials file --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7df3c060d5c9..142efdc19bbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,6 +90,7 @@ jobs: with: token_format: "access_token" credentials_json: "${{ secrets.IREE_OSS_GITHUB_RUNNER_BASIC_TRUST_SERVICE_ACCOUNT_KEY }}" + create_credentials_file: false - name: "Checking out repository" uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - name: "Updating git submodules" From 9d34b72cf852e3af8114ff76019bcb99f791d9e6 Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Wed, 19 Apr 2023 20:35:32 -0700 Subject: [PATCH 6/8] Sure why give helpful errors when you can just silently do the wrong thing? --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 142efdc19bbc..0d57f4e40080 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,7 +111,7 @@ jobs: # Finally: build and run tests. - name: "Building IREE" env: - IREE_WRITE_REMOTE_CCACHE: ${{ inputs.write-caches }} + IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }} IREE_CCACHE_GCP_TOKEN: ${{ steps.gcp-auth.outputs.access_token }} run: | ./build_tools/cmake/build_all.sh "${BUILD_DIR}" From 76f539b4b0d31106951ea2873e93ee2bb590c57e Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Wed, 19 Apr 2023 20:36:44 -0700 Subject: [PATCH 7/8] Use a ccache namespace --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d57f4e40080..230617856b1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,6 +113,7 @@ jobs: env: IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }} IREE_CCACHE_GCP_TOKEN: ${{ steps.gcp-auth.outputs.access_token }} + CCACHE_NAMESPACE: github-windows-2022-64core run: | ./build_tools/cmake/build_all.sh "${BUILD_DIR}" - name: "Testing IREE" From 227ade64b94881619a369fd0b3e421212e0e0bc3 Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Wed, 19 Apr 2023 21:43:14 -0700 Subject: [PATCH 8/8] Revert "REVERT ME: Enable testing on my branch, skipping other jobs" This reverts commit afe58a89db33bcbc52a14bd7ebef1a62c55656eb. --- .github/workflows/ci.yml | 3 +-- .github/workflows/setup.yml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 230617856b1d..0d46e54efec9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,6 @@ on: push: branches: - main - - gcmn-gcs-cache concurrency: # A PR number if a pull request and otherwise the commit hash. This cancels @@ -75,7 +74,7 @@ jobs: build_test_all_windows: needs: setup - # if: fromJson(needs.setup.outputs.should-run) && ! fromJson(needs.setup.outputs.is-pr) + if: fromJson(needs.setup.outputs.should-run) && ! fromJson(needs.setup.outputs.is-pr) runs-on: windows-2022-64core defaults: run: diff --git a/.github/workflows/setup.yml b/.github/workflows/setup.yml index 388b327bb91e..9a3ab91fbbd0 100644 --- a/.github/workflows/setup.yml +++ b/.github/workflows/setup.yml @@ -55,7 +55,7 @@ jobs: # parent will be the tip of main. BASE_REF: HEAD^ outputs: - should-run: "false" + should-run: ${{ steps.configure.outputs.should-run }} is-pr: ${{ steps.configure.outputs.is-pr }} runner-env: ${{ steps.configure.outputs.runner-env }} runner-group: ${{ steps.configure.outputs.runner-group }}