Prune ccache caches in CI on each run (#457) #766
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| tags: | |
| - 'wasi-sdk-*' | |
| branches: | |
| - main | |
| pull_request: | |
| workflow_dispatch: | |
| jobs: | |
| build: | |
| name: Build ${{ matrix.artifact }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - artifact: x86_64-linux | |
| os: ubuntu-latest | |
| - artifact: arm64-linux | |
| os: ubuntu-latest | |
| rust_target: aarch64-unknown-linux-gnu | |
| env: | |
| # Don't build a sysroot for this cross-compiled target since it | |
| # would require a host compiler and the sysroot is otherwise | |
| # already built on other CI builders. | |
| WASI_SDK_CI_SKIP_SYSROOT: 1 | |
| WASI_SDK_CI_TOOLCHAIN_LLVM_CMAKE_ARGS: >- | |
| -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc | |
| -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ | |
| -DCMAKE_CROSSCOMPILING=True | |
| -DCMAKE_CXX_FLAGS=-march=armv8-a | |
| -DCMAKE_SYSTEM_PROCESSOR=arm64 | |
| -DCMAKE_SYSTEM_NAME=Linux | |
| -DLLVM_HOST_TRIPLE=aarch64-linux-gnu | |
| -DRUST_TARGET=aarch64-unknown-linux-gnu | |
| - artifact: arm64-macos | |
| os: macos-latest | |
| rust_target: aarch64-apple-darwin | |
| env: | |
| WASI_SDK_CI_TOOLCHAIN_LLVM_CMAKE_ARGS: >- | |
| -DCMAKE_OSX_DEPLOYMENT_TARGET=10.12 | |
| -DCMAKE_OSX_ARCHITECTURES=arm64 | |
| - artifact: x86_64-macos | |
| os: macos-latest | |
| rust_target: x86_64-apple-darwin | |
| env: | |
| WASI_SDK_CI_SKIP_SYSROOT: 1 | |
| WASI_SDK_CI_TOOLCHAIN_LLVM_CMAKE_ARGS: >- | |
| -DCMAKE_OSX_DEPLOYMENT_TARGET=10.12 | |
| -DCMAKE_OSX_ARCHITECTURES=x86_64 | |
| - artifact: x86_64-windows | |
| os: windows-latest | |
| env: | |
| # TODO: tests are pretty close to passing on Windows but need some | |
| # final tweaks, namely testing the exit code doesn't work since | |
| # exit codes are different on Windows and the `mmap.c` tests seems | |
| # to have issues probably with line endings. Needs someone with a | |
| # Windows checkout tot test further. | |
| WASI_SDK_CI_SKIP_TESTS: 1 | |
| env: ${{ matrix.env || fromJSON('{}') }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - run: git fetch --tags --force | |
| name: Force-fetch tags to work around actions/checkout#290 | |
| # We can't use `--depth 1` here sadly because the GNU config | |
| # submodule is not pinned to a particular tag/branch. Please | |
| # bump depth (or even better, the submodule), in case of "error: | |
| # Server does not allow request for unadvertised object" in the | |
| # future. | |
| - run: git submodule update --init --depth 32 --jobs 3 | |
| # Persist ccache-based caches across builds. This directory is configured | |
| # via the CCACHE_DIR env var below for ccache to use. | |
| # | |
| # Bump the prefix number to evict all previous caches and enforce a clean | |
| # build, in the unlikely case that some weird build error occur and ccache | |
| # becomes a potential suspect. | |
| - uses: actions/cache@v4 | |
| id: cache-restore | |
| with: | |
| path: ${{ runner.tool_cache }}/ccache | |
| key: 0-cache-${{ matrix.artifact }}-${{ github.run_id }} | |
| restore-keys: | | |
| 0-cache-${{ matrix.artifact }}- | |
| - run: | | |
| mkdir -p '${{ runner.tool_cache }}/ccache' | |
| echo 'CCACHE_DIR=${{ runner.tool_cache }}/ccache' >> $GITHUB_ENV | |
| shell: bash | |
| # Configure CMake flags for `ci/build.sh` as necessary for each | |
| # matrix entry. | |
| - run: | | |
| cmake_args=-DWASI_SDK_ARTIFACT=${{ matrix.artifact }} | |
| if [ "${{ matrix.rust_target }}" != "" ]; then | |
| rustup target add ${{ matrix.rust_target }} | |
| cmake_args="$cmake_args -DRUST_TARGET=${{ matrix.rust_target }}" | |
| fi | |
| echo WASI_SDK_CI_TOOLCHAIN_CMAKE_ARGS="$cmake_args" >> $GITHUB_ENV | |
| shell: bash | |
| # Add some extra installed software on each runner as necessary. | |
| - name: Setup `wasmtime` for tests | |
| uses: bytecodealliance/actions/wasmtime/setup@v1 | |
| with: | |
| version: "18.0.2" | |
| - name: Install ccache, ninja (macOS) | |
| run: brew install ccache ninja | |
| if: runner.os == 'macOS' | |
| - name: Install ccache, ninja (Windows) | |
| run: choco install ccache ninja | |
| if: runner.os == 'Windows' | |
| - name: Install ccache, ninja (Linux) | |
| run: sudo apt install ccache | |
| if: runner.os == 'Linux' | |
| - name: Build and test (macOS) | |
| run: ./ci/build.sh | |
| if: runner.os == 'macOS' | |
| - name: Build and test (Linux) | |
| run: ./ci/docker-build.sh ${{ matrix.artifact }} | |
| if: runner.os == 'Linux' | |
| # Use a shorter build directory than the default on Windows to avoid | |
| # hitting path length and command line length limits. See | |
| # WebAssembly/wasi-libc#514 | |
| - name: Build and test (Windows) | |
| run: | | |
| ./ci/build.sh C:/wasi-sdk | |
| mkdir build | |
| cp -r C:/wasi-sdk/dist build | |
| shell: bash | |
| if: runner.os == 'Windows' | |
| # Upload the `dist` folder from the build as the artifacts for this | |
| # runner. | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ format( 'dist-{0}', matrix.artifact) }} | |
| path: build/dist | |
| # Caches are persisted across runs by restoring the latest cache which | |
| # means that quite a lot of cruft can accumulate. Prune older entries that | |
| # haven't been used by this run to avoid the cache continuously getting | |
| # larger. In theory this should use `--evict-older-than $dur` where `$dur` | |
| # is the time since the start of the run, but I'm not sure how to easily | |
| # calculate that so pick something loose like one day instead. | |
| - name: Prune ccache objects | |
| run: ccache --evict-older-than 1d | |
| # Help debug ccache issues by showing what happened. | |
| - if: always() | |
| name: Show ccache statistics | |
| run: ccache --show-stats | |
| # Always save a cache, even if the build failed. This ensures that if | |
| # live-debugging via CI the build gets to pick up where it left off last | |
| # time instead of having to recreate everything each time a failure | |
| # happens. | |
| - if: always() && steps.cache-restore.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ${{ runner.tool_cache }}/ccache | |
| key: 0-cache-${{ matrix.artifact }}-${{ github.run_id }} | |
| # Once all of the above matrix entries have completed this job will run and | |
| # assemble the final `wasi-sdk-*` artifacts by fusing the toolchain/sysroot | |
| # artifacts. | |
| finalize: | |
| name: Finalize wasi-sdk artifacts | |
| needs: build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - run: git fetch --tags --force | |
| name: Force-fetch tags to work around actions/checkout#290 | |
| # Download all artifacts from all platforms in `build`, merge them into | |
| # final wasi-sdk-* artifacts, and then upload them. | |
| - uses: actions/download-artifact@v4 | |
| - run: ./ci/merge-artifacts.sh | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: release-artifacts | |
| path: dist | |
| # Use the `wasi-sdk-*` artifacts just created to create a docker image | |
| # with a toolchain pre-installed. | |
| - uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: docker/setup-qemu-action@v2 | |
| - uses: docker/setup-buildx-action@v2 | |
| - uses: docker/metadata-action@v4 | |
| id: meta | |
| with: | |
| images: ghcr.io/${{ github.repository }} | |
| tags: | | |
| type=schedule | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=ref,event=pr | |
| type=sha | |
| - name: Build and push wasi-sdk docker image | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| file: docker/Dockerfile | |
| push: ${{ github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch' }} | |
| platforms: linux/amd64,linux/arm64 | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Publish a draft release | |
| if: startsWith(github.ref, 'refs/tags') | |
| run: gh release create --draft --prerelease --generate-notes ${{ github.ref_name }} ./dist/* | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Test the final artifacts as-is without passing `--sysroot` or | |
| # `-resource-dir` or any extra flags. This exercises running the compiler | |
| # as-is from the distribution tarballs and ensuring that it can build and pass | |
| # all tests. | |
| test-standalone: | |
| name: Test standalone toolchain | |
| needs: build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - run: git fetch --tags --force | |
| name: Force-fetch tags to work around actions/checkout#290 | |
| - run: git submodule update --init --depth 32 --jobs 3 | |
| - name: Setup `wasmtime` for tests | |
| uses: bytecodealliance/actions/wasmtime/setup@v1 | |
| with: | |
| version: "18.0.2" | |
| - name: Install ninja | |
| run: sudo apt-get install -y ninja-build | |
| if: runner.os == 'Linux' | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: dist-x86_64-linux | |
| path: dist-x86_64-linux | |
| - run: ./ci/merge-artifacts.sh | |
| - run: tar xf dist/wasi-sdk-*.tar.gz | |
| - run: | | |
| cmake -G Ninja -B build -S . \ | |
| -DWASI_SDK_INCLUDE_TESTS=ON \ | |
| -DWASI_SDK_TEST_HOST_TOOLCHAIN=ON \ | |
| -DCMAKE_TOOLCHAIN_FILE=$(ls ./wasi-sdk-*/share/cmake/wasi-sdk.cmake) | |
| - run: ninja -C build build-tests | |
| - run: ctest --output-on-failure --parallel 10 --test-dir build/tests |