From f79bebbb4ab9b2889f8e8032e88336ca97139a8a Mon Sep 17 00:00:00 2001 From: Erik de Castro Lopo Date: Sun, 25 Feb 2024 16:21:48 +1100 Subject: [PATCH 1/2] CI: Simplify Github Actions * Make it more like `cardano-base`. * Drop the ad-hoc caching in favour of Cabal caching. --- .github/workflows/haskell.yml | 87 ++++++++++------------------------- 1 file changed, 24 insertions(+), 63 deletions(-) diff --git a/.github/workflows/haskell.yml b/.github/workflows/haskell.yml index a22debd0eb..7af5b558b6 100644 --- a/.github/workflows/haskell.yml +++ b/.github/workflows/haskell.yml @@ -67,49 +67,29 @@ jobs: run: cabal update - name: Build dry run - run: cabal build all --dry-run --minimize-conflict-set + run: cabal build all --enable-tests --dry-run --minimize-conflict-set - # For users who fork cardano-cli and want to define a writable cache, then can set up their own - # S3 bucket then define in their forked repository settings the following secrets: - # - # CACHE_AWS_ACCESS_KEY_ID - # CACHE_AWS_SECRET_ACCESS_KEY - # CACHE_URI - # CACHE_AWS_REGION - - name: Cabal cache over S3 - uses: action-works/cabal-cache-s3@v1 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.CACHE_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.CACHE_AWS_SECRET_ACCESS_KEY }} - with: - region: ${{ vars.CACHE_AWS_REGION }} - dist-dir: dist-newstyle - store-path: ${{ steps.setup-haskell.outputs.cabal-store }} - threads: ${{ vars.CACHE_THREADS }} - archive-uri: ${{ vars.CACHE_URI }}/${{ env.CABAL_CACHE_VERSION }}/${{ runner.os }}/${{ matrix.cabal }}/${{ matrix.ghc }} - skip: "${{ vars.CACHE_URI == '' || env.CABAL_CACHE_VERSION == '' }}" - - # It's important to ensure that people who fork this repository can not only successfully build in - # CI by default, but also have meaning cabal store caching. - # - # Because syncing with S3 requires credentials, we cannot rely on S3 for this. For this reason a - # https fallback is used. The https server mirrors the content of the S3 bucket. The https cabal - # store archive is read-only for security reasons. - # - # Users who fork this repository who want to have a writable cabal store archive are encouraged - # to set up their own S3 bucket. - - name: Cabal cache over HTTPS - uses: action-works/cabal-cache-s3@v1 + - name: Record dependencies + id: record-deps + run: | + # The tests call out to msys2 commands. We generally do not want to mix toolchains, so + # we are very deliberate about only adding msys64 to the path where absolutely necessary. + ${{ (runner.os == 'Windows' && '$env:PATH=("C:\msys64\mingw64\bin;{0}" -f $env:PATH)') || '' }} + cat dist-newstyle/cache/plan.json | jq -r '."install-plan"[].id' | sort | uniq > dependencies.txt + + - name: Install dependencies + run: cabal build all --enable-tests --only-dependencies -j --ghc-option=-j4 + + - name: Cache Cabal store + uses: actions/cache@v3 with: - dist-dir: dist-newstyle - store-path: ${{ steps.setup-haskell.outputs.cabal-store }} - threads: 16 - archive-uri: https://iohk.cache.haskellworks.io/${{ env.CABAL_CACHE_VERSION }}/${{ runner.os }}/${{ matrix.cabal }}/${{ matrix.ghc }} - skip: "${{ vars.CACHE_URI != '' || env.CABAL_CACHE_VERSION == '' }}" - enable-save: false + path: | + ${{ steps.setup-haskell.outputs.cabal-store }} + dist-newstyle + key: cache-${{ env.CABAL_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }} - name: Build all - run: cabal build all + run: cabal build all --enable-tests - name: Run tests env: @@ -119,26 +99,7 @@ jobs: TMPDIR: ${{ runner.temp }} TMP: ${{ runner.temp }} KEEP_WORKSPACE: 1 - run: cabal test all - - - # Delete golden files and regenerate them in the next step to ensure - # files that tests designate as golden files are actually golden files - # and can be generated. - name: Delete golden files - shell: bash - run: | - rm -rf cardano-cli/test/cardano-cli-golden/files/golden - - - name: Run tests regenerate golden files - env: - # these two are msys2 env vars, they have no effect on non-msys2 installs. - MSYS2_PATH_TYPE: inherit - MSYSTEM: MINGW64 - TMPDIR: ${{ runner.temp }} - TMP: ${{ runner.temp }} - KEEP_WORKSPACE: 1 - CREATE_GOLDEN_FILES: 1 - run: cabal test all + run: cabal test all --enable-tests --test-show-details=direct -j1 - name: "Tar artifacts" shell: bash @@ -158,7 +119,7 @@ jobs: done - name: Save Artifact - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: artifacts-${{ matrix.os }}-${{ matrix.ghc }} path: ./artifacts @@ -175,9 +136,9 @@ jobs: # and will silently fail if msys2 is not in path. See the "Run tests" step. # # - name: Setup tmate session - # if: ${{ failure() }} - # uses: mxschmitt/action-tmate@v3 - # with: + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 + # with: # limit-access-to-actor: true build-complete: From 924711bf3d8ff55366fff82f1de958e33a6ea9a0 Mon Sep 17 00:00:00 2001 From: Erik de Castro Lopo Date: Tue, 27 Feb 2024 15:30:46 +1100 Subject: [PATCH 2/2] CI: Actually use the cache dependencies Previously the GHAs would store the cache after building the dependencies but never restore them. Did a thorough investigation of the issue and added proper comments about how it works and why. --- .github/workflows/haskell.yml | 36 +++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/haskell.yml b/.github/workflows/haskell.yml index 7af5b558b6..af2e49a81e 100644 --- a/.github/workflows/haskell.yml +++ b/.github/workflows/haskell.yml @@ -66,9 +66,12 @@ jobs: - name: Cabal update run: cabal update + # A dry run `build all` operation does *NOT* downlaod anything, it just looks at the package + # indices to generate an install plan. - name: Build dry run run: cabal build all --enable-tests --dry-run --minimize-conflict-set + # From the install plan we generate a dependency list. - name: Record dependencies id: record-deps run: | @@ -77,21 +80,49 @@ jobs: ${{ (runner.os == 'Windows' && '$env:PATH=("C:\msys64\mingw64\bin;{0}" -f $env:PATH)') || '' }} cat dist-newstyle/cache/plan.json | jq -r '."install-plan"[].id' | sort | uniq > dependencies.txt + # From the dependency list we restore the cached dependencies. + # We use the hash of `dependencies.txt` as part of the cache key because that will be stable + # until the `index-state` values in the `cabal.project` file changes. + - name: Restore cached dependencies + uses: actions/cache/restore@v4 + id: cache + with: + path: | + ${{ steps.setup-haskell.outputs.cabal-store }} + dist-newstyle + key: cache-${{ env.CABAL_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }} + restore-keys: | + ${{ runner.os }}-ghc-${{ env.GHC_VERSION }}-cabal-${{ env.CABAL_VERSION }}- + cache-${{ env.CABAL_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }} + + # Now we install the dependencies. If the cache was found and restored in the previous step, + # this should be a no-op, but if the cache key was not found we need to build stuff so we can + # cache it for the next step. - name: Install dependencies run: cabal build all --enable-tests --only-dependencies -j --ghc-option=-j4 + # Always store the cabal cache. - name: Cache Cabal store - uses: actions/cache@v3 + uses: actions/cache/save@v4 with: path: | ${{ steps.setup-haskell.outputs.cabal-store }} dist-newstyle key: cache-${{ env.CABAL_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }} + # Now we build. - name: Build all run: cabal build all --enable-tests - - name: Run tests + - # Delete golden files and regenerate them in the next step to ensure + # files that tests designate as golden files are actually golden files + # and can be generated. + name: Delete golden files + shell: bash + run: | + rm -rf cardano-cli/test/cardano-cli-golden/files/golden + + - name: Run tests regenerate golden files env: # these two are msys2 env vars, they have no effect on non-msys2 installs. MSYS2_PATH_TYPE: inherit @@ -99,6 +130,7 @@ jobs: TMPDIR: ${{ runner.temp }} TMP: ${{ runner.temp }} KEEP_WORKSPACE: 1 + CREATE_GOLDEN_FILES: 1 run: cabal test all --enable-tests --test-show-details=direct -j1 - name: "Tar artifacts"