Rework racket based runtime/compilation #7357
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | ||
| defaults: | ||
| run: | ||
| shell: bash | ||
| on: | ||
| # Build on every pull request (and new PR commit) | ||
| pull_request: | ||
| # Build on new pushes to trunk (E.g. Merge commits) | ||
| # Without the branch filter, each commit on a branch with a PR is triggered twice. | ||
| # See: https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 | ||
| push: | ||
| branches: | ||
| - trunk | ||
| tags: | ||
| - release/* | ||
| workflow_dispatch: | ||
| env: | ||
| ormolu_version: "0.5.0.1" | ||
| racket_version: "8.7" | ||
| ucm_local_bin: "ucm-local-bin" | ||
| jit_version: "@unison/internal/releases/0.0.10" | ||
| jit_src_scheme: "unison-jit-src/scheme-libs/racket" | ||
| jit_bin: "unison-jit-bin" | ||
| jit_generator_os: ubuntu-20.04 | ||
| jobs: | ||
| ormolu: | ||
| runs-on: ubuntu-20.04 | ||
| # Only run formatting on trunk commits | ||
| # This is because the job won't have permission to push back to | ||
| # contributor forks on contributor PRs. | ||
| if: github.ref_name == 'trunk' | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Get changed files | ||
| id: changed-files | ||
| uses: tj-actions/changed-files@v41 | ||
| with: | ||
| # globs copied from default settings for run-ormolu | ||
| files: | | ||
| **/*.hs | ||
| **/*.hs-boot | ||
| separator: "\n" | ||
| - uses: haskell-actions/run-ormolu@v14 | ||
| with: | ||
| version: ${{ env.ormolu_version }} | ||
| mode: inplace | ||
| pattern: ${{ steps.changed-files.outputs.all_changed_files }} | ||
| - name: apply formatting changes | ||
| uses: stefanzweifel/git-auto-commit-action@v4 | ||
| if: ${{ always() }} | ||
| with: | ||
| commit_message: automatically run ormolu | ||
| build-ucm: | ||
| name: Build UCM ${{ matrix.os }} | ||
| runs-on: ${{ matrix.os }} | ||
| if: ${{ always() }} | ||
| needs: ormolu | ||
| strategy: | ||
| # Run each build to completion, regardless of if any have failed | ||
| fail-fast: false | ||
| matrix: | ||
| os: | ||
| # While iterating on this file, you can disable one or more of these to speed things up | ||
| - ubuntu-20.04 | ||
| - macOS-12 | ||
| - windows-2019 | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| # The number towards the beginning of the cache keys allow you to manually avoid using a previous cache. | ||
| # GitHub will automatically delete caches that haven't been accessed in 7 days, but there is no way to | ||
| # purge one manually. | ||
| - id: stackage-resolver | ||
| name: record stackage resolver | ||
| # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#environment-files | ||
| # looks for `resolver: nightly-yyyy-mm-dd` or `resolver: lts-xx.yy` in `stack.yaml` and splits it into | ||
| # `nightly` or `lts-xx`. the whole resolver string is put into resolver_long as a backup cache key | ||
| # ${{ steps.stackage-resolver.outputs.resolver_short }} | ||
| # ${{ steps.stackage-resolver.outputs.resolver_long }} | ||
| run: | | ||
| grep resolver stack.yaml | awk '{ x="resolver_short="; if (split($2,a,"-") > 2) print x a[1]; else {split($2,b,"."); print x b[1]}}' >> "$GITHUB_OUTPUT" | ||
| grep resolver stack.yaml | awk '{print "resolver_long="$2}' >> "$GITHUB_OUTPUT" | ||
| # Cache ~/.stack, keyed by the contents of 'stack.yaml'. | ||
| - uses: actions/cache@v4 | ||
| name: cache ~/.stack (unix) | ||
| if: runner.os != 'Windows' | ||
| with: | ||
| path: ~/.stack | ||
| key: stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-${{hashFiles('**/stack.yaml')}}-${{github.sha}} | ||
| # Fall-back to use the most recent cache for the stack.yaml, or failing that the OS | ||
| restore-keys: | | ||
| stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-${{hashFiles('**/stack.yaml')}}- | ||
| stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}- | ||
| stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}- | ||
| stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}. | ||
| stack-1_${{matrix.os}}- | ||
| # Cache ~/.stack, keyed by the contents of 'stack.yaml'. | ||
| - uses: actions/cache@v4 | ||
| name: cache ~/.stack (Windows) | ||
| if: runner.os == 'Windows' | ||
| with: | ||
| save-always: true | ||
| path: | | ||
| C:\Users\runneradmin\AppData\Roaming\stack | ||
| C:\Users\runneradmin\AppData\Local\Programs\stack | ||
| key: stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-${{hashFiles('**/stack.yaml')}}-${{github.sha}} | ||
| # Fall-back to use the most recent cache for the stack.yaml, or failing that the OS | ||
| restore-keys: | | ||
| stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-${{hashFiles('**/stack.yaml')}}- | ||
| stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}- | ||
| stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}- | ||
| stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}. | ||
| stack-1_${{matrix.os}}- | ||
| # Cache each local package's ~/.stack-work for fast incremental builds in CI. | ||
| - uses: actions/cache@v4 | ||
| name: cache .stack-work | ||
| with: | ||
| save-always: true | ||
| path: | | ||
| **/.stack-work | ||
| # Main cache key: commit hash. This should always result in a cache miss... | ||
| # So when loading a cache we'll always fall back to the restore-keys, | ||
| # which should load the most recent cache via a prefix search on the most | ||
| # recent branch cache. | ||
| # Then it will save a new cache at this commit sha, which should be used by | ||
| # the next build on this branch. | ||
| key: stack-work-4_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}-${{hashFiles('**/stack.yaml')}}-${{github.sha}} | ||
| restore-keys: | | ||
| stack-work-4_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-${{hashFiles('**/stack.yaml')}}- | ||
| stack-work-4_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}- | ||
| stack-work-4_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}- | ||
| stack-work-4_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}. | ||
| stack-work-4_${{matrix.os}}- | ||
| # Install stack by downloading the binary from GitHub. | ||
| # The installation process differs by OS. | ||
| - name: install stack (Linux) | ||
| if: runner.os == 'Linux' | ||
| working-directory: ${{ runner.temp }} | ||
| run: | | ||
| mkdir stack && cd stack | ||
| curl -L https://github.com/commercialhaskell/stack/releases/download/v2.9.1/stack-2.9.1-linux-x86_64.tar.gz | tar -xz | ||
| echo "$PWD/stack-"* >> $GITHUB_PATH | ||
| - name: install stack (macOS) | ||
| working-directory: ${{ runner.temp }} | ||
| if: runner.os == 'macOS' | ||
| run: | | ||
| mkdir stack && cd stack | ||
| curl -L https://github.com/commercialhaskell/stack/releases/download/v2.9.1/stack-2.9.1-osx-x86_64.tar.gz | tar -xz | ||
| echo "$PWD/stack-"* >> $GITHUB_PATH | ||
| - name: install stack (windows) | ||
| working-directory: ${{ runner.temp }} | ||
| if: runner.os == 'Windows' | ||
| run: | | ||
| mkdir stack && cd stack | ||
| curl -L https://github.com/commercialhaskell/stack/releases/download/v2.9.1/stack-2.9.1-windows-x86_64.tar.gz | tar -xz | ||
| echo "$PWD/stack-"* >> $GITHUB_PATH | ||
| # One of the transcripts fails if the user's git name hasn't been set. | ||
| - name: set git user info | ||
| run: | | ||
| git config --global user.name "GitHub Actions" | ||
| git config --global user.email "[email protected]" | ||
| - name: remove ~/.stack/setup-exe-cache on macOS | ||
| if: runner.os == 'macOS' | ||
| run: rm -rf ~/.stack/setup-exe-cache | ||
| # Build deps, then build local code. Splitting it into two steps just allows us to see how much time each step | ||
| # takes. | ||
| - name: build dependencies | ||
| # Run up to 5 times in a row before giving up. | ||
| # It's very unlikely that our build-dependencies step will fail on most builds, | ||
| # so if it fails its almost certainly due to a race condition on the Windows | ||
| # file-system API that stack runs into. Since any successful packages are | ||
| # cached within a single build, it should get further along on each re-start | ||
| # and should hopefully finish! | ||
| run: | | ||
| stack --version | ||
| tries=1 | ||
| if [[ ${{matrix.os}} = "windows-"* ]]; then | ||
| tries=5 | ||
| fi | ||
| for (( i = 0; i < $tries; i++ )); do | ||
| stack --no-terminal build --fast --only-dependencies && break; | ||
| done | ||
| - name: build | ||
| run: | | ||
| stack build \ | ||
| --fast \ | ||
| --test \ | ||
| --no-run-tests \ | ||
| --no-terminal \ | ||
| --local-bin-path ${{env.ucm_local_bin}} \ | ||
| --copy-bins | ||
| - name: output ucm path (Windows only) | ||
| id: ucm-path | ||
| if: runner.os == 'Windows' | ||
| run: echo "ucm-path=$(stack exec -- where unison)" >> $GITHUB_OUTPUT | ||
| - name: output ucm path (non-Windows) | ||
| id: ucm-path | ||
| if: runner.os != 'Windows' | ||
| run: echo "ucm-path=$(stack exec -- which unison)" >> $GITHUB_OUTPUT | ||
| # Run each test suite (tests and transcripts) | ||
| # - name: check disk space before | ||
| # if: ${{ always() }} | ||
| # run: df -h | ||
| # - name: unison-cli test | ||
| # run: stack --no-terminal build --fast --test unison-cli | ||
| # - name: check disk space after | ||
| # if: ${{ always() }} | ||
| # run: df -h | ||
| # - name: unison-core tests | ||
| # run: stack --no-terminal build --fast --test unison-core | ||
| # - name: unison-parser-typechecker tests | ||
| # run: stack --no-terminal build --fast --test unison-parser-typechecker | ||
| # - name: unison-sqlite tests | ||
| # run: stack --no-terminal build --fast --test unison-sqlite | ||
| # - name: unison-syntax tests | ||
| # run: stack --no-terminal build --fast --test unison-syntax | ||
| # - name: unison-util-bytes tests | ||
| # run: stack --no-terminal build --fast --test unison-util-bytes | ||
| # - name: unison-util-cache tests | ||
| # run: stack --no-terminal build --fast --test unison-util-cache | ||
| # - name: unison-util-relation tests | ||
| # run: stack --no-terminal build --fast --test unison-util-relation | ||
| # - name: round-trip-tests | ||
| # run: | | ||
| # stack --no-terminal exec unison transcript unison-src/transcripts-round-trip/main.md | ||
| # git add unison-src/transcripts-round-trip/main.output.md | ||
| # # Fail if any transcripts cause git diffs. | ||
| # git diff --cached --ignore-cr-at-eol --exit-code | ||
| # stack --no-terminal exec unison transcript unison-src/transcripts-manual/rewrites.md | ||
| # git add unison-src/transcripts-manual/rewrites.output.md | ||
| # # Fail if any transcripts cause git diffs. | ||
| # git diff --cached --ignore-cr-at-eol --exit-code | ||
| # - name: transcripts | ||
| # run: | | ||
| # stack --no-terminal exec transcripts | ||
| # # Add all changes to the index for when we diff. | ||
| # git add --all | ||
| # # Fail if any transcripts cause git diffs. | ||
| # git diff --cached --ignore-cr-at-eol --exit-code | ||
| # - name: cli-integration-tests | ||
| # run: stack --no-terminal exec cli-integration-tests | ||
| # - name: verify stack ghci startup | ||
| # if: runner.os == 'macOS' | ||
| # run: echo | stack ghci | ||
| - name: debug check ucm artifact | ||
| run: | | ||
| ls -l ${{ env.ucm_local_bin }} | ||
| ls -l ${{ steps.ucm-path.outputs.ucm-path }} | ||
| - name: save ucm artifact | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: unison-${{ matrix.os }} | ||
| path: ${{ steps.ucm-path.outputs.ucm-path }} | ||
| if-no-files-found: error | ||
| - name: Setup tmate session | ||
| uses: mxschmitt/action-tmate@v3 | ||
| if: ${{ always() }} | ||
| generate-jit-source: | ||
| name: Generate JIT source | ||
| needs: build-ucm | ||
| runs-on: ubuntu-20.04 | ||
| steps: | ||
| - name: set up environment | ||
| run: | | ||
| echo "jit_src_scheme=${{ runner.temp }}/${{ env.jit_src_scheme }}" >> $GITHUB_ENV | ||
| echo "ucm=${{ runner.temp }}/unison-${{ matrix.os }}" >> $GITHUB_ENV | ||
| - uses: actions/cache@v4 | ||
| name: cache jit source | ||
| if: runner.os == 'Linux' | ||
| with: | ||
| path: ${{ env.jit_src_scheme }} | ||
| key: jit_src_scheme.racket_${{env.racket_version}}.jit_${{env.jit_src_scheme}}.${{github.sha}} | ||
| restore-keys: jit_src_scheme.racket_${{env.racket_version}}.jit_${{env.jit_src_scheme}}. | ||
| - name: check source exists | ||
| id: jit_src_exists | ||
| uses: thebinaryfelix/check-file-existence-action@v1 | ||
| with: | ||
| files: "${{ env.jit_src_scheme }}/unison/{data-info,boot-generated,simple-wrappers,builtin-generated,compound-wrappers}.ss" | ||
| - name: create transcript | ||
| if: steps.jit_src_exists.outputs.exists == 'false' | ||
| uses: 1arp/[email protected] | ||
| with: | ||
| file: 'setup-jit.md' | ||
| content: | | ||
| ```ucm | ||
| .> project.create-empty jit-setup | ||
| jit-setup/main> pull ${{ env.jit_version }} lib.jit | ||
| ``` | ||
| ```unison | ||
| generateSchemeBoot dir = do | ||
| saveDataInfoFile dir dataInfos | ||
| saveBaseFile dir bootSpec | ||
| saveWrapperFile dir simpleWrapperSpec | ||
| saveBaseFile dir builtinSpec | ||
| saveWrapperFile dir compoundWrapperSpec | ||
| go = generateSchemeBoot "${{ env.jit_src_scheme }}" | ||
| ``` | ||
| ```ucm | ||
| jit-setup/main> run go | ||
| ``` | ||
| - name: download ucm artifact | ||
| if: steps.jit_src_exists.outputs.exists == 'false' | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: unison-${{ env.jit_generator_os }} | ||
| path: ${{ runner.temp }} | ||
| - name: generate source | ||
| if: steps.jit_src_exists.outputs.exists == 'false' | ||
| run: | | ||
| cp -R scheme-libs/racket/* ${{ env.jit_src_scheme }} | ||
| chmod +x ${{ env.ucm }} | ||
| ${{ env.ucm }} transcript setup-jit.md | ||
| - name: save jit source | ||
| if: steps.jit_src_exists.outputs.exists == 'false' | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: jit-source | ||
| path: ${{ env.jit_src_scheme }}/** | ||
| build-jit-binary: | ||
| name: Build JIT binary ${{ matrix.os }} | ||
| needs: [build-ucm, generate-jit-source] | ||
| runs-on: ${{ matrix.os }} | ||
| strategy: | ||
| # Run each build to completion, regardless of if any have failed | ||
| fail-fast: false | ||
| matrix: | ||
| os: | ||
| # While iterating on this file, you can disable one or more of these to speed things up | ||
| - ubuntu-20.04 | ||
| - macOS-12 | ||
| - windows-2019 | ||
| steps: | ||
| - name: set up environment | ||
| run: | | ||
| echo "jit_src_scheme=${{ runner.temp }}/${{ env.jit_src_scheme }}" >> $GITHUB_ENV | ||
| echo "jit_bin=${{ runner.temp }}/${{ env.jit_bin }}" >> $GITHUB_ENV | ||
| echo "ucm=${{ runner.temp }}/unison-${{ matrix.os }}" >> $GITHUB_ENV | ||
| - name: cache jit binaries | ||
| uses: actions/cache@v4 | ||
| with: | ||
| path: ${{ env.jit_bin }} | ||
| key: jit_bin.racket_${{ env.racket_version }}.jit_${{ env.jit_src_scheme }}.${{ github.sha }} | ||
| restore-keys: jit_bin.racket_${{ env.racket_version }}.jit_${{ env.jit_src_scheme }}. | ||
| - name: check binary exists | ||
| id: jit_bin_exists | ||
| uses: thebinaryfelix/check-file-existence-action@v1 | ||
| with: | ||
| files: "${{ env.jit_bin }}/unison-runtime" | ||
| - name: Cache Racket dependencies | ||
| if: steps.jit_bin_exists.outputs.exists == 'false' | ||
| uses: actions/cache@v4 | ||
| with: | ||
| path: | | ||
| ~/.cache/racket | ||
| ~/.local/share/racket | ||
| key: ${{ runner.os }}-racket-${{env.racket_version}} | ||
| - uses: Bogdanp/[email protected] | ||
| # if: steps.jit_bin_exists.outputs.exists == 'false' | ||
| with: | ||
| architecture: 'x64' | ||
| distribution: 'full' | ||
| variant: 'CS' | ||
| version: ${{env.racket_version}} | ||
| - uses: awalsh128/cache-apt-pkgs-action@latest | ||
| # read this if a package isn't installing correctly | ||
| # https://github.com/awalsh128/cache-apt-pkgs-action#caveats | ||
| with: | ||
| packages: libb2-dev | ||
| version: 1.0 # cache key version afaik | ||
| - name: download jit source | ||
| # if: steps.jit_src_exists.outputs.exists == 'false' | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: jit-source | ||
| path: ${{ env.jit_src_scheme }} | ||
| - uses: actions/checkout@v4 # checkout scheme-libs from unison repo | ||
| # if: steps.jit_src_exists.outputs.exists == 'false' | ||
| - name: download ucm | ||
| # if: steps.jit_bin_exists.outputs.exists == 'false' | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: unison-${{ matrix.os }} | ||
| path: ${{ runner.temp }} | ||
| - run: | | ||
| chmod +x ${{ env.ucm }} | ||
| ${{ env.ucm }} transcript setup-jit.md | ||
| - name: build jit binary | ||
| # if: steps.jit_bin_exists.outputs.exists == 'false' | ||
| run: | | ||
| cp -R scheme-libs/racket/* ${{ env.jit_src_scheme }} | ||
| raco pkg install --auto ${{ env.jit_src_scheme }}/unison | ||
| raco exe ${{ env.jit_src_scheme }}/unison-runtime.rkt | ||
| raco distribute ${{ env.jit_bin }} ${{ env.jit_src_scheme }} | ||
| - name: save jit binary | ||
| # if: steps.jit_bin_exists.outputs.exists == 'false' | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: jit-binary-${{ matrix.os }} | ||
| path: ${{ env.jit_bin }}/** | ||
| - name: jit tests ${{ matrix.os }} | ||
| # if: steps.jit_bin_exists.outputs.exists == 'false' | ||
| run: | | ||
| time ${{ env.ucm }} transcript unison-src/builtin-tests/jit-tests.md | ||
| cat unison-src/builtin-tests/jit-tests.output.md | ||
| CHANGE=$(git diff unison-src/builtin-tests/jit-tests.output.md) | ||
| if [ -n "$CHANGE" ]; then | ||
| echo "The jit-tests output has changed" | ||
| exit 1 | ||
| fi | ||
| - name: interpreter tests | ||
| # if: steps.jit_bin_exists.outputs.exists == 'false' | ||
| run: | | ||
| time ${{ env.ucm }} transcript unison-src/builtin-tests/interpreter-tests.md | ||
| cat unison-src/builtin-tests/interpreter-tests.output.md | ||
| CHANGE=$(git diff unison-src/builtin-tests/interpreter-tests.output.md) | ||
| if [ -n "$CHANGE" ]; then | ||
| echo "The interpreter-tests output has changed" | ||
| exit 1 | ||
| fi | ||
