Rework racket based runtime/compilation #7351

Workflow file for this run

	name: CI

	defaults:
	run:
	shell: bash

	on:
	# Build on every pull request (and new PR commit)
	pull_request:
	# Build on new pushes to trunk (E.g. Merge commits)
	# Without the branch filter, each commit on a branch with a PR is triggered twice.
	# See: https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662
	push:
	branches:
	- trunk
	tags:
	- release/*
	workflow_dispatch:

	env:
	ormolu_version: "0.5.0.1"
	racket_version: "8.7"
	jit_version: "@unison/internal/releases/0.0.10"
	jit_src_scheme: "unison-jit-src/scheme-libs/racket"
	jit_bin: "unison-jit-bin"
	jit_generator_os: ubuntu-20.04

	jobs:
	ormolu:
	runs-on: ubuntu-20.04
	# Only run formatting on trunk commits
	# This is because the job won't have permission to push back to
	# contributor forks on contributor PRs.
	if: github.ref_name == 'trunk'
	steps:
	- uses: actions/checkout@v4
	- name: Get changed files
	id: changed-files
	uses: tj-actions/changed-files@v41
	with:
	# globs copied from default settings for run-ormolu
	files: \|
	*/.hs
	*/.hs-boot
	separator: "\n"
	- uses: haskell-actions/run-ormolu@v14
	with:
	version: ${{ env.ormolu_version }}
	mode: inplace
	pattern: ${{ steps.changed-files.outputs.all_changed_files }}
	- name: apply formatting changes
	uses: stefanzweifel/git-auto-commit-action@v4
	if: ${{ always() }}
	with:
	commit_message: automatically run ormolu
	build-ucm:
	name: Build UCM ${{ matrix.os }}
	runs-on: ${{ matrix.os }}
	if: ${{ always() }}
	needs: ormolu
	strategy:
	# Run each build to completion, regardless of if any have failed
	fail-fast: false
	matrix:
	os:
	# While iterating on this file, you can disable one or more of these to speed things up
	- ubuntu-20.04
	- macOS-12
	- windows-2019

	steps:
	- uses: actions/checkout@v4
	# The number towards the beginning of the cache keys allow you to manually avoid using a previous cache.
	# GitHub will automatically delete caches that haven't been accessed in 7 days, but there is no way to
	# purge one manually.
	- id: stackage-resolver
	name: record stackage resolver
	# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#environment-files
	# looks for `resolver: nightly-yyyy-mm-dd` or `resolver: lts-xx.yy` in `stack.yaml` and splits it into
	# `nightly` or `lts-xx`. the whole resolver string is put into resolver_long as a backup cache key
	# ${{ steps.stackage-resolver.outputs.resolver_short }}
	# ${{ steps.stackage-resolver.outputs.resolver_long }}
	run: \|
	grep resolver stack.yaml \| awk '{ x="resolver_short="; if (split($2,a,"-") > 2) print x a[1]; else {split($2,b,"."); print x b[1]}}' >> "$GITHUB_OUTPUT"
	grep resolver stack.yaml \| awk '{print "resolver_long="$2}' >> "$GITHUB_OUTPUT"
	# Cache ~/.stack, keyed by the contents of 'stack.yaml'.
	- uses: actions/cache@v4
	name: cache ~/.stack (unix)
	if: runner.os != 'Windows'
	with:
	path: ~/.stack
	key: stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-${{hashFiles('**/stack.yaml')}}-${{github.sha}}
	# Fall-back to use the most recent cache for the stack.yaml, or failing that the OS
	restore-keys: \|
	stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-${{hashFiles('**/stack.yaml')}}-
	stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-
	stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}-
	stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}.
	stack-1_${{matrix.os}}-

	# Cache ~/.stack, keyed by the contents of 'stack.yaml'.
	- uses: actions/cache@v4
	name: cache ~/.stack (Windows)
	if: runner.os == 'Windows'
	with:
	save-always: true
	path: \|
	C:\Users\runneradmin\AppData\Roaming\stack
	C:\Users\runneradmin\AppData\Local\Programs\stack
	key: stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-${{hashFiles('**/stack.yaml')}}-${{github.sha}}
	# Fall-back to use the most recent cache for the stack.yaml, or failing that the OS
	restore-keys: \|
	stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-${{hashFiles('**/stack.yaml')}}-
	stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-
	stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}-
	stack-1_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}.
	stack-1_${{matrix.os}}-

	# Cache each local package's ~/.stack-work for fast incremental builds in CI.
	- uses: actions/cache@v4
	name: cache .stack-work
	with:
	save-always: true
	path: \|
	**/.stack-work
	# Main cache key: commit hash. This should always result in a cache miss...
	# So when loading a cache we'll always fall back to the restore-keys,
	# which should load the most recent cache via a prefix search on the most
	# recent branch cache.
	# Then it will save a new cache at this commit sha, which should be used by
	# the next build on this branch.
	key: stack-work-4_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}-${{hashFiles('**/stack.yaml')}}-${{github.sha}}
	restore-keys: \|
	stack-work-4_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-${{hashFiles('**/stack.yaml')}}-
	stack-work-4_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_long }}-
	stack-work-4_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}-
	stack-work-4_${{matrix.os}}-${{ steps.stackage-resolver.outputs.resolver_short }}.
	stack-work-4_${{matrix.os}}-

	# Install stack by downloading the binary from GitHub.
	# The installation process differs by OS.
	- name: install stack (Linux)
	if: runner.os == 'Linux'
	working-directory: ${{ runner.temp }}
	run: \|
	mkdir stack && cd stack
	curl -L https://github.com/commercialhaskell/stack/releases/download/v2.9.1/stack-2.9.1-linux-x86_64.tar.gz \| tar -xz
	echo "$PWD/stack-"* >> $GITHUB_PATH

	- name: install stack (macOS)
	working-directory: ${{ runner.temp }}
	if: runner.os == 'macOS'
	run: \|
	mkdir stack && cd stack
	curl -L https://github.com/commercialhaskell/stack/releases/download/v2.9.1/stack-2.9.1-osx-x86_64.tar.gz \| tar -xz
	echo "$PWD/stack-"* >> $GITHUB_PATH

	- name: install stack (windows)
	working-directory: ${{ runner.temp }}
	if: runner.os == 'Windows'
	run: \|
	mkdir stack && cd stack
	curl -L https://github.com/commercialhaskell/stack/releases/download/v2.9.1/stack-2.9.1-windows-x86_64.tar.gz \| tar -xz
	echo "$PWD/stack-"* >> $GITHUB_PATH

	# One of the transcripts fails if the user's git name hasn't been set.
	- name: set git user info
	run: \|
	git config --global user.name "GitHub Actions"
	git config --global user.email "[email protected]"

	- name: remove ~/.stack/setup-exe-cache on macOS
	if: runner.os == 'macOS'
	run: rm -rf ~/.stack/setup-exe-cache

	# Build deps, then build local code. Splitting it into two steps just allows us to see how much time each step
	# takes.
	- name: build dependencies
	# Run up to 5 times in a row before giving up.
	# It's very unlikely that our build-dependencies step will fail on most builds,
	# so if it fails its almost certainly due to a race condition on the Windows
	# file-system API that stack runs into. Since any successful packages are
	# cached within a single build, it should get further along on each re-start
	# and should hopefully finish!
	run: \|
	stack --version
	tries=1
	if [[ ${{matrix.os}} = "windows-"* ]]; then
	tries=5
	fi

	for (( i = 0; i < $tries; i++ )); do
	stack --no-terminal build --fast --only-dependencies && break;
	done
	- name: build
	run: stack --no-terminal build --fast --no-run-tests --test
	- name: output ucm path
	id: ucm-path
	run: \|
	echo "ucm-path=$(stack exec -- which unison)" >> $GITHUB_OUTPUT
	cp "$(stack exec -- which unison)" .

	# Run each test suite (tests and transcripts)
	# - name: check disk space before
	# if: ${{ always() }}
	# run: df -h
	# - name: unison-cli test
	# run: stack --no-terminal build --fast --test unison-cli
	# - name: check disk space after
	# if: ${{ always() }}
	# run: df -h
	# - name: unison-core tests
	# run: stack --no-terminal build --fast --test unison-core
	# - name: unison-parser-typechecker tests
	# run: stack --no-terminal build --fast --test unison-parser-typechecker
	# - name: unison-sqlite tests
	# run: stack --no-terminal build --fast --test unison-sqlite
	# - name: unison-syntax tests
	# run: stack --no-terminal build --fast --test unison-syntax
	# - name: unison-util-bytes tests
	# run: stack --no-terminal build --fast --test unison-util-bytes
	# - name: unison-util-cache tests
	# run: stack --no-terminal build --fast --test unison-util-cache
	# - name: unison-util-relation tests
	# run: stack --no-terminal build --fast --test unison-util-relation
	# - name: round-trip-tests
	# run: \|
	# stack --no-terminal exec unison transcript unison-src/transcripts-round-trip/main.md
	# git add unison-src/transcripts-round-trip/main.output.md
	# # Fail if any transcripts cause git diffs.
	# git diff --cached --ignore-cr-at-eol --exit-code
	# stack --no-terminal exec unison transcript unison-src/transcripts-manual/rewrites.md
	# git add unison-src/transcripts-manual/rewrites.output.md
	# # Fail if any transcripts cause git diffs.
	# git diff --cached --ignore-cr-at-eol --exit-code
	# - name: transcripts
	# run: \|
	# stack --no-terminal exec transcripts
	# # Add all changes to the index for when we diff.
	# git add --all
	# # Fail if any transcripts cause git diffs.
	# git diff --cached --ignore-cr-at-eol --exit-code
	# - name: cli-integration-tests
	# run: stack --no-terminal exec cli-integration-tests
	# - name: verify stack ghci startup
	# if: runner.os == 'macOS'
	# run: echo \| stack ghci

	- name: save ucm artifact
	uses: actions/upload-artifact@v4
	with:
	name: unison-${{ matrix.os }}
	path: ${{ steps.ucm-path.outputs.ucm-path }}
	if-no-files-found: error

	# - name: Setup tmate session
	# uses: mxschmitt/action-tmate@v3
	# if: ${{ always() }}


	generate-jit-source:
	name: Generate JIT source
	needs: build-ucm
	runs-on: ubuntu-20.04
	steps:
	- name: set up environment
	run: \|
	echo "jit_src_scheme=${{ runner.temp }}/${{ env.jit_src_scheme }}" >> $GITHUB_ENV
	echo "ucm=${{ runner.temp }}/unison-${{ matrix.os }}" >> $GITHUB_ENV
	- uses: actions/cache@v4
	name: cache jit source
	if: runner.os == 'Linux'
	with:
	path: ${{ env.jit_src_scheme }}
	key: jit_src_scheme.racket_${{env.racket_version}}.jit_${{env.jit_src_scheme}}.${{github.sha}}
	restore-keys: jit_src_scheme.racket_${{env.racket_version}}.jit_${{env.jit_src_scheme}}.

	- name: check source exists
	id: jit_src_exists
	uses: thebinaryfelix/check-file-existence-action@v1
	with:
	files: "${{ env.jit_src_scheme }}/unison/{data-info,boot-generated,simple-wrappers,builtin-generated,compound-wrappers}.ss"

	- name: create transcript
	if: steps.jit_src_exists.outputs.exists == 'false'
	uses: 1arp/[email protected]
	with:
	file: 'setup-jit.md'
	content: \|
	```ucm
	.> project.create-empty jit-setup
	jit-setup/main> pull ${{ env.jit_version }} lib.jit
	```
	```unison
	generateSchemeBoot dir = do
	saveDataInfoFile dir dataInfos
	saveBaseFile dir bootSpec
	saveWrapperFile dir simpleWrapperSpec
	saveBaseFile dir builtinSpec
	saveWrapperFile dir compoundWrapperSpec

	go = generateSchemeBoot "${{ env.jit_src_scheme }}"
	```
	```ucm
	jit-setup/main> run go
	```

	- name: download ucm artifact
	if: steps.jit_src_exists.outputs.exists == 'false'
	uses: actions/download-artifact@v4
	with:
	name: unison-${{ env.jit_generator_os }}
	path: ${{ runner.temp }}

	- name: generate source
	if: steps.jit_src_exists.outputs.exists == 'false'
	run: \|
	cp -R scheme-libs/racket/* ${{ env.jit_src_scheme }}
	chmod +x ${{ env.ucm }}
	${{ env.ucm }} transcript setup-jit.md

	- name: save jit source
	if: steps.jit_src_exists.outputs.exists == 'false'
	uses: actions/upload-artifact@v4
	with:
	name: jit-source
	path: ${{ env.jit_src_scheme }}/**


	build-jit-binary:
	name: Build JIT binary ${{ matrix.os }}
	needs: [build-ucm, generate-jit-source]
	runs-on: ${{ matrix.os }}
	strategy:
	# Run each build to completion, regardless of if any have failed
	fail-fast: false
	matrix:
	os:
	# While iterating on this file, you can disable one or more of these to speed things up
	- ubuntu-20.04
	- macOS-12
	- windows-2019
	steps:
	- name: set up environment
	run: \|
	echo "jit_src_scheme=${{ runner.temp }}/${{ env.jit_src_scheme }}" >> $GITHUB_ENV
	echo "jit_bin=${{ runner.temp }}/${{ env.jit_bin }}" >> $GITHUB_ENV
	echo "ucm=${{ runner.temp }}/unison-${{ matrix.os }}" >> $GITHUB_ENV
	- name: cache jit binaries
	uses: actions/cache@v4
	with:
	path: ${{ env.jit_bin }}
	key: jit_bin.racket_${{ env.racket_version }}.jit_${{ env.jit_src_scheme }}.${{ github.sha }}
	restore-keys: jit_bin.racket_${{ env.racket_version }}.jit_${{ env.jit_src_scheme }}.

	- name: check binary exists
	id: jit_bin_exists
	uses: thebinaryfelix/check-file-existence-action@v1
	with:
	files: "${{ env.jit_bin }}/unison-runtime"

	- name: Cache Racket dependencies
	if: steps.jit_bin_exists.outputs.exists == 'false'
	uses: actions/cache@v4
	with:
	path: \|
	~/.cache/racket
	~/.local/share/racket
	key: ${{ runner.os }}-racket-${{env.racket_version}}
	- uses: Bogdanp/[email protected]
	# if: steps.jit_bin_exists.outputs.exists == 'false'
	with:
	architecture: 'x64'
	distribution: 'full'
	variant: 'CS'
	version: ${{env.racket_version}}
	- uses: awalsh128/cache-apt-pkgs-action@latest
	# read this if a package isn't installing correctly
	# https://github.com/awalsh128/cache-apt-pkgs-action#caveats
	with:
	packages: libb2-dev
	version: 1.0 # cache key version afaik
	- name: download jit source
	# if: steps.jit_src_exists.outputs.exists == 'false'
	uses: actions/download-artifact@v4
	with:
	name: jit-source
	path: ${{ env.jit_src_scheme }}

	- uses: actions/checkout@v4 # checkout scheme-libs from unison repo
	# if: steps.jit_src_exists.outputs.exists == 'false'

	- name: download ucm
	# if: steps.jit_bin_exists.outputs.exists == 'false'
	uses: actions/download-artifact@v4
	with:
	name: unison-${{ matrix.os }}
	path: ${{ runner.temp }}
	- run: \|
	chmod +x ${{ env.ucm }}
	${{ env.ucm }} transcript setup-jit.md

	- name: build jit binary
	# if: steps.jit_bin_exists.outputs.exists == 'false'
	run: \|
	cp -R scheme-libs/racket/* ${{ env.jit_src_scheme }}
	raco pkg install --auto ${{ env.jit_src_scheme }}/unison
	raco exe ${{ env.jit_src_scheme }}/unison-runtime.rkt
	raco distribute ${{ env.jit_bin }} ${{ env.jit_src_scheme }}

	- name: save jit binary
	# if: steps.jit_bin_exists.outputs.exists == 'false'
	uses: actions/upload-artifact@v4
	with:
	name: jit-binary-${{ matrix.os }}
	path: ${{ env.jit_bin }}/**

	- name: jit tests ${{ matrix.os }}
	# if: steps.jit_bin_exists.outputs.exists == 'false'
	run: \|
	time ${{ env.ucm }} transcript unison-src/builtin-tests/jit-tests.md
	cat unison-src/builtin-tests/jit-tests.output.md
	CHANGE=$(git diff unison-src/builtin-tests/jit-tests.output.md)
	if [ -n "$CHANGE" ]; then
	echo "The jit-tests output has changed"
	exit 1
	fi

	- name: interpreter tests
	# if: steps.jit_bin_exists.outputs.exists == 'false'
	run: \|
	time ${{ env.ucm }} transcript unison-src/builtin-tests/interpreter-tests.md
	cat unison-src/builtin-tests/interpreter-tests.output.md
	CHANGE=$(git diff unison-src/builtin-tests/interpreter-tests.output.md)
	if [ -n "$CHANGE" ]; then
	echo "The interpreter-tests output has changed"
	exit 1
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Rework racket based runtime/compilation #7351

Workflow file

Rework racket based runtime/compilation #7351

Jobs

Run details

Workflow file for this run