Compiler - Build and Test (CPU) #1627
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compiler - Build and Test (CPU) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| instance_id: | |
| description: 'Instance ID' | |
| type: string | |
| instance_image_id: | |
| description: 'Instance AMI ID' | |
| type: string | |
| instance_type: | |
| description: 'Instance product type' | |
| type: string | |
| runner_name: | |
| description: 'Action runner name' | |
| type: string | |
| request_id: | |
| description: 'Slab request ID' | |
| type: string | |
| matrix_item: | |
| description: 'Build matrix item' | |
| type: string | |
| # concurrency: | |
| # group: compiler_build_and_test_cpu-${{ github.ref }} | |
| # cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| env: | |
| DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler | |
| GLIB_VER: 2_28 | |
| jobs: | |
| BuildAndTest: | |
| name: Build and test compiler in EC2 | |
| runs-on: ${{ github.event.inputs.runner_name }} | |
| if: ${{ !cancelled() }} | |
| steps: | |
| - name: Instance configuration used | |
| run: | | |
| echo "IDs: ${{ inputs.instance_id }}" | |
| echo "AMI: ${{ inputs.instance_image_id }}" | |
| echo "Type: ${{ inputs.instance_type }}" | |
| echo "Request ID: ${{ inputs.request_id }}" | |
| echo "Matrix item: ${{ inputs.matrix_item }}" | |
| # A SSH private key is required as some dependencies are from private repos | |
| - name: Set up SSH agent | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} | |
| - name: Set up env | |
| run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" | |
| echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" | |
| - name: Fetch repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} | |
| - name: Setup rust toolchain for concrete-cpu | |
| uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu | |
| - name: Create build dir | |
| run: mkdir build | |
| - name: Build compiler | |
| uses: addnab/docker-run-action@v3 | |
| id: build-compiler | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| -v ${{ github.workspace }}/wheels:/wheels | |
| -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket | |
| -e SSH_AUTH_SOCK=/ssh.socket | |
| ${{ env.DOCKER_GPU_OPTION }} | |
| shell: bash | |
| run: | | |
| rustup toolchain install nightly-2024-01-31 | |
| set -e | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| rm -rf /build/* | |
| make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all build-end-to-end-dataflow-tests | |
| echo "Debug: ccache statistics (after the build):" | |
| ccache -s | |
| - name: Build compiler Dialects docs and check diff | |
| uses: addnab/docker-run-action@v3 | |
| id: build-compiler-docs | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| -v ${{ github.workspace }}/wheels:/wheels | |
| -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket | |
| -e SSH_AUTH_SOCK=/ssh.socket | |
| ${{ env.DOCKER_GPU_OPTION }} | |
| shell: bash | |
| run: | | |
| set -e | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| make BUILD_DIR=/build doc | |
| cd /build/tools/concretelang/docs/concretelang/ | |
| sed -i -e 's/\[TOC\]//' *Dialect.md | |
| for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done; | |
| - name: Enable complete tests on push to main | |
| if: github.ref == 'refs/heads/main' | |
| run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV | |
| - name: Enable minimal tests otherwise | |
| if: github.ref != 'refs/heads/main' | |
| run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV | |
| - name: Test compiler | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| ${{ env.DOCKER_GPU_OPTION }} | |
| shell: bash | |
| run: | | |
| set -e | |
| rustup toolchain install nightly-2024-01-31 | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| pip install pytest | |
| dnf install -y libzstd libzstd-devel | |
| sed "s/pytest/python -m pytest/g" -i Makefile | |
| mkdir -p /tmp/concrete_compiler/gpu_tests/ | |
| make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests run-end-to-end-dataflow-tests | |
| chmod -R ugo+rwx /tmp/KeySetCache | |
| - name: Analyze logs | |
| run: | | |
| cd build/gtest-parallel-logs/passed | |
| ls -1 | xargs grep -H "WARNING RETRY" | sed -e "s/.log.*//g" | uniq -c | sed -re "s/ *([0-9]*) (.*)/::warning ::Test \2 retried \1 times/g" | cat | |
| # - name: Archive python package | |
| # uses: actions/upload-artifact@v3 | |
| # with: | |
| # name: concrete-compiler.whl | |
| # path: build/wheels/concrete_compiler-*-manylinux_{{ env.GLIB_VER }}_x86_64.whl | |
| # retention-days: 14 |