Compiler - Performance benchmarks #599
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run benchmarks on an AWS instance for compiler and return parsed results to Slab CI bot. | |
| name: Compiler - Performance benchmarks | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| instance_id: | |
| description: 'Instance ID' | |
| type: string | |
| instance_image_id: | |
| description: 'Instance AMI ID' | |
| type: string | |
| instance_type: | |
| description: 'Instance product type' | |
| type: string | |
| runner_name: | |
| description: 'Action runner name' | |
| type: string | |
| request_id: | |
| description: 'Slab request ID' | |
| type: string | |
| # concurrency: | |
| # group: ${{ github.workflow }}-${{ github.ref }} | |
| # cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json | |
| CUDA_PATH: /usr/local/cuda-11.8 | |
| GCC_VERSION: 8 | |
| jobs: | |
| run-benchmarks: | |
| name: Execute end-to-end benchmarks in EC2 | |
| runs-on: ${{ github.event.inputs.runner_name }} | |
| if: ${{ !cancelled() }} | |
| steps: | |
| - name: Instance configuration used | |
| run: | | |
| echo "IDs: ${{ inputs.instance_id }}" | |
| echo "AMI: ${{ inputs.instance_image_id }}" | |
| echo "Type: ${{ inputs.instance_type }}" | |
| echo "Request ID: ${{ inputs.request_id }}" | |
| - name: Get benchmark date | |
| run: | | |
| echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" | |
| # SSH private key is required as some dependencies are from private repos | |
| - uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} | |
| - name: Fetch submodules | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} | |
| - name: Set up home | |
| # "Install rust" step require root user to have a HOME directory which is not set. | |
| run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| - name: Export specific variables (CPU) | |
| if: ${{ !startswith(inputs.instance_type, 'p3.') }} | |
| run: | | |
| echo "CUDA_SUPPORT=OFF" >> "${GITHUB_ENV}" | |
| echo "BENCHMARK_TARGET=run-cpu-benchmarks" >> "${GITHUB_ENV}" | |
| - name: Export specific variables (GPU) | |
| if: ${{ startswith(inputs.instance_type, 'p3.') }} | |
| run: | | |
| echo "CUDA_SUPPORT=ON" >> "${GITHUB_ENV}" | |
| echo "BENCHMARK_TARGET=run-gpu-benchmarks" >> "${GITHUB_ENV}" | |
| echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}" | |
| echo "$CUDA_PATH/bin" >> "${GITHUB_PATH}" | |
| echo "LD_LIBRARY_PATH=$CUDA_PATH/lib:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}" | |
| echo "CC=/usr/bin/gcc-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" | |
| echo "CXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" | |
| echo "CUDAHOSTCXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" | |
| echo "CUDACXX=$CUDA_PATH/bin/nvcc" >> "${GITHUB_ENV}" | |
| - name: Setup rust toolchain for concrete-cpu | |
| uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu | |
| - name: Build compiler benchmarks | |
| run: | | |
| set -e | |
| git config --global --add safe.directory '*' | |
| cd compilers/concrete-compiler/compiler | |
| make BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} build-benchmarks | |
| - name: Run end-to-end benchmarks | |
| run: | | |
| set -e | |
| cd compilers/concrete-compiler/compiler | |
| make ${{ env.BENCHMARK_TARGET }} | |
| - name: Upload raw results artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: compiler_${{ github.sha }}_raw | |
| path: compilers/concrete-compiler/compiler/benchmarks_results.json | |
| - name: Parse results | |
| shell: bash | |
| run: | | |
| COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" | |
| COMMIT_HASH="$(git describe --tags --dirty)" | |
| python3 ./ci/benchmark_parser.py compilers/concrete-compiler/compiler/benchmarks_results.json ${{ env.RESULTS_FILENAME }} \ | |
| --database compiler_benchmarks \ | |
| --hardware ${{ inputs.instance_type }} \ | |
| --project-version ${COMMIT_HASH} \ | |
| --branch ${{ github.ref_name }} \ | |
| --commit-date ${COMMIT_DATE} \ | |
| --bench-date "${{ env.BENCH_DATE }}" \ | |
| --throughput | |
| - name: Upload parsed results artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: compiler_${{ github.sha }} | |
| path: ${{ env.RESULTS_FILENAME }} | |
| - name: Checkout Slab repo | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: zama-ai/slab | |
| path: slab | |
| token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} | |
| - name: Send data to Slab | |
| shell: bash | |
| run: | | |
| echo "Computing HMac on downloaded artifact" | |
| SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" | |
| echo "Sending results to Slab..." | |
| curl -v -k \ | |
| -H "Content-Type: application/json" \ | |
| -H "X-Slab-Repository: ${{ github.repository }}" \ | |
| -H "X-Slab-Command: store_data" \ | |
| -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ | |
| -d @${{ env.RESULTS_FILENAME }} \ | |
| ${{ secrets.SLAB_URL }} |