Benchmark CML #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark CML | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| git-ref: | |
| description: Repo reference (branch, tag or SHA) | |
| default: "main" | |
| required: true | |
| type: string | |
| list: | |
| description: Which kind of list to consider | |
| default: "short" | |
| type: choice | |
| options: | |
| - "long" | |
| - "short" | |
| classification: | |
| description: Launch classification benchmarks | |
| default: true | |
| type: boolean | |
| regression: | |
| description: Launch regression benchmarks | |
| default: true | |
| type: boolean | |
| glm: | |
| description: Launch glm benchmarks | |
| default: true | |
| type: boolean | |
| deep-learning: | |
| description: Launch deep learning benchmarks | |
| default: true | |
| type: boolean | |
| fhe-samples: | |
| description: Number of FHE samples | |
| required: false | |
| default: 100 | |
| type: number | |
| model: | |
| description: The model to run | |
| required: false | |
| type: string | |
| # Add recurrent launching | |
| # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/1851 | |
| # Global environnement variables | |
| env: | |
| # Github action url (used by slack notification) | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache | |
| RUNNER_TOOL_CACHE: /opt/hostedtoolcache | |
| # Jobs | |
| jobs: | |
| prepare-scripts: | |
| name: Prepare scripts to launch on individual AWS instances | |
| runs-on: ubuntu-20.04 | |
| defaults: | |
| run: | |
| shell: bash | |
| container: | |
| image: ubuntu:20.04 | |
| outputs: | |
| commands: ${{ steps.prepare-commands.outputs.commands }} | |
| length: ${{ steps.prepare-commands.outputs.length }} | |
| env: | |
| PIP_INDEX_URL: ${{ secrets.PIP_INDEX_URL }} | |
| PIP_EXTRA_INDEX_URL: ${{ secrets.PIP_EXTRA_INDEX_URL }} | |
| steps: | |
| - name: Add masks | |
| run: | | |
| echo "::add-mask::${{ secrets.INTERNAL_PYPI_URL_FOR_MASK }}" | |
| echo "::add-mask::${{ secrets.INTERNAL_REPO_URL_FOR_MASK }}" | |
| echo "::add-mask::${{ secrets.INTERNAL_PYPI_URL }}" | |
| echo "::add-mask::${{ secrets.INTERNAL_REPO_URL }}" | |
| - name: Docker container related setup and git installation | |
| id: docker-git-config | |
| run: | | |
| TZ=Europe/Paris | |
| echo "TZ=${TZ}" >> "$GITHUB_ENV" | |
| ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone | |
| sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list | |
| apt update && apt install git git-lfs -y | |
| apt -y install sudo | |
| # We don't need to specify the git-ref here since he are just generating the cli arguments | |
| - name: Checkout code | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 | |
| - name: Set up Python | |
| uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c | |
| with: | |
| python-version: "3.9" | |
| - name: Install dependencies | |
| id: install-deps | |
| run: | | |
| apt update | |
| apt install --no-install-recommends -y gnome-keyring | |
| apt install -y graphviz* graphviz-dev libgraphviz-dev pkg-config python3-dev | |
| apt-mark hold docker.io | |
| ./script/make_utils/setup_os_deps.sh | |
| make setup_env | |
| - name: Prepare scripts to launch on individual AWS instances | |
| id: prepare-commands | |
| run: | | |
| source .venv/bin/activate | |
| python3 ./script/actions_utils/generate_scripts_benchmark.py --list_length ${{ github.event.inputs.list }} --classification ${{ github.event.inputs.classification }} --regression ${{ github.event.inputs.regression }} --glm ${{ github.event.inputs.glm }} --deep_learning ${{ github.event.inputs.deep-learning }} --fhe_samples ${{ github.event.inputs.fhe-samples }} --model ${{ github.event.inputs.model }} > commands.json | |
| # Needs to be done before populating COMMANDS otherwise it will crash | |
| LENGTH=$(jq length commands.json) | |
| COMMANDS=$(<commands.json) | |
| export COMMANDS | |
| # Echo for debug purposes | |
| echo "${COMMANDS}" | |
| echo "${LENGTH}" | |
| # Set output | |
| echo "commands=${COMMANDS}" >> $GITHUB_OUTPUT | |
| echo "length=${LENGTH}" >> $GITHUB_OUTPUT | |
| run-job: | |
| needs: [prepare-scripts] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| # To avoid stopping all workflows if one fails | |
| fail-fast: false | |
| matrix: | |
| index: ${{ fromJson(needs.prepare-scripts.outputs.commands) }} | |
| max-parallel: ${{ fromJson(needs.prepare-scripts.outputs.length) }} | |
| steps: | |
| - name: Checkout Slab repo | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 | |
| with: | |
| repository: zama-ai/slab | |
| path: slab | |
| token: ${{ secrets.BOT_TOKEN }} | |
| - name: Start AWS job in Slab | |
| shell: bash | |
| run: | | |
| USER_COMMANDS=$(echo -n '${{ toJSON(matrix.index.commands) }}' | jq -rcM | tr -d "\n") | |
| USER_INPUTS="$(echo -n "{\"git-ref\": \"${{ inputs.git-ref}}\", \"commands\": \"${USER_COMMANDS}\"}" | sed 's|\([^\\]\)"|\1\\\\\\"|g')" | |
| echo -n {\"command\": \"bench\", \"git_ref\": \"${{ github.ref }}\", \"sha\": \"${{ github.sha }}\", \"user_inputs\": \"${USER_INPUTS}\"} > command.json | |
| SIGNATURE="$(slab/scripts/hmac_calculator.sh command.json '${{ secrets.JOB_SECRET }}')" | |
| curl -v -k \ | |
| -H "Content-Type: application/json" \ | |
| -H "X-Slab-Repository: ${{ github.repository }}" \ | |
| -H "X-Slab-Command: start_aws" \ | |
| -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ | |
| -d @command.json \ | |
| ${{ secrets.SLAB_URL }} | |
| slack-notification: | |
| runs-on: ubuntu-20.04 | |
| needs: [run-job] | |
| steps: | |
| - name: Slack Notification | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@b24d75fe0e728a4bf9fc42ee217caa686d141ee8 | |
| env: | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png | |
| SLACK_COLOR: ${{ needs.run-job.result }} | |
| SLACK_MESSAGE: "Benchmark action: (${{ env.ACTION_RUN_URL }}) ended with result: ${{ needs.run-job.result }}" | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} |