benchmarks.yaml

name: Benchmark CML

on:
  workflow_dispatch:
    inputs:
      git-ref:
        description: Repo reference (branch, tag or SHA)
        default: "main"
        required: true
        type: string
      list:
        description: Which kind of list to consider
        default: "short"
        type: choice
        options:
          - "long"
          - "short"
      classification:
        description: Launch classification benchmarks
        default: true
        type: boolean
      regression:
        description: Launch regression benchmarks
        default: true
        type: boolean
      glm:
        description: Launch glm benchmarks
        default: true
        type: boolean
      deep-learning:
        description: Launch deep learning benchmarks
        default: true
        type: boolean 
      fhe-samples:
        description: Number of FHE samples
        required: false
        default: 100
        type: number
      model:
        description: The model to run
        required: false
        type: string

  # Add recurrent launching
  # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/1851

# Global environnement variables
env:
  # Github action url (used by slack notification)
  ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
  AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache
  RUNNER_TOOL_CACHE: /opt/hostedtoolcache

# Jobs
jobs:
  prepare-scripts:
    name: Prepare scripts to launch on individual AWS instances
    runs-on: ubuntu-20.04
    defaults:
      run:
        shell: bash
    container:
      image: ubuntu:20.04
    outputs:
      commands: ${{ steps.prepare-commands.outputs.commands }}
      length: ${{ steps.prepare-commands.outputs.length }}
    env:
      PIP_INDEX_URL: ${{ secrets.PIP_INDEX_URL }}
      PIP_EXTRA_INDEX_URL: ${{ secrets.PIP_EXTRA_INDEX_URL }}
    steps:
      - name: Add masks
        run: |
          echo "::add-mask::${{ secrets.INTERNAL_PYPI_URL_FOR_MASK }}"
          echo "::add-mask::${{ secrets.INTERNAL_REPO_URL_FOR_MASK }}"
          echo "::add-mask::${{ secrets.INTERNAL_PYPI_URL }}"
          echo "::add-mask::${{ secrets.INTERNAL_REPO_URL }}"
      - name: Docker container related setup and git installation
        id: docker-git-config
        run: |
          TZ=Europe/Paris
          echo "TZ=${TZ}" >> "$GITHUB_ENV"
          ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone
          sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list
          apt update && apt install git git-lfs -y
          apt -y install sudo

      # We don't need to specify the git-ref here since he are just generating the cli arguments 
      - name: Checkout code
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11

      - name: Set up Python
        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
        with:
          python-version: "3.9"

      - name: Install dependencies
        id: install-deps
        run: |
          apt update
          apt install --no-install-recommends -y gnome-keyring
          apt install -y graphviz* graphviz-dev libgraphviz-dev pkg-config python3-dev
          apt-mark hold docker.io
          ./script/make_utils/setup_os_deps.sh
          make setup_env

      - name: Prepare scripts to launch on individual AWS instances
        id: prepare-commands
        run: |
          source .venv/bin/activate
          python3 ./script/actions_utils/generate_scripts_benchmark.py --list_length ${{ github.event.inputs.list }} --classification ${{ github.event.inputs.classification }} --regression ${{ github.event.inputs.regression }} --glm ${{ github.event.inputs.glm }} --deep_learning ${{ github.event.inputs.deep-learning }} --fhe_samples ${{ github.event.inputs.fhe-samples }} --model ${{ github.event.inputs.model }} > commands.json
          # Needs to be done before populating COMMANDS otherwise it will crash
          LENGTH=$(jq length commands.json)
          COMMANDS=$(<commands.json)
          export COMMANDS
          # Echo for debug purposes
          echo "${COMMANDS}"
          echo "${LENGTH}"
          # Set output
          echo "commands=${COMMANDS}" >> $GITHUB_OUTPUT
          echo "length=${LENGTH}" >> $GITHUB_OUTPUT

  run-job:
    needs: [prepare-scripts]
    runs-on: ubuntu-latest
    strategy:
      # To avoid stopping all workflows if one fails
      fail-fast: false
      matrix:
        index: ${{ fromJson(needs.prepare-scripts.outputs.commands) }}
      max-parallel: ${{ fromJson(needs.prepare-scripts.outputs.length) }}
    steps:
      - name: Checkout Slab repo
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
        with:
          repository: zama-ai/slab
          path: slab
          token: ${{ secrets.BOT_TOKEN }}

      - name: Start AWS job in Slab
        shell: bash
        run: |
          USER_COMMANDS=$(echo -n '${{ toJSON(matrix.index.commands) }}' | jq -rcM | tr -d "\n")
          USER_INPUTS="$(echo -n "{\"git-ref\": \"${{ inputs.git-ref}}\", \"commands\": \"${USER_COMMANDS}\"}" | sed 's|\([^\\]\)"|\1\\\\\\"|g')"
          echo -n {\"command\": \"bench\", \"git_ref\": \"${{ github.ref }}\", \"sha\": \"${{ github.sha }}\", \"user_inputs\": \"${USER_INPUTS}\"} > command.json
          SIGNATURE="$(slab/scripts/hmac_calculator.sh command.json '${{ secrets.JOB_SECRET }}')"
          curl -v -k \
          -H "Content-Type: application/json" \
          -H "X-Slab-Repository: ${{ github.repository }}" \
          -H "X-Slab-Command: start_aws" \
          -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \
          -d @command.json \
          ${{ secrets.SLAB_URL }}

  slack-notification:
    runs-on: ubuntu-20.04
    needs: [run-job]
    steps:
      - name: Slack Notification
        if: ${{ always() }}
        continue-on-error: true
        uses: rtCamp/action-slack-notify@b24d75fe0e728a4bf9fc42ee217caa686d141ee8
        env:
          SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
          SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
          SLACK_COLOR: ${{ needs.run-job.result }}
          SLACK_MESSAGE: "Benchmark action: (${{ env.ACTION_RUN_URL }}) ended with result: ${{ needs.run-job.result }}"
          SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}