CIFAR-10 benchmark CML #88
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CIFAR-10 benchmark CML | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| git-ref: | |
| description: Repo reference (branch, tag or SHA) | |
| default: "main" | |
| required: true | |
| type: string | |
| alternative-cp-wheel-artifact-id: | |
| description: Alternative Concrete-Python Wheel Artifact-ID (see https://github.com/zama-ai/concrete/actions/workflows/concrete_python_release.yml) | |
| default: "none" | |
| required: true | |
| type: string | |
| alternative-cp-branch: | |
| description: Alternative Concrete-Python Branch | |
| default: "none" | |
| required: true | |
| type: string | |
| benchmark: | |
| description: Benchmark to run (cifar-10-8b or cifar-10-16b) | |
| default: "cifar-10-16b" | |
| type: choice | |
| options: | |
| - "cifar-10-8b" | |
| - "cifar-10-16b" | |
| num_samples: | |
| description: Number of samples to use | |
| default: "3" | |
| type: string | |
| required: true | |
| p_error: | |
| description: P-error to use | |
| default: "0.01" | |
| type: string | |
| required: true | |
| # FIXME: Add recurrent launching | |
| # https://github.com/zama-ai/concrete-ml-internal/issues/1851 | |
| # Global environnement variables | |
| env: | |
| # Github action url (used by slack notification) | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache | |
| RUNNER_TOOL_CACHE: /opt/hostedtoolcache | |
| SLAB_PROFILE: big-cpu | |
| # Jobs | |
| jobs: | |
| setup-ec2: | |
| name: Setup EC2 instance | |
| runs-on: ubuntu-latest | |
| outputs: | |
| runner-name: ${{ steps.start-instance.outputs.label }} | |
| steps: | |
| - name: Start instance | |
| id: start-instance | |
| uses: zama-ai/slab-github-runner@58f2cae4bf2c0b6728083f5f009b6dc0eb6dc3ac | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| backend: aws | |
| profile: ${{ env.SLAB_PROFILE }} | |
| run-cifar-10: | |
| needs: [setup-ec2] | |
| name: Run benchmark | |
| runs-on: ${{ needs.setup-ec2.outputs.runner-name }} | |
| env: | |
| PIP_INDEX_URL: ${{ secrets.PIP_INDEX_URL }} | |
| PIP_EXTRA_INDEX_URL: ${{ secrets.PIP_EXTRA_INDEX_URL }} | |
| steps: | |
| - name: Add masks | |
| run: | | |
| echo "::add-mask::${{ secrets.INTERNAL_PYPI_URL_FOR_MASK }}" | |
| echo "::add-mask::${{ secrets.INTERNAL_REPO_URL_FOR_MASK }}" | |
| echo "::add-mask::${{ secrets.INTERNAL_PYPI_URL }}" | |
| echo "::add-mask::${{ secrets.INTERNAL_REPO_URL }}" | |
| - name: Checkout code | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 | |
| with: | |
| lfs: true | |
| ref: ${{ github.event.inputs.git-ref }} | |
| - name: Set up Python | |
| uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d | |
| with: | |
| python-version: "3.8" | |
| - name: Install dependencies | |
| id: install-deps | |
| run: | | |
| apt update | |
| apt install --no-install-recommends -y gnome-keyring | |
| apt install -y graphviz* graphviz-dev libgraphviz-dev pkg-config python3-dev | |
| apt-mark hold docker.io | |
| ./script/make_utils/setup_os_deps.sh | |
| make setup_env | |
| - name: Alternative Concrete Python Wheel Download | |
| if: github.event.inputs.alternative-cp-wheel-artifact-id != 'none' | |
| run: | | |
| curl -L \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| -o concrete-python.whl.zip \ | |
| https://api.github.com/repos/zama-ai/concrete/actions/artifacts/${{ github.event.inputs.alternative-cp-wheel-artifact-id }}/zip | |
| - name: Alternative Concrete Python Wheel Install | |
| if: github.event.inputs.alternative-cp-wheel-artifact-id != 'none' | |
| run: | | |
| source .venv/bin/activate | |
| unzip concrete-python.whl.zip | |
| pip install concrete_python-*.whl | |
| - name: Alternative Concrete Python Branch Checkout | |
| if: github.event.inputs.alternative-cp-branch != 'none' | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 | |
| with: | |
| path: concrete | |
| repository: zama-ai/concrete | |
| ref: ${{ github.event.inputs.alternative-cp-branch }} | |
| - name: Alternative Concrete Python Branch Source Install | |
| if: github.event.inputs.alternative-cp-branch != 'none' | |
| run: | | |
| cp -R concrete/frontends/concrete-python/concrete/* .venv/lib/python3.*/site-packages/concrete/ | |
| # CIFAR-10-8b benchmark | |
| - name: Benchmark - CIFAR-10-8b | |
| if: github.event.inputs.benchmark == 'cifar-10-8b' | |
| run: | | |
| source .venv/bin/activate | |
| NUM_SAMPLES=${{ github.event.inputs.num_samples }} python3 ./use_case_examples/cifar/cifar_brevitas_with_model_splitting/infer_fhe.py | |
| python3 ./benchmarks/convert_cifar.py --model-name "8-bit-split-v0" | |
| # CIFAR-10-16b benchmark | |
| - name: Benchmark - CIFAR-10-16b | |
| if: github.event.inputs.benchmark == 'cifar-10-16b' | |
| run: | | |
| source .venv/bin/activate | |
| NUM_SAMPLES=${{ github.event.inputs.num_samples }} P_ERROR=${{ github.event.inputs.p_error }} python3 ./use_case_examples/cifar/cifar_brevitas_training/evaluate_one_example_fhe.py | |
| python3 ./benchmarks/convert_cifar.py --model-name "16-bits-trained-v0" | |
| - name: Archive raw predictions | |
| uses: actions/[email protected] | |
| with: | |
| name: predictions.csv | |
| path: inference_results.csv | |
| - name: Archive metrics | |
| uses: actions/[email protected] | |
| with: | |
| name: metrics.json | |
| path: to_upload.json | |
| - name: Archive MLIR | |
| uses: actions/[email protected] | |
| with: | |
| name: mlir.txt | |
| path: cifar10.mlir | |
| - name: Archive Graph | |
| uses: actions/[email protected] | |
| with: | |
| name: graph.txt | |
| path: cifar10.graph | |
| - name: Archive client | |
| uses: actions/[email protected] | |
| with: | |
| name: client.zip | |
| path: client_server/client.zip | |
| - name: Archive server | |
| uses: actions/[email protected] | |
| with: | |
| name: server.zip | |
| path: client_server/server.zip | |
| # We need to keep this as the last step to avoid not uploading the artifacts | |
| # if the step crashes | |
| - name: Upload results | |
| id: upload-results | |
| if: github.event.inputs.alternative-cp-branch == 'none' && github.event.inputs.alternative-cp-wheel-artifact-id == 'none' | |
| run: | | |
| # Log the json | |
| cat to_upload.json | jq | |
| # We need to sleep to avoid log issues | |
| sleep 1. | |
| # Upload the json to the benchmark database | |
| curl --fail-with-body \ | |
| -H "Authorization: Bearer ${{ secrets.NEW_ML_PROGRESS_TRACKER_TOKEN }}" \ | |
| -H "Content-Type: application/json; charset=UTF-8" \ | |
| -d @to_upload.json \ | |
| -X POST "${{ secrets.NEW_ML_PROGRESS_TRACKER_URL }}experiment" | |
| teardown-ec2: | |
| name: Teardown EC2 instance (fast-tests) | |
| if: ${{ always() }} | |
| needs: [ setup-ec2, run-cifar-10 ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Stop instance | |
| id: stop-instance | |
| uses: zama-ai/slab-github-runner@58f2cae4bf2c0b6728083f5f009b6dc0eb6dc3ac | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| label: ${{ needs.setup-ec2.outputs.runner-name }} | |
| slack-notification: | |
| runs-on: ubuntu-20.04 | |
| needs: [run-cifar-10] | |
| if: github.event.inputs.alternative-cp-branch == 'none' && github.event.inputs.alternative-cp-wheel-artifact-id == 'none' | |
| steps: | |
| - name: Slack Notification | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 | |
| env: | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png | |
| SLACK_COLOR: ${{ needs.run-cifar-10.result }} | |
| SLACK_MESSAGE: "Benchmark action: ${{ github.event.inputs.benchmark }} (${{ env.ACTION_RUN_URL }}) ended with result: ${{ needs.run-cifar-10.result }}" | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |