From 0323a7cc902dd263a39353fea93f252a7b274b53 Mon Sep 17 00:00:00 2001 From: Bourgerie Quentin Date: Fri, 18 Oct 2024 18:35:32 +0200 Subject: [PATCH] refactor(ci): Refactoring CI workflows --- ...action-pin.yaml => action_compliance.yaml} | 16 +- .github/workflows/actionlint.yml | 16 - .github/workflows/check_commit_signature.yml | 11 - ...{block_merge.yml => commit_compliance.yml} | 28 +- .github/workflows/compiler_benchmark.yml | 142 --------- .../workflows/compiler_build_and_test_cpu.yml | 158 ---------- ...ompiler_build_and_test_cpu_distributed.yml | 89 ------ .../workflows/compiler_format_and_linting.yml | 39 --- .../workflows/concrete_compiler_benchmark.yml | 162 ++++++++++ .../workflows/concrete_compiler_test_cpu.yml | 187 +++++++++++ ...concrete_compiler_test_cpu_distributed.yml | 112 +++++++ .../workflows/concrete_python_test_macos.yml | 96 +++--- .../workflows/concrete_python_tests_linux.yml | 293 +++++++++--------- .github/workflows/linelint.yml | 18 -- .github/workflows/main.yml | 18 +- ci/slab.toml | 25 ++ frontends/concrete-python/Makefile | 9 +- 17 files changed, 714 insertions(+), 705 deletions(-) rename .github/workflows/{action-pin.yaml => action_compliance.yaml} (54%) delete mode 100644 .github/workflows/actionlint.yml delete mode 100644 .github/workflows/check_commit_signature.yml rename .github/workflows/{block_merge.yml => commit_compliance.yml} (59%) delete mode 100644 .github/workflows/compiler_benchmark.yml delete mode 100644 .github/workflows/compiler_build_and_test_cpu.yml delete mode 100644 .github/workflows/compiler_build_and_test_cpu_distributed.yml delete mode 100644 .github/workflows/compiler_format_and_linting.yml create mode 100644 .github/workflows/concrete_compiler_benchmark.yml create mode 100644 .github/workflows/concrete_compiler_test_cpu.yml create mode 100644 .github/workflows/concrete_compiler_test_cpu_distributed.yml delete mode 100644 .github/workflows/linelint.yml diff --git a/.github/workflows/action-pin.yaml b/.github/workflows/action_compliance.yaml similarity index 54% rename from .github/workflows/action-pin.yaml rename to .github/workflows/action_compliance.yaml index feb3cb4ba0..f24c6b508d 100644 --- a/.github/workflows/action-pin.yaml +++ b/.github/workflows/action_compliance.yaml @@ -1,19 +1,29 @@ -name: Action Pin +name: check action compliance on: pull_request: + paths: + - .github/workflows/** push: branches: - main + - 'release/*' jobs: - check-action-pin: + action-pin: runs-on: ubuntu-latest steps: - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - name: Ensure SHA pinned actions uses: zgosalvez/github-actions-ensure-sha-pinned-actions@0901cf7b71c7ea6261ec69a3dc2bd3f9264f893e # v3.0.12 with: allowlist: | slsa-framework/slsa-github-generator + + action-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + + - name: actionlint + uses: raven-actions/actionlint@01fce4f43a270a612932cb1c64d40505a029f821 # v2.0.0 diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml deleted file mode 100644 index fbd5087f24..0000000000 --- a/.github/workflows/actionlint.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Action Lint - -on: - pull_request: - push: - branches: - - main - -jobs: - action-lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - - name: actionlint - uses: raven-actions/actionlint@01fce4f43a270a612932cb1c64d40505a029f821 # v2.0.0 diff --git a/.github/workflows/check_commit_signature.yml b/.github/workflows/check_commit_signature.yml deleted file mode 100644 index ad2fa41191..0000000000 --- a/.github/workflows/check_commit_signature.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: Check Commit Signatures - -on: - pull_request: - -jobs: - check-commit-signatures: - runs-on: ubuntu-latest - steps: - - name: Check commit signatures - uses: 1Password/check-signed-commits-action@ed2885f3ed2577a4f5d3c3fe895432a557d23d52 diff --git a/.github/workflows/block_merge.yml b/.github/workflows/commit_compliance.yml similarity index 59% rename from .github/workflows/block_merge.yml rename to .github/workflows/commit_compliance.yml index 6500845837..fe0f13e8de 100644 --- a/.github/workflows/block_merge.yml +++ b/.github/workflows/commit_compliance.yml @@ -1,13 +1,14 @@ -# Check commit and PR compliance -name: Check commit message compliance +name: check commit compliance on: pull_request: - types: [opened, synchronize, reopened] + push: + branches: + - 'main' + - 'release/*' jobs: - check-commit-pr: - name: Check commit and PR - runs-on: ubuntu-20.04 + format: + runs-on: ubuntu-latest steps: - name: Check first line uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee # v2.0.0 @@ -19,3 +20,18 @@ jobs: excludeTitle: 'true' # optional: this excludes the title of a pull request checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true + + signatures: + runs-on: ubuntu-latest + steps: + - name: Check commit signatures + uses: 1Password/check-signed-commits-action@ed2885f3ed2577a4f5d3c3fe895432a557d23d52 + + linelint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Linelint + uses: fernandrone/linelint@8136e0fa9997122d80f5f793e0bb9a45e678fbb1 # 0.0.4 + id: linelint diff --git a/.github/workflows/compiler_benchmark.yml b/.github/workflows/compiler_benchmark.yml deleted file mode 100644 index 0b17545144..0000000000 --- a/.github/workflows/compiler_benchmark.yml +++ /dev/null @@ -1,142 +0,0 @@ -# Run benchmarks on an AWS instance for compiler and return parsed results to Slab CI bot. -name: Compiler - Performance benchmarks - -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - -# concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - CUDA_PATH: /usr/local/cuda-11.8 - GCC_VERSION: 8 - -jobs: - run-benchmarks: - name: Execute end-to-end benchmarks in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Get benchmark date - run: | - echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" - - - name: Fetch submodules - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Export specific variables (CPU) - if: ${{ !startswith(inputs.instance_type, 'p3.') }} - run: | - echo "CUDA_SUPPORT=OFF" >> "${GITHUB_ENV}" - echo "BENCHMARK_TARGET=run-cpu-benchmarks" >> "${GITHUB_ENV}" - - - name: Export specific variables (GPU) - if: ${{ startswith(inputs.instance_type, 'p3.') }} - run: | - echo "CUDA_SUPPORT=ON" >> "${GITHUB_ENV}" - echo "BENCHMARK_TARGET=run-gpu-benchmarks" >> "${GITHUB_ENV}" - echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}" - echo "$CUDA_PATH/bin" >> "${GITHUB_PATH}" - echo "LD_LIBRARY_PATH=$CUDA_PATH/lib:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}" - echo "CC=/usr/bin/gcc-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" - echo "CXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" - echo "CUDAHOSTCXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" - echo "CUDACXX=$CUDA_PATH/bin/nvcc" >> "${GITHUB_ENV}" - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Build compiler benchmarks - run: | - set -e - git config --global --add safe.directory '*' - cd compilers/concrete-compiler/compiler - make BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} build-benchmarks - - - name: Run end-to-end benchmarks - run: | - set -e - cd compilers/concrete-compiler/compiler - make ${{ env.BENCHMARK_TARGET }} - - - name: Upload raw results artifact - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: compiler_${{ github.sha }}_raw - path: compilers/concrete-compiler/compiler/benchmarks_results.json - - - name: Parse results - shell: bash - run: | - COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" - COMMIT_HASH="$(git describe --tags --dirty)" - python3 ./ci/benchmark_parser.py compilers/concrete-compiler/compiler/benchmarks_results.json ${{ env.RESULTS_FILENAME }} \ - --database compiler_benchmarks \ - --hardware ${{ inputs.instance_type }} \ - --project-version ${COMMIT_HASH} \ - --branch ${{ github.ref_name }} \ - --commit-date ${COMMIT_DATE} \ - --bench-date "${{ env.BENCH_DATE }}" \ - --throughput - - - name: Upload parsed results artifact - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: compiler_${{ github.sha }} - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - echo "Computing HMac on downloaded artifact" - SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" - echo "Sending results to Slab..." - curl -v -k \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: store_data" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @${{ env.RESULTS_FILENAME }} \ - ${{ secrets.SLAB_URL }} diff --git a/.github/workflows/compiler_build_and_test_cpu.yml b/.github/workflows/compiler_build_and_test_cpu.yml deleted file mode 100644 index e20cc664e0..0000000000 --- a/.github/workflows/compiler_build_and_test_cpu.yml +++ /dev/null @@ -1,158 +0,0 @@ -name: Compiler - Build and Test (CPU) - -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - matrix_item: - description: 'Build matrix item' - type: string - -# concurrency: -# group: compiler_build_and_test_cpu-${{ github.ref }} -# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -env: - DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler - GLIB_VER: 2_28 - -jobs: - BuildAndTest: - name: Build and test compiler in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "Matrix item: ${{ inputs.matrix_item }}" - - - name: Set up env - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - - - name: Fetch repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Create build dir - run: mkdir build - - - name: Build compiler - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - id: build-compiler - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - ${{ env.DOCKER_GPU_OPTION }} - shell: bash - run: | - rustup toolchain install nightly-2024-09-30 - pip install mypy - set -e - cd /concrete/compilers/concrete-compiler/compiler - rm -rf /build/* - make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all - echo "Debug: ccache statistics (after the build):" - ccache -s - - - name: Build compiler Dialects docs and check diff - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - id: build-compiler-docs - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - ${{ env.DOCKER_GPU_OPTION }} - shell: bash - run: | - set -e - cd /concrete/compilers/concrete-compiler/compiler - make BUILD_DIR=/build doc - cd /build/tools/concretelang/docs/concretelang/ - sed -i -e 's/\[TOC\]//' *Dialect.md - for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done; - - - name: Enable complete tests on push to main - if: github.ref == 'refs/heads/main' - run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV - - - name: Enable minimal tests otherwise - if: github.ref != 'refs/heads/main' - run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV - - - name: Test compiler - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - ${{ env.DOCKER_GPU_OPTION }} - shell: bash - run: | - set -e - rustup toolchain install nightly-2024-09-30 - cd /concrete/compilers/concrete-compiler/compiler - pip install pytest - pip install mypy - dnf install -y libzstd libzstd-devel - sed "s/pytest/python -m pytest/g" -i Makefile - mkdir -p /tmp/concrete_compiler/gpu_tests/ - make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests - chmod -R ugo+rwx /tmp/KeySetCache - - - name: Analyze logs - run: | - cd build/gtest-parallel-logs/passed - ls -1 | xargs grep -H "WARNING RETRY" | sed -e "s/.log.*//g" | uniq -c | sed -re "s/ *([0-9]*) (.*)/::warning ::Test \2 retried \1 times/g" | cat - - # - name: Archive python package - # uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - # with: - # name: concrete-compiler.whl - # path: build/wheels/concrete_compiler-*-manylinux_{{ env.GLIB_VER }}_x86_64.whl - # retention-days: 14 diff --git a/.github/workflows/compiler_build_and_test_cpu_distributed.yml b/.github/workflows/compiler_build_and_test_cpu_distributed.yml deleted file mode 100644 index 713653b4c8..0000000000 --- a/.github/workflows/compiler_build_and_test_cpu_distributed.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Compiler - Distributed Build and Test (CPU) - -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - matrix_item: - description: 'Build matrix item' - type: string - - -env: - GLIB_VER: 2_28 - -jobs: - BuildAndTest: - name: Build and test compiler on Slurm cluster in EC2 - runs-on: distributed-ci - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "ID: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "Matrix item: ${{ inputs.matrix_item }}" - - - name: Instance cleanup - run: | - sudo rm -rf /home/ubuntu/actions-runner/_work/concrete/concrete - mkdir -p /home/ubuntu/actions-runner/_work/concrete/concrete - docker system prune -af - - - name: Fetch repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/shared" >> "${GITHUB_ENV}" - - - name: Export specific variables (CPU) - if: ${{ !startswith(inputs.instance_type, 'p3.') }} - run: | - echo "CUDA_SUPPORT=OFF" >> "${GITHUB_ENV}" - echo "DATAFLOW_EXECUTION_ENABLED=ON" >> "${GITHUB_ENV}" - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Build compiler benchmarks - run: | - set -e - git config --global --add safe.directory '*' - cd compilers/concrete-compiler/compiler - rm -rf /shared/build - make HPX_DIR=/shared/hpx install-hpx-from-source - make HPX_DIR=/shared/hpx BUILD_DIR=/shared/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} build-end-to-end-tests - - - name: Run end-to-end benchmarks - run: | - set -e - cd compilers/concrete-compiler/compiler - rm -rf /shared/KeyCache - make BUILD_DIR=/shared/build KEY_CACHE_DIRECTORY=/shared/KeyCache run-end-to-end-distributed-tests - - - name: Instance cleanup - run: | - sudo rm -rf /home/ubuntu/actions-runner/_work/concrete/concrete/* - docker system prune -af diff --git a/.github/workflows/compiler_format_and_linting.yml b/.github/workflows/compiler_format_and_linting.yml deleted file mode 100644 index 4057fce6b5..0000000000 --- a/.github/workflows/compiler_format_and_linting.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Compiler - Compliance - -on: - workflow_call: - workflow_dispatch: - -jobs: - FormattingAndLinting: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Format with clang-format (Cpp) - run: | - sudo apt install moreutils - cd compilers/concrete-compiler/compiler - ./scripts/format_cpp.sh - - name: Format with cmake-format (Cmake) - run: | - pip3 install cmakelang - cd compilers/concrete-compiler/compiler - ./scripts/format_cmake.sh - - name: Format with black (Python) - run: | - cd compilers/concrete-compiler/compiler - pip install -r lib/Bindings/Python/requirements_dev.txt - make check-python-format - - name: Lint with pylint (Python) - run: | - cd compilers/concrete-compiler/compiler - # compiler requirements to lint - pip install numpy - make python-lint - - CheckLicense: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Check if sources include the license header - run: .github/workflows/scripts/check_for_license.sh diff --git a/.github/workflows/concrete_compiler_benchmark.yml b/.github/workflows/concrete_compiler_benchmark.yml new file mode 100644 index 0000000000..d9d2d314b2 --- /dev/null +++ b/.github/workflows/concrete_compiler_benchmark.yml @@ -0,0 +1,162 @@ +name: concrete-compiler benchmark linux-cpu + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_benchmark.yml + - compilers/** + - backends/** + - tools/** + push: + branches: + - 'main' + - 'release/*' + +env: + DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_compiler_benchmark_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + name: Setup instance + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-bench + + build-and-run-benchmarks: + name: Build and run concrete-compiler benchmarks + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Checkout concrete + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + fetch-depth: 0 + - name: Set up home + # "Install rust" step require root user to have a HOME directory which is not set. + run: | + echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + - name: Setup rust toolchain for concrete-cpu + uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + - name: Build compiler benchmarks + run: | + set -e + git config --global --add safe.directory '*' + cd compilers/concrete-compiler/compiler + make BINDINGS_PYTHON_ENABLED=OFF build-benchmarks + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Build compiler benchmark finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + - name: Run compiler benchmarks + run: | + set -e + cd compilers/concrete-compiler/compiler + make run-cpu-benchmarks + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Run compiler benchmark finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + - name: Upload raw results artifact + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: compiler-benchmarks-result + path: compilers/concrete-compiler/compiler/benchmarks_results.json + + parse-and-send-results: + name: Parse and send results + needs: [setup-instance, build-and-run-benchmarks] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Download compiler-benchmarks-result + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: compiler-benchmarks-result + - name: Parse results + shell: bash + run: | + COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" + COMMIT_HASH="$(git describe --tags --dirty)" + python3 ./ci/benchmark_parser.py benchmarks_results.json parsed_benchmark_results.json \ + --database compiler_benchmarks \ + --hardware "hpc7a.96xlarge" \ + --project-version ${COMMIT_HASH} \ + --branch ${{ github.ref_name }} \ + --commit-date ${COMMIT_DATE} \ + --bench-date "$(date --iso-8601=seconds)" \ + --throughput + - name: Upload parsed results artifact + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: compiler-benchmarks-result + path: parsed_benchmark_results.json + - name: Checkout Slab repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: zama-ai/slab + path: slab + token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} + - name: Send data to Slab + shell: bash + run: | + echo "Computing HMac on downloaded artifact" + SIGNATURE="$(slab/scripts/hmac_calculator.sh parsed_benchmark_results.json '${{ secrets.JOB_SECRET }}')" + echo "Sending results to Slab..." + curl -v -k \ + -H "Content-Type: application/json" \ + -H "X-Slab-Repository: ${{ github.repository }}" \ + -H "X-Slab-Command: store_data" \ + -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ + -d @parsed_benchmark_results.json \ + ${{ secrets.SLAB_URL }} + + + teardown-instance: + name: Teardown instance + needs: [ setup-instance, parse-and-send-results ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" \ No newline at end of file diff --git a/.github/workflows/concrete_compiler_test_cpu.yml b/.github/workflows/concrete_compiler_test_cpu.yml new file mode 100644 index 0000000000..41aef15c86 --- /dev/null +++ b/.github/workflows/concrete_compiler_test_cpu.yml @@ -0,0 +1,187 @@ +name: concrete-compiler test linux-cpu + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_test_cpu.yml + - compilers/** + - backends/** + - tools/** + push: + branches: + - 'main' + - 'release/*' + +env: + DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_compiler_test_cpu_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + name: Setup instance + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-test + + format-and-lint: + name: Format and linting checks + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Format with clang-format (Cpp) + run: | + sudo apt install moreutils + cd compilers/concrete-compiler/compiler + ./scripts/format_cpp.sh + - name: Format with cmake-format (Cmake) + run: | + pip3 install cmakelang + cd compilers/concrete-compiler/compiler + ./scripts/format_cmake.sh + - name: Format with black (Python) + run: | + cd compilers/concrete-compiler/compiler + pip install -r lib/Bindings/Python/requirements_dev.txt + make check-python-format + - name: Lint with pylint (Python) + run: | + cd compilers/concrete-compiler/compiler + # compiler requirements to lint + pip install numpy + make python-lint + - name: Check if sources include the license header + run: .github/workflows/scripts/check_for_license.sh + + build-and-run-test: + name: Build and test compiler in EC2 + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Fetch repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + fetch-depth: 0 + - name: Create build dir + run: mkdir build + + - name: Build compiler + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + id: build-compiler + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + -v ${{ github.workspace }}/wheels:/wheels + shell: bash + run: | + cd /concrete/compilers/concrete-compiler/compiler + rm -rf /build/* + make DATAFLOW_EXECUTION_ENABLED=ON BUILD_DIR=/build all + echo "Debug: ccache statistics (after the build):" + ccache -s + + - name: Build compiler Dialects docs and check diff + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + id: build-compiler-docs + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + -v ${{ github.workspace }}/wheels:/wheels + shell: bash + run: | + set -e + cd /concrete/compilers/concrete-compiler/compiler + make BUILD_DIR=/build doc + cd /build/tools/concretelang/docs/concretelang/ + sed -i -e 's/\[TOC\]//' *Dialect.md + for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done; + - name: Enable complete tests on push to main + if: github.ref == 'refs/heads/main' + run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV + - name: Enable minimal tests otherwise + if: github.ref != 'refs/heads/main' + run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV + - name: Run compiler tests + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + shell: bash + run: | + set -e + rustup toolchain install nightly-2024-09-30 + cd /concrete/compilers/concrete-compiler/compiler + pip install pytest + pip install mypy + dnf install -y libzstd libzstd-devel + sed "s/pytest/python -m pytest/g" -i Makefile + mkdir -p /tmp/concrete_compiler/gpu_tests/ + make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} run-tests + - name: Analyze logs + run: | + cd build/gtest-parallel-logs/passed + ls -1 | xargs grep -H "WARNING RETRY" | sed -e "s/.log.*//g" | uniq -c | sed -re "s/ *([0-9]*) (.*)/::warning ::Test \2 retried \1 times/g" | cat + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + name: Teardown instance + needs: [ setup-instance, build-and-run-test ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_compiler_test_cpu_distributed.yml b/.github/workflows/concrete_compiler_test_cpu_distributed.yml new file mode 100644 index 0000000000..12dcfb2df4 --- /dev/null +++ b/.github/workflows/concrete_compiler_test_cpu_distributed.yml @@ -0,0 +1,112 @@ +name: concrete-compiler test linux-cpu-distributed + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_test_cpu_distributed.yml + - compilers/concrete-compiler/** + push: + branches: + - 'main' + - 'release/*' + +env: + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_compiler_test_cpu_distributed_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + name: Setup instance + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: slurm-cluster + + build-and-run-test: + name: Build and test compiler on Slurm cluster in EC2 + # The distributed-ci runner is registered on the instance configured in the slurm-cluster profile. + # It's why we need to setup-instance + needs: setup-instance + runs-on: distributed-ci + steps: + - name: Instance cleanup + run: | + sudo rm -rf /home/ubuntu/actions-runner/_work/concrete/concrete + mkdir -p /home/ubuntu/actions-runner/_work/concrete/concrete + docker system prune -af + + - name: Fetch repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + submodules: recursive + + - name: Set up home + # "Install rust" step require root user to have a HOME directory which is not set. + run: | + echo "HOME=/shared" >> "${GITHUB_ENV}" + + - name: Setup rust toolchain for concrete-cpu + uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + + - name: Build end-to-end distributed test + run: | + cd compilers/concrete-compiler/compiler + rm -rf /shared/build + make HPX_DIR=/shared/hpx install-hpx-from-source + make HPX_DIR=/shared/hpx BUILD_DIR=/shared/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON BINDINGS_PYTHON_ENABLED=OFF build-end-to-end-tests + + - name: Run end-to-end distributed test + run: | + cd compilers/concrete-compiler/compiler + rm -rf /shared/KeyCache + make BUILD_DIR=/shared/build KEY_CACHE_DIRECTORY=/shared/KeyCache run-end-to-end-distributed-tests + + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + name: Teardown instance + needs: [ setup-instance, build-and-run-test ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" \ No newline at end of file diff --git a/.github/workflows/concrete_python_test_macos.yml b/.github/workflows/concrete_python_test_macos.yml index 79e9409218..9c12d56386 100644 --- a/.github/workflows/concrete_python_test_macos.yml +++ b/.github/workflows/concrete_python_test_macos.yml @@ -1,40 +1,37 @@ -name: Concrete Python Tests (macOS) +name: concrete-python tests macos on: - workflow_call: workflow_dispatch: - secrets: - CONCRETE_CI_SSH_PRIVATE: - required: true - CONCRETE_ACTIONS_TOKEN: - required: true + pull_request: + paths: + - .github/workflows/concrete_python_tests_macos.yml + push: + branches: + - 'main' + - 'release/*' concurrency: - group: concrete_python_tests_macos-${{ github.ref }} + group: concrete_python_tests_macos_${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - BuildConcreteCompilerAndTestConcretePythonInMacOS: + concrete-python-test-pytest: strategy: fail-fast: false matrix: machine: ["aws-mac1-metal", "aws-mac2-metal"] - runs-on: ${{ matrix.machine }} steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Install OS Dependencies + fetch-depth: 0 + - name: Install build dependencies run: | brew install ninja ccache - - name: Setup rust toolchain for concrete-cpu uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Cache Compilation (push) + - name: Cache compilation (push) if: github.event_name == 'push' uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -42,8 +39,7 @@ jobs: key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.sha }} restore-keys: | ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - - name: Cache Compilation (pull_request) + - name: Cache compilation (pull_request) if: github.event_name == 'pull_request' uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -51,88 +47,72 @@ jobs: key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.event.pull_request.base.sha }} restore-keys: | ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - - name: Get tmpdir path - if: github.event_name == 'push' - id: tmpdir-path - run: echo "::set-output name=TMPDIR_PATH::$TMPDIR" - - - name: Build + - name: Prepare build environment run: | set -e - cd frontends/concrete-python - + cd $GITHUB_WORKSPACE/frontends/concrete-python + # Setup pkg-config to find OpenBLAS (scipy need it) export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" - + rm -rf .venv python3.10 -m venv .venv - - . .venv/bin/activate - + + . $GITHUB_WORKSPACE/frontends/concrete-python/.venv/bin/activate + pip install -r requirements.dev.txt pip install -r requirements.txt - + - name: Build concrete-compiler python-bindings + run: | + $GITHUB_WORKSPACE/frontends/concrete-python .venv/bin/activate cd $GITHUB_WORKSPACE/compilers/concrete-compiler/compiler - - echo "Debug: ccache statistics (prior to the build):" - ccache -s - + ccache -z make Python3_EXECUTABLE=$(which python) python-bindings - - echo "Debug: ccache statistics (after the build):" ccache -s - - export COMPILER_BUILD_DIRECTORY=$(pwd)/build + - name: Create wheels + run: | + $GITHUB_WORKSPACE/frontends/concrete-python .venv/bin/activate cd $GITHUB_WORKSPACE/frontends/concrete-python - - rm -rf dist - mkdir -p dist - + + export COMPILER_BUILD_DIRECTORY=$GITHUB_WORKSPACE/compilers/concrete-compiler/compiler + rm -rf dist && mkdir -p dist pip wheel -v --no-deps -w dist . delocate-wheel -v dist/*macos*.whl - - deactivate + deactivate - name: Prepare test environment run: | set -e export TEST_TMP_DIR=$(mktemp -d) echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}" cd $TEST_TMP_DIR - + python3.10 -m venv .testenv . .testenv/bin/activate - + pip install $GITHUB_WORKSPACE/frontends/concrete-python/dist/*macos*.whl pip install -r $GITHUB_WORKSPACE/frontends/concrete-python/requirements.dev.txt # MacOS x86 have conflict between our OpenMP library, and one from torch # we fix it by using a single one (from torch) # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8 - + find .testenv/lib/python3.10/site-packages -not \( -path .testenv/lib/python3.10/site-packages/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1 ln -f -s $(pwd)/.testenv/lib/python3.10/site-packages/concrete/.dylibs/libomp.dylib cp -R $GITHUB_WORKSPACE/frontends/concrete-python/examples ./examples cp -R $GITHUB_WORKSPACE/frontends/concrete-python/tests ./tests - - cp $GITHUB_WORKSPACE/frontends/concrete-python/Makefile . - - name: Test + cp $GITHUB_WORKSPACE/frontends/concrete-python/Makefile . + - name: Run pytest-macos run: | - set -e - export TEST_TMP_DIR="testing_concrete_python" cd $TEST_TMP_DIR && . .testenv/bin/activate KEY_CACHE_DIRECTORY=./KeySetCache PYTEST_MARKERS="not dataflow and not graphviz" make pytest-macos - - - name: Test notebooks + - name: Run test-notebooks run: | set -e - export TEST_TMP_DIR="testing_concrete_python" cd $TEST_TMP_DIR && . .testenv/bin/activate make test-notebooks - - name: Cleanup host if: success() || failure() run: | diff --git a/.github/workflows/concrete_python_tests_linux.yml b/.github/workflows/concrete_python_tests_linux.yml index 6960926957..fe31ff378b 100644 --- a/.github/workflows/concrete_python_tests_linux.yml +++ b/.github/workflows/concrete_python_tests_linux.yml @@ -1,69 +1,60 @@ -name: Concrete Python Tests (Linux) +name: concrete-python tests linux-cpu on: workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - -# concurrency: -# group: concrete_python_tests_linux-${{ github.ref }} -# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + pull_request: + paths: + - .github/workflows/concrete_python_tests_linux.yml + - frontends/concrete-python/** + push: + branches: + - 'main' + - 'release/*' env: DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler - CUDA_PATH: /usr/local/cuda-11.8 - GCC_VERSION: 11 - GLIB_VER: 2_28 + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_python_tests_linux_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - BuildConcreteCompilerAndTestConcretePythonInLinux: - name: Build Concrete Compiler and Test Concrete Python in Linux - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} + setup-instance: + name: Setup instance + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} steps: - - name: Log instance configuration - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Set up GitHub environment - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - - - name: Checkout + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-test + + build-python-bindings: + name: Build concrete-compiler python bindings + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Checkout concrete uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Create build directory + fetch-depth: 0 + - name: Create concrete build directory run: mkdir build - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Build bindings + - name: Build concrete-compiler python bindings uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} - id: build-compiler-bindings with: registry: ghcr.io image: ${{ env.DOCKER_IMAGE_TEST }} @@ -72,73 +63,59 @@ jobs: options: >- -v ${{ github.workspace }}:/concrete -v ${{ github.workspace }}/build:/build - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket shell: bash run: | - rustup toolchain install nightly-2024-09-30 - pip install mypy set -e - rm -rf /build/* + rustup toolchain install nightly-2024-09-30 dnf -y install graphviz graphviz-devel + #rm -rf /build/* + cd /concrete/frontends/concrete-python make venv source .venv/bin/activate - + cd /concrete/compilers/concrete-compiler/compiler make BUILD_DIR=/build DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$(which python3) python-bindings - + echo "Debug: ccache statistics (after the build):" ccache -s - - - name: Prepare test environment - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-python-bindings finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + - name: Create artifact archive + run: | + cd build + tar czvf artifacts.tgz lib/libConcretelangRuntime.so tools/concretelang/python_packages + - name: Upload concrete-compiler python-bindings + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - shell: bash - run: | - set -e - - dnf -y install graphviz graphviz-devel - - cd /concrete/frontends/concrete-python - make venv - - - name: Test - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} + name: concrete-compiler-python-bindings + include-hidden-files: true + retention-days: 3 + path: build/artifacts.tgz + + test-pytest: + name: Test concrete-python pytest + needs: [setup-instance, build-python-bindings] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Download concrete-compiler python-bindings + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - shell: bash - run: | - set -e - - cd /concrete/frontends/concrete-python - source .venv/bin/activate - - export COMPILER_BUILD_DIRECTORY=/build - - mkdir ./KeySetCache - KEY_CACHE_DIRECTORY=./KeySetCache make pytest - - - name: Test notebooks + name: concrete-compiler-python-bindings + path: compiler-artifacts + - name: Extract artifacts archive + run: | + cd compiler-artifacts + tar xzvf artifacts.tgz + - name: Run pytest uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} with: registry: ghcr.io image: ${{ env.DOCKER_IMAGE_TEST }} @@ -146,53 +123,40 @@ jobs: password: ${{ secrets.GHCR_PASSWORD }} options: >- -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build + -v ${{ github.workspace }}/compiler-artifacts:/compiler-artifacts shell: bash run: | set -e - + export COMPILER_BUILD_DIRECTORY=/compiler-artifacts cd /concrete/frontends/concrete-python source .venv/bin/activate - - export COMPILER_BUILD_DIRECTORY=/build - - make test-notebooks - - - - name: Build bindings gpu - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ contains(inputs.instance_type, 'p3') }} - id: build-compiler-bindings-gpu + export KEY_CACHE_DIRECTORY=./key-set-cache + mkdir $KEY_CACHE_DIRECTORY + make pytest + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "test-pytest finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + test-notebooks: + name: Test concrete-python notebooks + needs: [setup-instance, build-python-bindings] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Download concrete-compiler python-bindings + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - --gpus all - shell: bash - run: | - set -e - rm -rf /build/* - - cd /concrete/frontends/concrete-python - make venv - source .venv/bin/activate - - cd /concrete/compilers/concrete-compiler/compiler - make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python3) CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} python-bindings - - echo "Debug: ccache statistics (after the build):" - ccache -s - - - name: Test gpu + name: concrete-compiler-python-bindings + path: /compiler-artifacts + - name: Extract artifacts archive + run: | + cd compiler-artifacts + tar xzvf artifacts.tgz + - name: Run pytest uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ contains(inputs.instance_type, 'p3') }} with: registry: ghcr.io image: ${{ env.DOCKER_IMAGE_TEST }} @@ -200,19 +164,42 @@ jobs: password: ${{ secrets.GHCR_PASSWORD }} options: >- -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - --gpus all + -v ${{ github.workspace }}/compiler-artifacts:/compiler-artifacts shell: bash run: | set -e - + export COMPILER_BUILD_DIRECTORY=/compiler-artifacts cd /concrete/frontends/concrete-python - make venv source .venv/bin/activate - - export COMPILER_BUILD_DIRECTORY=/build - KEY_CACHE_DIRECTORY=/tmp/KeySetCache mkdir ./KeySetCache - make pytest-gpu - - chmod -R ugo+rwx /tmp/KeySetCache + make test-notebooks + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "test-notebooks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + name: Teardown instance + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, test-pytest, test-notebooks ] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/linelint.yml b/.github/workflows/linelint.yml deleted file mode 100644 index 03b814c839..0000000000 --- a/.github/workflows/linelint.yml +++ /dev/null @@ -1,18 +0,0 @@ -# This job is the main jobs will dispatch build and test for every modules of our mono repo. -name: Linelint - -on: - pull_request: - push: - branches: - - 'main' - -jobs: - linelint: - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Linelint - uses: fernandrone/linelint@8136e0fa9997122d80f5f793e0bb9a45e678fbb1 # 0.0.4 - id: linelint diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e39d983fde..1a3471214f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,18 +2,18 @@ name: Main on: - pull_request: +# pull_request: push: branches: - 'main' - - 'release/*' - - 'force-docker-images' - - 'private_release/*' - tags: - - 'v[0-9]+.[0-9]+.[0-9]+*' - schedule: - # Nightly Release @ 3AM after each work day - - cron: "0 3 * * 2-6" +# - 'release/*' +# - 'force-docker-images' +# - 'private_release/*' +# tags: +# - 'v[0-9]+.[0-9]+.[0-9]+*' +# schedule: +# # Nightly Release @ 3AM after each work day +# - cron: "0 3 * * 2-6" jobs: # This jobs outputs for each modules of our mono-repo if it changed, diff --git a/ci/slab.toml b/ci/slab.toml index 0a42840d02..c69399c4a1 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -1,3 +1,28 @@ +[backend.aws.cpu-test] +region = "eu-west-1" +image_id = "ami-002bdcd64b8472cf9" # Based on Ubuntu 22.4 +instance_type = "m7i.16xlarge" +security_group = ["sg-0e55cc31dfda0d8a7", ] + +[backend.aws.cpu-bench] +region = "eu-west-1" +image_id = "ami-002bdcd64b8472cf9" +instance_type = "hpc7a.96xlarge" + +[backend.aws.gpu-test] +region = "us-east-1" +image_id = "ami-0257c6ad39f902b5e" +instance_type = "p3.2xlarge" +subnet_id = "subnet-8123c9e7" +security_group= ["sg-017afab1f328af917", ] + +[backend.aws.slurm-cluster] +region = "eu-west-3" +image_id = "ami-0bb5bb9cb747b5ddd" +instance_id = "i-0e5ae2a14134d6275" +instance_type = "m6i.8xlarge" +security_group= ["sg-02dd8470fa845f31b", ] + # This is the new version of Slab that handles multi backend providers. [backend.aws.m7i-cpu-bench] region = "eu-west-1" diff --git a/frontends/concrete-python/Makefile b/frontends/concrete-python/Makefile index ef04545803..7322a5cf9f 100644 --- a/frontends/concrete-python/Makefile +++ b/frontends/concrete-python/Makefile @@ -30,9 +30,6 @@ CONCRETE_VERSION?="" # empty mean latest venv: $(PYTHON) -m venv .venv . .venv/bin/activate -ifeq (,$(wildcard ${RUNTIME_LIBRARY})) - $(PIP) install --extra-index-url https://pypi.zama.ai/cpu "concrete-python$(CONCRETE_VERSION)" -endif $(PIP) install -r requirements.dev.txt $(PIP) install -r requirements.extra-full.txt $(PIP) install -r requirements.txt @@ -66,6 +63,12 @@ tfhers-utils: pytest: pytest-default +debug: + echo "Printing environment" + env + echo "ls -R ${BINDINGS_DIRECTORY}" + ls -R ${BINDINGS_DIRECTORY} + pytest-default: tfhers-utils eval $(shell make silent_cp_activate) pytest tests -svv -n auto \