diff --git a/.github/workflows/action-pin.yaml b/.github/workflows/action_compliance.yaml similarity index 54% rename from .github/workflows/action-pin.yaml rename to .github/workflows/action_compliance.yaml index feb3cb4ba0..f24c6b508d 100644 --- a/.github/workflows/action-pin.yaml +++ b/.github/workflows/action_compliance.yaml @@ -1,19 +1,29 @@ -name: Action Pin +name: check action compliance on: pull_request: + paths: + - .github/workflows/** push: branches: - main + - 'release/*' jobs: - check-action-pin: + action-pin: runs-on: ubuntu-latest steps: - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - name: Ensure SHA pinned actions uses: zgosalvez/github-actions-ensure-sha-pinned-actions@0901cf7b71c7ea6261ec69a3dc2bd3f9264f893e # v3.0.12 with: allowlist: | slsa-framework/slsa-github-generator + + action-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + + - name: actionlint + uses: raven-actions/actionlint@01fce4f43a270a612932cb1c64d40505a029f821 # v2.0.0 diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml deleted file mode 100644 index fbd5087f24..0000000000 --- a/.github/workflows/actionlint.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Action Lint - -on: - pull_request: - push: - branches: - - main - -jobs: - action-lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - - name: actionlint - uses: raven-actions/actionlint@01fce4f43a270a612932cb1c64d40505a029f821 # v2.0.0 diff --git a/.github/workflows/check_commit_signature.yml b/.github/workflows/check_commit_signature.yml deleted file mode 100644 index ad2fa41191..0000000000 --- a/.github/workflows/check_commit_signature.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: Check Commit Signatures - -on: - pull_request: - -jobs: - check-commit-signatures: - runs-on: ubuntu-latest - steps: - - name: Check commit signatures - uses: 1Password/check-signed-commits-action@ed2885f3ed2577a4f5d3c3fe895432a557d23d52 diff --git a/.github/workflows/block_merge.yml b/.github/workflows/commit_compliance.yml similarity index 59% rename from .github/workflows/block_merge.yml rename to .github/workflows/commit_compliance.yml index 6500845837..fe0f13e8de 100644 --- a/.github/workflows/block_merge.yml +++ b/.github/workflows/commit_compliance.yml @@ -1,13 +1,14 @@ -# Check commit and PR compliance -name: Check commit message compliance +name: check commit compliance on: pull_request: - types: [opened, synchronize, reopened] + push: + branches: + - 'main' + - 'release/*' jobs: - check-commit-pr: - name: Check commit and PR - runs-on: ubuntu-20.04 + format: + runs-on: ubuntu-latest steps: - name: Check first line uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee # v2.0.0 @@ -19,3 +20,18 @@ jobs: excludeTitle: 'true' # optional: this excludes the title of a pull request checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true + + signatures: + runs-on: ubuntu-latest + steps: + - name: Check commit signatures + uses: 1Password/check-signed-commits-action@ed2885f3ed2577a4f5d3c3fe895432a557d23d52 + + linelint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Linelint + uses: fernandrone/linelint@8136e0fa9997122d80f5f793e0bb9a45e678fbb1 # 0.0.4 + id: linelint diff --git a/.github/workflows/compiler_benchmark.yml b/.github/workflows/compiler_benchmark.yml deleted file mode 100644 index 0b17545144..0000000000 --- a/.github/workflows/compiler_benchmark.yml +++ /dev/null @@ -1,142 +0,0 @@ -# Run benchmarks on an AWS instance for compiler and return parsed results to Slab CI bot. -name: Compiler - Performance benchmarks - -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - -# concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - CUDA_PATH: /usr/local/cuda-11.8 - GCC_VERSION: 8 - -jobs: - run-benchmarks: - name: Execute end-to-end benchmarks in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Get benchmark date - run: | - echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" - - - name: Fetch submodules - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Export specific variables (CPU) - if: ${{ !startswith(inputs.instance_type, 'p3.') }} - run: | - echo "CUDA_SUPPORT=OFF" >> "${GITHUB_ENV}" - echo "BENCHMARK_TARGET=run-cpu-benchmarks" >> "${GITHUB_ENV}" - - - name: Export specific variables (GPU) - if: ${{ startswith(inputs.instance_type, 'p3.') }} - run: | - echo "CUDA_SUPPORT=ON" >> "${GITHUB_ENV}" - echo "BENCHMARK_TARGET=run-gpu-benchmarks" >> "${GITHUB_ENV}" - echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}" - echo "$CUDA_PATH/bin" >> "${GITHUB_PATH}" - echo "LD_LIBRARY_PATH=$CUDA_PATH/lib:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}" - echo "CC=/usr/bin/gcc-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" - echo "CXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" - echo "CUDAHOSTCXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" - echo "CUDACXX=$CUDA_PATH/bin/nvcc" >> "${GITHUB_ENV}" - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Build compiler benchmarks - run: | - set -e - git config --global --add safe.directory '*' - cd compilers/concrete-compiler/compiler - make BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} build-benchmarks - - - name: Run end-to-end benchmarks - run: | - set -e - cd compilers/concrete-compiler/compiler - make ${{ env.BENCHMARK_TARGET }} - - - name: Upload raw results artifact - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: compiler_${{ github.sha }}_raw - path: compilers/concrete-compiler/compiler/benchmarks_results.json - - - name: Parse results - shell: bash - run: | - COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" - COMMIT_HASH="$(git describe --tags --dirty)" - python3 ./ci/benchmark_parser.py compilers/concrete-compiler/compiler/benchmarks_results.json ${{ env.RESULTS_FILENAME }} \ - --database compiler_benchmarks \ - --hardware ${{ inputs.instance_type }} \ - --project-version ${COMMIT_HASH} \ - --branch ${{ github.ref_name }} \ - --commit-date ${COMMIT_DATE} \ - --bench-date "${{ env.BENCH_DATE }}" \ - --throughput - - - name: Upload parsed results artifact - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: compiler_${{ github.sha }} - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - echo "Computing HMac on downloaded artifact" - SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" - echo "Sending results to Slab..." - curl -v -k \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: store_data" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @${{ env.RESULTS_FILENAME }} \ - ${{ secrets.SLAB_URL }} diff --git a/.github/workflows/compiler_build_and_test_cpu.yml b/.github/workflows/compiler_build_and_test_cpu.yml deleted file mode 100644 index e20cc664e0..0000000000 --- a/.github/workflows/compiler_build_and_test_cpu.yml +++ /dev/null @@ -1,158 +0,0 @@ -name: Compiler - Build and Test (CPU) - -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - matrix_item: - description: 'Build matrix item' - type: string - -# concurrency: -# group: compiler_build_and_test_cpu-${{ github.ref }} -# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -env: - DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler - GLIB_VER: 2_28 - -jobs: - BuildAndTest: - name: Build and test compiler in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "Matrix item: ${{ inputs.matrix_item }}" - - - name: Set up env - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - - - name: Fetch repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Create build dir - run: mkdir build - - - name: Build compiler - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - id: build-compiler - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - ${{ env.DOCKER_GPU_OPTION }} - shell: bash - run: | - rustup toolchain install nightly-2024-09-30 - pip install mypy - set -e - cd /concrete/compilers/concrete-compiler/compiler - rm -rf /build/* - make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all - echo "Debug: ccache statistics (after the build):" - ccache -s - - - name: Build compiler Dialects docs and check diff - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - id: build-compiler-docs - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - ${{ env.DOCKER_GPU_OPTION }} - shell: bash - run: | - set -e - cd /concrete/compilers/concrete-compiler/compiler - make BUILD_DIR=/build doc - cd /build/tools/concretelang/docs/concretelang/ - sed -i -e 's/\[TOC\]//' *Dialect.md - for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done; - - - name: Enable complete tests on push to main - if: github.ref == 'refs/heads/main' - run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV - - - name: Enable minimal tests otherwise - if: github.ref != 'refs/heads/main' - run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV - - - name: Test compiler - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - ${{ env.DOCKER_GPU_OPTION }} - shell: bash - run: | - set -e - rustup toolchain install nightly-2024-09-30 - cd /concrete/compilers/concrete-compiler/compiler - pip install pytest - pip install mypy - dnf install -y libzstd libzstd-devel - sed "s/pytest/python -m pytest/g" -i Makefile - mkdir -p /tmp/concrete_compiler/gpu_tests/ - make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests - chmod -R ugo+rwx /tmp/KeySetCache - - - name: Analyze logs - run: | - cd build/gtest-parallel-logs/passed - ls -1 | xargs grep -H "WARNING RETRY" | sed -e "s/.log.*//g" | uniq -c | sed -re "s/ *([0-9]*) (.*)/::warning ::Test \2 retried \1 times/g" | cat - - # - name: Archive python package - # uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - # with: - # name: concrete-compiler.whl - # path: build/wheels/concrete_compiler-*-manylinux_{{ env.GLIB_VER }}_x86_64.whl - # retention-days: 14 diff --git a/.github/workflows/compiler_build_and_test_cpu_distributed.yml b/.github/workflows/compiler_build_and_test_cpu_distributed.yml deleted file mode 100644 index 713653b4c8..0000000000 --- a/.github/workflows/compiler_build_and_test_cpu_distributed.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Compiler - Distributed Build and Test (CPU) - -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - matrix_item: - description: 'Build matrix item' - type: string - - -env: - GLIB_VER: 2_28 - -jobs: - BuildAndTest: - name: Build and test compiler on Slurm cluster in EC2 - runs-on: distributed-ci - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "ID: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "Matrix item: ${{ inputs.matrix_item }}" - - - name: Instance cleanup - run: | - sudo rm -rf /home/ubuntu/actions-runner/_work/concrete/concrete - mkdir -p /home/ubuntu/actions-runner/_work/concrete/concrete - docker system prune -af - - - name: Fetch repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/shared" >> "${GITHUB_ENV}" - - - name: Export specific variables (CPU) - if: ${{ !startswith(inputs.instance_type, 'p3.') }} - run: | - echo "CUDA_SUPPORT=OFF" >> "${GITHUB_ENV}" - echo "DATAFLOW_EXECUTION_ENABLED=ON" >> "${GITHUB_ENV}" - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Build compiler benchmarks - run: | - set -e - git config --global --add safe.directory '*' - cd compilers/concrete-compiler/compiler - rm -rf /shared/build - make HPX_DIR=/shared/hpx install-hpx-from-source - make HPX_DIR=/shared/hpx BUILD_DIR=/shared/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} build-end-to-end-tests - - - name: Run end-to-end benchmarks - run: | - set -e - cd compilers/concrete-compiler/compiler - rm -rf /shared/KeyCache - make BUILD_DIR=/shared/build KEY_CACHE_DIRECTORY=/shared/KeyCache run-end-to-end-distributed-tests - - - name: Instance cleanup - run: | - sudo rm -rf /home/ubuntu/actions-runner/_work/concrete/concrete/* - docker system prune -af diff --git a/.github/workflows/compiler_build_and_test_gpu.yml b/.github/workflows/compiler_build_and_test_gpu.yml deleted file mode 100644 index 9879d54764..0000000000 --- a/.github/workflows/compiler_build_and_test_gpu.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: Compiler - Build and Test (GPU) - -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - matrix_item: - description: 'Build matrix item' - type: string - -# concurrency: -# group: compiler_build_and_test_gpu-${{ github.ref }} -# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -env: - DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler - CUDA_PATH: /usr/local/cuda-11.8 - GCC_VERSION: 11 - -jobs: - BuildAndTest: - name: Build and test compiler in EC2 with CUDA support - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "Matrix item: ${{ inputs.matrix_item }}" - - - name: Set up env - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - - - name: Fetch repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Create build dir - run: mkdir build - - - name: Build and test compiler - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - id: build-compiler - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - --gpus all - shell: bash - run: | - rustup toolchain install nightly-2024-09-30 - pip install mypy - set -e - cd /concrete/compilers/concrete-compiler/compiler - rm -rf /build/* - mkdir -p /tmp/concrete_compiler/gpu_tests/ - make BINDINGS_PYTHON_ENABLED=OFF CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} run-end-to-end-tests-gpu - echo "Debug: ccache statistics (after the build):" - ccache -s diff --git a/.github/workflows/compiler_format_and_linting.yml b/.github/workflows/compiler_format_and_linting.yml deleted file mode 100644 index 4057fce6b5..0000000000 --- a/.github/workflows/compiler_format_and_linting.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Compiler - Compliance - -on: - workflow_call: - workflow_dispatch: - -jobs: - FormattingAndLinting: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Format with clang-format (Cpp) - run: | - sudo apt install moreutils - cd compilers/concrete-compiler/compiler - ./scripts/format_cpp.sh - - name: Format with cmake-format (Cmake) - run: | - pip3 install cmakelang - cd compilers/concrete-compiler/compiler - ./scripts/format_cmake.sh - - name: Format with black (Python) - run: | - cd compilers/concrete-compiler/compiler - pip install -r lib/Bindings/Python/requirements_dev.txt - make check-python-format - - name: Lint with pylint (Python) - run: | - cd compilers/concrete-compiler/compiler - # compiler requirements to lint - pip install numpy - make python-lint - - CheckLicense: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Check if sources include the license header - run: .github/workflows/scripts/check_for_license.sh diff --git a/.github/workflows/compiler_macos_build_and_test.yml b/.github/workflows/compiler_macos_build_and_test.yml deleted file mode 100644 index b44ec01481..0000000000 --- a/.github/workflows/compiler_macos_build_and_test.yml +++ /dev/null @@ -1,104 +0,0 @@ -# Perform a build on MacOS platform with M1 chip. -name: Compiler - Build and Test (MacOS) - -on: - workflow_call: - workflow_dispatch: - secrets: - CONCRETE_CI_SSH_PRIVATE: - required: true - CONCRETE_ACTIONS_TOKEN: - required: true - -concurrency: - group: compiler_macos_build_and_test-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - BuildAndTestMacOS: - strategy: - # if a failure happens, we want to know if it's specific - # to the architecture or the operating system - fail-fast: false - matrix: - runson: ["aws-mac1-metal", "aws-mac2-metal"] - runs-on: ${{ matrix.runson }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Install Deps - run: | - brew install ninja ccache - pip3.10 install numpy pybind11==2.8 wheel delocate - pip3.10 install pytest - pip3.10 install mypy - - - name: Cache compilation (push) - if: github.event_name == 'push' - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - path: /Users/runner/Library/Caches/ccache - key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - - name: Cache compilation (pull_request) - if: github.event_name == 'pull_request' - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - path: /Users/runner/Library/Caches/ccache - key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.event.pull_request.base.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - - name: Get tmpdir path - if: github.event_name == 'push' - id: tmpdir-path - run: echo "::set-output name=TMPDIR_PATH::$TMPDIR" - - # We do run run-check-tests as part of the build, as they aren't that costly - # and will at least give minimum confidence that the compiler works in PRs - - name: Build - run: | - set -e - cd compilers/concrete-compiler/compiler - echo "Debug: ccache statistics (prior to the build):" - ccache -s - make Python3_EXECUTABLE=$(which python3.10) all run-check-tests python-package - echo "Debug: ccache statistics (after the build):" - ccache -s - - - name: Enable complete tests on push to main - if: github.ref == 'refs/heads/main' - run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV - - - name: Enable minimal tests otherwise - if: github.ref != 'refs/heads/main' - run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV - - - name: Test - run: | - set -e - export KEY_CACHE_DIRECTORY=$(mktemp -d)/KeySetCache - echo "KEY_CACHE_DIRECTORY=$KEY_CACHE_DIRECTORY" >> "${GITHUB_ENV}" - mkdir $KEY_CACHE_DIRECTORY - - cd compilers/concrete-compiler/compiler - echo "Debug: ccache statistics (prior to the tests):" - ccache -s - export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF - pip3.10 install build/wheels/*macosx*.whl - make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE=$(which python3.10) run-tests - echo "Debug: ccache statistics (after the tests):" - ccache -s - - - name: Cleanup host - if: success() || failure() - run: | - rm -rf $KEY_CACHE_DIRECTORY diff --git a/.github/workflows/concrete_compiler_benchmark.yml b/.github/workflows/concrete_compiler_benchmark.yml new file mode 100644 index 0000000000..e405c1002d --- /dev/null +++ b/.github/workflows/concrete_compiler_benchmark.yml @@ -0,0 +1,166 @@ +name: concrete-compiler benchmark linux-cpu + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_benchmark.yml + - compilers/** + - backends/** + - tools/** + push: + branches: + - 'main' + - 'release/*' + +env: + DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_compiler_benchmark_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-bench + + build-and-run-benchmarks: + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Checkout concrete + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + fetch-depth: 0 + - name: Set up home + # "Install rust" step require root user to have a HOME directory which is not set. + run: | + echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + - name: Setup rust toolchain for concrete-cpu + uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + - name: Build compiler benchmarks + run: | + set -e + git config --global --add safe.directory '*' + cd compilers/concrete-compiler/compiler + make BINDINGS_PYTHON_ENABLED=OFF build-benchmarks + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Build compiler benchmark finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + - name: Run compiler benchmarks + run: | + set -e + cd compilers/concrete-compiler/compiler + make run-cpu-benchmarks + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Run compiler benchmark finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + - name: Upload raw results artifact + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: compiler-benchmarks-result + path: compilers/concrete-compiler/compiler/benchmarks_results.json + + parse-and-send-results: + name: Parse and send results + needs: [setup-instance, build-and-run-benchmarks] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Download compiler-benchmarks-result + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: compiler-benchmarks-result + - name: Parse results + shell: bash + run: | + COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" + COMMIT_HASH="$(git describe --tags --dirty)" + # TODO output setup-instance (https://github.com/zama-ai/slab-github-runner/issues/38) + python3 ./ci/benchmark_parser.py benchmarks_results.json parsed_benchmark_results.json \ + --database compiler_benchmarks \ + --hardware "hpc7a.96xlarge" \ + --project-version ${COMMIT_HASH} \ + --branch ${{ github.ref_name }} \ + --commit-date ${COMMIT_DATE} \ + --bench-date "$(date --iso-8601=seconds)" \ + --throughput + - name: Upload parsed results artifact + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: compiler-benchmarks-parsed-result + path: parsed_benchmark_results.json + - name: Checkout Slab repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: zama-ai/slab + path: slab + token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} + - name: Send data to Slab + shell: bash + run: | + echo "Computing HMac on downloaded artifact" + SIGNATURE="$(slab/scripts/hmac_calculator.sh parsed_benchmark_results.json '${{ secrets.JOB_SECRET }}')" + echo "Sending results to Slab..." + curl -v -k \ + -H "Content-Type: application/json" \ + -H "X-Slab-Repository: ${{ github.repository }}" \ + -H "X-Slab-Command: store_data" \ + -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ + -d @parsed_benchmark_results.json \ + ${{ secrets.SLAB_URL }} + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "parse-and-send-results finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + name: Teardown instance + needs: [ setup-instance, parse-and-send-results ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/compiler_publish_docker_images.yml b/.github/workflows/concrete_compiler_publish_docker_images.yml similarity index 55% rename from .github/workflows/compiler_publish_docker_images.yml rename to .github/workflows/concrete_compiler_publish_docker_images.yml index 30c015228b..8ee1fb1649 100644 --- a/.github/workflows/compiler_publish_docker_images.yml +++ b/.github/workflows/concrete_compiler_publish_docker_images.yml @@ -1,69 +1,62 @@ -# Build and publish Docker images for different applications using AWS EC2. -name: Compiler - Docker images build & publish +name: concrete-compiler publish docker images on: workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - matrix_item: - description: 'Build matrix item' - type: string + pull_request: -# concurrency: -# group: compiler_publish_docker_images-${{ github.ref }} -# cancel-in-progress: true + push: + paths: + - .github/workflows/concrete_compiler_publish_docker_images.yml + branches: + - 'main' + - 'force-docker-images' env: - THIS_FILE: .github/workflows/compiler_publish_docker_images.yml + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + THIS_FILE: .github/workflows/concrete_compiler_publish_docker_images.yml -jobs: - BuildAndPushDockerImages: - needs: [BuildAndPublishHPXDockerImage, BuildAndPublishCUDADockerImage] - name: Build & Publish Docker Images - runs-on: ${{ github.event.inputs.runner_name }} - strategy: - matrix: - include: - - name: test-env - image: ghcr.io/zama-ai/concrete-compiler - dockerfile: docker/Dockerfile.concrete-compiler-env +concurrency: + group: concrete_compiler_publich_docker_images + cancel-in-progress: true +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-test + + compiler-image: + needs: [setup-instance, hpx-image, cuda-image] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + env: + image: ghcr.io/zama-ai/concrete-compiler + dockerfile: docker/Dockerfile.concrete-compiler-env steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: + fetch-depth: 0 submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - name: Login to Registry run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io - # label was initially a need from the frontend CI - name: Build Image run: | DOCKER_BUILDKIT=1 docker build --no-cache \ - --label "commit-sha=${{ github.sha }}" -t ${{ matrix.image }} -f ${{ matrix.dockerfile }} . - + --label "commit-sha=${{ github.sha }}" -t ${{ env.image }} -f ${{ env.dockerfile }} . # disabled because of https://github.com/aquasecurity/trivy/discussions/7668 # - name: Run Trivy vulnerability scanner # uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0 @@ -74,47 +67,43 @@ jobs: # ignore-unfixed: true # vuln-type: 'os,library' # severity: 'CRITICAL,HIGH' - - name: Tag and Publish Image run: | - docker image tag ${{ matrix.image }} ${{ matrix.image }}:${{ github.sha }} - docker image push ${{ matrix.image }}:latest - docker image push ${{ matrix.image }}:${{ github.sha }} - + docker image tag ${{ env.image }} ${{ env.image }}:${{ github.sha }} + docker image push ${{ env.image }}:latest + docker image push ${{ env.image }}:${{ github.sha }} - name: Tag and Publish Release Image if: startsWith(github.ref, 'refs/tags/v') run: | - docker image tag ${{ matrix.image }} ${{ matrix.image }}:${{ github.ref_name }} - docker image push ${{ matrix.image }}:${{ github.ref_name }} - - BuildAndPublishHPXDockerImage: - name: Build & Publish HPX Docker Image + docker image tag ${{ env.image }} ${{ env.image }}:${{ github.ref_name }} + docker image push ${{ env.image }}:${{ github.ref_name }} + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "compiler-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + hpx-image: runs-on: ${{ github.event.inputs.runner_name }} env: - IMAGE: ghcr.io/zama-ai/hpx - + image: ghcr.io/zama-ai/hpx + dockerfile: docker/Dockerfile.hpx-env steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 - - - name: Set up env - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - name: Get changed files id: changed-files uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24 - - name: Login id: login - if: contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.hpx-env') || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE) + if: contains(steps.changed-files.outputs.modified_files, '${{ env.dockerfile }}') || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE) run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io - - name: Build if: ${{ steps.login.conclusion != 'skipped' }} - run: docker build -t "${IMAGE}" -f docker/Dockerfile.hpx-env . - + run: docker build -t "${{ env.image }}" -f ${{ env.dockerfile }} . # disabled because of https://github.com/aquasecurity/trivy/discussions/7668 # - name: Run Trivy vulnerability scanner # if: ${{ steps.login.conclusion != 'skipped' }} @@ -126,16 +115,21 @@ jobs: # ignore-unfixed: true # vuln-type: 'os,library' # severity: 'CRITICAL,HIGH' - - name: Publish if: ${{ steps.login.conclusion != 'skipped' }} - run: docker push "${IMAGE}:latest" - - BuildAndPublishCUDADockerImage: - name: Build & Publish CUDA Docker Image + run: docker push "${{ env.image }}:latest" + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "hpx-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + cuda-image: runs-on: ${{ github.event.inputs.runner_name }} env: - IMAGE: ghcr.io/zama-ai/cuda + image: ghcr.io/zama-ai/cuda strategy: matrix: include: @@ -145,20 +139,16 @@ jobs: - name: cuda-11-8 tag: 11-8 dockerfile: docker/Dockerfile.cuda-118-env - steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 - - name: Set up env run: | echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - name: Get changed files id: changed-files uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24 - - name: Login id: login # from the docs: The jobs.<job_id>.if condition is evaluated before jobs.<job_id>.strategy.matrix is applied. So we can't just use matrix.dockerfile @@ -166,26 +156,30 @@ jobs: # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idif if: contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.cuda-118-env') || contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.cuda-123-env') || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE) run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io - - name: Build Tag and Publish if: ${{ steps.login.conclusion != 'skipped' }} run: | - docker build -t "${IMAGE}" -f ${{ matrix.dockerfile }} . - docker image tag "${IMAGE}" "${IMAGE}:${{ matrix.tag }}" - docker push "${IMAGE}:${{ matrix.tag }}" - + docker build -t "${{ env.image }}" -f ${{ matrix.dockerfile }} . + docker image tag "${{ env.image }}" "${{ env.image }}:${{ matrix.tag }}" + docker push "${{ env.image }}:${{ matrix.tag }}" # disabled because of https://github.com/aquasecurity/trivy/discussions/7668 # - name: Run Trivy vulnerability scanner # if: ${{ steps.login.conclusion != 'skipped' }} # uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0 # with: - # image-ref: '${{ env.IMAGE }}' + # image-ref: '${{ env.image }}' # format: 'table' # exit-code: '1' # ignore-unfixed: true # vuln-type: 'os,library' # severity: 'CRITICAL,HIGH' - - name: Push Latest Image if: ${{ steps.login.conclusion != 'skipped' && matrix.tag == '11-8' }} - run: docker push "${IMAGE}:latest" + run: docker push "${{ env.image }}:latest" + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "cuda-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_compiler_test_cpu.yml b/.github/workflows/concrete_compiler_test_cpu.yml new file mode 100644 index 0000000000..c71e67b6c2 --- /dev/null +++ b/.github/workflows/concrete_compiler_test_cpu.yml @@ -0,0 +1,180 @@ +name: concrete-compiler test linux-cpu + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_test_cpu.yml + - compilers/** + - backends/concrete-cpu/** + - tools/** + push: + branches: + - 'main' + - 'release/*' + +env: + DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_compiler_test_cpu_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-test + + format-and-lint: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Format with clang-format (Cpp) + run: | + sudo apt install moreutils + cd compilers/concrete-compiler/compiler + ./scripts/format_cpp.sh + - name: Format with cmake-format (Cmake) + run: | + pip3 install cmakelang + cd compilers/concrete-compiler/compiler + ./scripts/format_cmake.sh + - name: Format with black (Python) + run: | + cd compilers/concrete-compiler/compiler + pip install -r lib/Bindings/Python/requirements_dev.txt + make check-python-format + - name: Lint with pylint (Python) + run: | + cd compilers/concrete-compiler/compiler + # compiler requirements to lint + pip install numpy + make python-lint + - name: Check if sources include the license header + run: .github/workflows/scripts/check_for_license.sh + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "format-and-lint finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + build-and-run-test: + needs: [ setup-instance ] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Fetch repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + fetch-depth: 0 + - name: Create build dir + run: mkdir build + - name: Build compiler + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + id: build-compiler + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + -v ${{ github.workspace }}/wheels:/wheels + shell: bash + run: | + set -e + cd /concrete/compilers/concrete-compiler/compiler + rm -rf /build/* + make DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all + echo "Debug: ccache statistics (after the build):" + ccache -s + - name: Check compiler dialects docs is up to date + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + id: build-compiler-docs + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + -v ${{ github.workspace }}/wheels:/wheels + shell: bash + run: | + set -e + cd /build/tools/concretelang/docs/concretelang/ + sed -i -e 's/\[TOC\]//' *Dialect.md + for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done; + - name: Enable complete tests on push to main + if: github.ref == 'refs/heads/main' + run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV + - name: Enable minimal tests otherwise + if: github.ref != 'refs/heads/main' + run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV + - name: Run compiler tests + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + shell: bash + run: | + set -e + cd /concrete/compilers/concrete-compiler/compiler + mkdir -p /tmp/concrete_compiler/gpu_tests/ + pip install pytest + make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + needs: [ setup-instance, build-and-run-test ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_compiler_test_cpu_distributed.yml b/.github/workflows/concrete_compiler_test_cpu_distributed.yml new file mode 100644 index 0000000000..f09ca686b6 --- /dev/null +++ b/.github/workflows/concrete_compiler_test_cpu_distributed.yml @@ -0,0 +1,109 @@ +name: concrete-compiler test linux-cpu-distributed + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_test_cpu_distributed.yml + - compilers/concrete-compiler/** + push: + branches: + - 'main' + - 'release/*' + +env: + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_compiler_test_cpu_distributed_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: slurm-cluster + + build-and-run-test: + # The distributed-ci runner is registered on the instance configured in the slurm-cluster profile. + # It's why we need to setup-instance + needs: setup-instance + runs-on: distributed-ci + steps: + - name: Instance cleanup + run: | + sudo rm -rf /home/ubuntu/actions-runner/_work/concrete/concrete + mkdir -p /home/ubuntu/actions-runner/_work/concrete/concrete + docker system prune -af + + - name: Fetch repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + submodules: recursive + + - name: Set up home + # "Install rust" step require root user to have a HOME directory which is not set. + run: | + echo "HOME=/shared" >> "${GITHUB_ENV}" + + - name: Setup rust toolchain for concrete-cpu + uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + + - name: Build end-to-end distributed test + run: | + cd compilers/concrete-compiler/compiler + rm -rf /shared/build + make HPX_DIR=/shared/hpx install-hpx-from-source + make HPX_DIR=/shared/hpx BUILD_DIR=/shared/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON BINDINGS_PYTHON_ENABLED=OFF build-end-to-end-tests + + - name: Run end-to-end distributed test + run: | + cd compilers/concrete-compiler/compiler + rm -rf /shared/KeyCache + make BUILD_DIR=/shared/build KEY_CACHE_DIRECTORY=/shared/KeyCache run-end-to-end-distributed-tests + + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + needs: [ setup-instance, build-and-run-test ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_compiler_test_gpu.yml b/.github/workflows/concrete_compiler_test_gpu.yml new file mode 100644 index 0000000000..a164a124c8 --- /dev/null +++ b/.github/workflows/concrete_compiler_test_gpu.yml @@ -0,0 +1,82 @@ +name: concrete-compiler test linux-gpu + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_test_gpu.yml + - compilers/** + - backends/concrete-cuda/** + - tools/** + push: + branches: + - 'main' + - 'release/*' + +env: + DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + CUDA_PATH: /usr/local/cuda-11.8 + +concurrency: + group: concrete_compiler_test_gpu_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: gpu-test + + build-and-test: + runs-on: ${{ github.event.inputs.runner_name }} + if: ${{ !cancelled() }} + steps: + - name: Fetch repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + submodules: recursive + - name: Create build dir + run: mkdir build + - name: Build and test compiler + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + id: build-compiler + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + --gpus all + shell: bash + run: | + set -e + cd /concrete/compilers/concrete-compiler/compiler + rm -rf /build/* + mkdir -p /tmp/concrete_compiler/gpu_tests/ + make BINDINGS_PYTHON_ENABLED=OFF Python3_EXECUTABLE=$PYTHON_EXEC CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} run-end-to-end-tests-gpu + echo "Debug: ccache statistics (after the build):" + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_compiler_test_macos_cpu.yml b/.github/workflows/concrete_compiler_test_macos_cpu.yml new file mode 100644 index 0000000000..b5d9c5c30e --- /dev/null +++ b/.github/workflows/concrete_compiler_test_macos_cpu.yml @@ -0,0 +1,89 @@ +name: concrete-compiler test macos-cpu + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_test_macos_cpu.yml + - compilers/** + - backends/** + - tools/** + push: + branches: + - 'main' + - 'release/*' + +env: + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_compiler_test_macos_cpu_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + build-and-test: + strategy: + # if a failure happens, we want to know if it's specific + # to the architecture or the operating system + fail-fast: false + matrix: + runson: ["aws-mac1-metal", "aws-mac2-metal"] + python-version: ["3.10"] + runs-on: ${{ matrix.runson }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + fetch-depth: 0 + - name: Setup rust toolchain for concrete-cpu + uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + - name: Set python variables + run: | + set -e + echo "PIP=${{ format('pip{0}', matrix.python-version) }}" >> "${GITHUB_ENV}" + echo "PYTHON=${{ format('python{0}', matrix.python-version) }}" >> "${GITHUB_ENV}" + echo "PYTHON_EXEC=$(which ${{ format('python{0}', matrix.python-version) }})" >> "${GITHUB_ENV}" + - name: Install dependencies + run: | + brew install ninja ccache + ${PIP} install pytest + - name: Build compiler + run: | + set -e + cd compilers/concrete-compiler/compiler + echo "Debug: ccache statistics (prior to the build):" + ccache -s + make Python3_EXECUTABLE=$PYTHON_EXEC all + echo "Debug: ccache statistics (after the build):" + ccache -s + - name: Enable complete tests on push to main + if: github.ref == 'refs/heads/main' + run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV + - name: Enable minimal tests otherwise + if: github.ref != 'refs/heads/main' + run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV + - name: Create keyset cache directory + run: | + export KEY_CACHE_DIRECTORY=$(mktemp -d)/KeySetCache + echo "KEY_CACHE_DIRECTORY=$KEY_CACHE_DIRECTORY" >> "${GITHUB_ENV}" + mkdir $KEY_CACHE_DIRECTORY + - name: Test + run: | + set -e + cd compilers/concrete-compiler/compiler + export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF + make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE=$PYTHON_EXEC run-tests + - name: Cleanup host + if: success() || failure() + run: | + rm -rf $KEY_CACHE_DIRECTORY + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_cpu_test.yml b/.github/workflows/concrete_cpu_test.yml index 40a1ed2549..8868da4d7e 100644 --- a/.github/workflows/concrete_cpu_test.yml +++ b/.github/workflows/concrete_cpu_test.yml @@ -1,17 +1,28 @@ -name: Concrete CPU - Tests +name: concrete-cpu test on: - workflow_call: workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_cpu_test.yml + - backends/concrete-cpu/** + push: + branches: + - 'main' + - 'release/*' concurrency: - group: concrete_cpu_test-${{ github.ref }} + group: concrete_cpu_test_${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: CARGO_TERM_COLOR: always jobs: tests-linux: + strategy: + fail-fast: false + matrix: + runson: ["ubuntu-20.04", "aws-mac1-metal", "aws-mac2-metal"] runs-on: ubuntu-20.04 env: RUSTFLAGS: -D warnings @@ -57,39 +68,3 @@ jobs: run: | cd backends/concrete-cpu/implementation cargo test --no-fail-fast --all-targets --features=nightly - - tests-mac_x86: - runs-on: macos-11 - env: - RUSTFLAGS: -D warnings - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Download cargo cache - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 - - - name: Tests - run: | - cd backends/concrete-cpu/implementation - cargo test --no-fail-fast --all-targets - - tests-mac-m1: - runs-on: "aws-mac2-metal" - env: - RUSTFLAGS: -D warnings - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Download cargo cache - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 - - - name: Tests - run: | - cd backends/concrete-cpu/implementation - cargo test --no-fail-fast --all-targets diff --git a/.github/workflows/concrete_ml_test.yml b/.github/workflows/concrete_ml_test.yml new file mode 100644 index 0000000000..a3ccc346a1 --- /dev/null +++ b/.github/workflows/concrete_ml_test.yml @@ -0,0 +1,135 @@ +name: concrete-ml test + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_ml_test.yml + - frontends/concrete-python/** + push: + branches: + - 'main' + - 'release/*' + +env: + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_ml_test_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-test + + build-and-run-tests: + strategy: + matrix: + python-version: ["3.8"] + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - name: Set release version + run: echo "__version__ = \"$(date +"%Y.%m.%d")\"" >| frontends/concrete-python/version.txt + - name: Expose release version from Python + run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py + + - name: Create build directory + run: mkdir build + + - name: Build wheel + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + id: build-compiler-bindings + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + shell: bash + run: | + set -e + rm -rf /build/* + + export PYTHON=${{ format('python{0}', matrix.python-version) }} + echo "Using $PYTHON" + + cd /concrete/frontends/concrete-python + make PYTHON=$PYTHON venv + source .venv/bin/activate + + cd /concrete/compilers/concrete-compiler/compiler + make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which $PYTHON) python-bindings + + echo "Debug: ccache statistics (after the build):" + ccache -s + + cd /concrete/frontends/concrete-python + + export COMPILER_BUILD_DIRECTORY="/build" + make whl + + deactivate + - name: Setup Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: ${{ matrix.python-version }} + - name: ML Tests + run: | + export CONCRETE_PYTHON_WHEEL=$(pwd)/frontends/concrete-python/dist/*manylinux*.whl + apt update + apt install git git-lfs -y + pip install poetry==1.7.1 + ./ci/scripts/test_cml.sh --use-wheel $CONCRETE_PYTHON_WHEEL --verbose + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-run-tests finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, build-and-run-tests ] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_ml_tests.yml b/.github/workflows/concrete_ml_tests.yml deleted file mode 100644 index 47f7e6e695..0000000000 --- a/.github/workflows/concrete_ml_tests.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: Concrete ML Tests -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - - -env: - DOCKER_IMAGE: ghcr.io/zama-ai/concrete-compiler - -jobs: - linux-x86: - strategy: - matrix: - python-version: ["3.8"] - - runs-on: ${{ github.event.inputs.runner_name }} - steps: - - name: Log instance configuration - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "User Inputs: ${{ inputs.user_inputs }}" - - - name: Set up GitHub environment - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Set release version - run: echo "__version__ = \"$(date +"%Y.%m.%d")\"" >| frontends/concrete-python/version.txt - - - name: Expose release version from Python - run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py - - - name: Create build directory - run: mkdir build - - - name: Build wheel - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - id: build-compiler-bindings - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - ${{ env.DOCKER_GPU_OPTION }} - shell: bash - run: | - set -e - rm -rf /build/* - - export PYTHON=${{ format('python{0}', matrix.python-version) }} - echo "Using $PYTHON" - - cd /concrete/frontends/concrete-python - make PYTHON=$PYTHON venv - source .venv/bin/activate - - cd /concrete/compilers/concrete-compiler/compiler - make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings - - echo "Debug: ccache statistics (after the build):" - ccache -s - - cd /concrete/frontends/concrete-python - - export COMPILER_BUILD_DIRECTORY="/build" - make whl - - deactivate - - - name: Setup Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 - with: - python-version: ${{ matrix.python-version }} - - - name: ML Tests - run: | - export CONCRETE_PYTHON_WHEEL=$(pwd)/frontends/concrete-python/dist/*manylinux*.whl - apt update - apt install git git-lfs -y - pip install poetry==1.7.1 - ./ci/scripts/test_cml.sh --use-wheel $CONCRETE_PYTHON_WHEEL --verbose diff --git a/.github/workflows/optimizer.yml b/.github/workflows/concrete_optimizer.yml similarity index 74% rename from .github/workflows/optimizer.yml rename to .github/workflows/concrete_optimizer.yml index 48e86b8a6a..96292a56eb 100644 --- a/.github/workflows/optimizer.yml +++ b/.github/workflows/concrete_optimizer.yml @@ -1,73 +1,72 @@ -name: Optimizer - Tests +name: concrete-optimizer test on: - workflow_call: workflow_dispatch: - secrets: - CONCRETE_CI_SSH_PRIVATE: - required: true - CONCRETE_ACTIONS_TOKEN: - required: true + pull_request: + paths: + - .github/workflows/concrete_optimizer.yml + - compilers/concrete-optimizer/** + - backends/** + - tools/** + push: + branches: + - 'main' + - 'release/*' + +env: + CARGO_TERM_COLOR: always + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} concurrency: - group: optimizer-${{ github.ref }} + group: concrete_optimizer-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} -env: - CARGO_TERM_COLOR: always jobs: tests: strategy: matrix: - os: [ubuntu-20.04, macos-11] - runs-on: ${{ matrix.os }} + runson: ["ubuntu-22.04", "aws-mac1-metal", "aws-mac2-metal"] + runs-on: ${{ matrix.runson }} env: RUSTFLAGS: -D warnings steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: "Setup" + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Setup uses: ./.github/workflows/optimizer_setup - with: - ssh_private_key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} - - name: Formatting run: | + cargo --version cd compilers/concrete-optimizer cargo fmt --check - - name: Build run: | cd compilers/concrete-optimizer cargo build --release --all-targets - - name: Lint run: | cd compilers/concrete-optimizer cargo clippy --release --all-targets - - name: Tests - if: matrix.os == 'ubuntu-20.04' run: | cd compilers/concrete-optimizer cargo test --release --no-fail-fast --all-targets make -C concrete-optimizer-cpp test-ci benchmarks: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: "Setup" + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Setup uses: ./.github/workflows/optimizer_setup - with: - ssh_private_key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} - - name: Run benchmark run: | cd compilers/concrete-optimizer cargo bench -p v0-parameters -- --output-format bencher | tee bench_result.txt - - name: Download PR base benchmark data if: ${{ github.event_name == 'pull_request' }} # for artifacts restrictions see https://github.com/actions/download-artifact/issues/3 @@ -81,7 +80,6 @@ jobs: name: ${{ runner.os }}-benchmark if_no_artifact_found: warn path: ./benchmark - - name: Save benchmark result to file uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 # v1.20.3 with: @@ -94,7 +92,6 @@ jobs: comment-always: true # Enable Job Summary for PRs summary-always: true - - name: Upload benchmark data uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: diff --git a/.github/workflows/concrete_python_benchmark.yml b/.github/workflows/concrete_python_benchmark.yml index b62b99836b..34fcf6a7df 100644 --- a/.github/workflows/concrete_python_benchmark.yml +++ b/.github/workflows/concrete_python_benchmark.yml @@ -1,17 +1,22 @@ -name: Concrete Python Benchmark +name: concrete-python benchmark linux-cpu on: workflow_dispatch: schedule: - cron: "0 1 * * SAT" - + pull_request: + paths: + - .github/workflows/concrete_pyhon_benchmark.yml + push: + branches: + - 'main' + - 'release/*' env: DOCKER_IMAGE: ghcr.io/zama-ai/concrete-compiler GLIB_VER: 2_28 jobs: setup-instance: - name: Setup Instance runs-on: ubuntu-latest outputs: runner-name: ${{ steps.start-instance.outputs.label }} @@ -25,10 +30,9 @@ jobs: slab-url: ${{ secrets.SLAB_BASE_URL }} job-secret: ${{ secrets.JOB_SECRET }} backend: aws - profile: m7i-cpu-bench + profile: cpu-bench concrete-python-benchmarks: - name: Run Concrete Python Benchmarks needs: setup-instance runs-on: ${{ needs.setup-instance.outputs.runner-name }} steps: @@ -49,22 +53,14 @@ jobs: options: >- -v ${{ github.workspace }}:/concrete -v ${{ github.workspace }}/build:/build - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - ${{ env.DOCKER_GPU_OPTION }} shell: bash run: | set -e - - rustup toolchain install nightly-2024-09-30 - pip install mypy rm -rf /build/* export PYTHON=${{ format('python{0}', matrix.python-version) }} echo "Using $PYTHON" - dnf -y install graphviz graphviz-devel - cd /concrete/frontends/concrete-python make PYTHON=$PYTHON venv source .venv/bin/activate @@ -72,13 +68,11 @@ jobs: cd /concrete/compilers/concrete-compiler/compiler make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings - echo "Debug: ccache statistics (after the build):" - ccache -s - cd /concrete/frontends/concrete-python export COMPILER_BUILD_DIRECTORY="/build" - export PROGRESS_MACHINE_NAME="m7i.48xlarge" + # TODO output setup-instance (https://github.com/zama-ai/slab-github-runner/issues/38) + export PROGRESS_MACHINE_NAME="hpc7a.96xlarge" make benchmark make process-benchmark-results-for-grafana @@ -110,7 +104,6 @@ jobs: ${{ secrets.SLAB_URL }} teardown-instance: - name: Teardown Instance if: ${{ always() && needs.setup-instance.result != 'skipped' }} needs: [ setup-instance, concrete-python-benchmarks ] runs-on: ubuntu-latest @@ -124,3 +117,10 @@ jobs: slab-url: ${{ secrets.SLAB_BASE_URL }} job-secret: ${{ secrets.JOB_SECRET }} label: ${{ needs.setup-instance.outputs.runner-name }} + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_python_checks.yml b/.github/workflows/concrete_python_checks.yml deleted file mode 100644 index fa2f908b4d..0000000000 --- a/.github/workflows/concrete_python_checks.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Concrete Python Checks - -on: - workflow_call: - -jobs: - Checks: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Install Platform Dependencies - run: | - sudo apt install -y graphviz libgraphviz-dev - - name: Pre-Commit Checks - run: | - ./frontends/concrete-python/scripts/checks/checks.sh diff --git a/.github/workflows/concrete_python_test_macos.yml b/.github/workflows/concrete_python_test_macos.yml index 79e9409218..9c12d56386 100644 --- a/.github/workflows/concrete_python_test_macos.yml +++ b/.github/workflows/concrete_python_test_macos.yml @@ -1,40 +1,37 @@ -name: Concrete Python Tests (macOS) +name: concrete-python tests macos on: - workflow_call: workflow_dispatch: - secrets: - CONCRETE_CI_SSH_PRIVATE: - required: true - CONCRETE_ACTIONS_TOKEN: - required: true + pull_request: + paths: + - .github/workflows/concrete_python_tests_macos.yml + push: + branches: + - 'main' + - 'release/*' concurrency: - group: concrete_python_tests_macos-${{ github.ref }} + group: concrete_python_tests_macos_${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - BuildConcreteCompilerAndTestConcretePythonInMacOS: + concrete-python-test-pytest: strategy: fail-fast: false matrix: machine: ["aws-mac1-metal", "aws-mac2-metal"] - runs-on: ${{ matrix.machine }} steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Install OS Dependencies + fetch-depth: 0 + - name: Install build dependencies run: | brew install ninja ccache - - name: Setup rust toolchain for concrete-cpu uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Cache Compilation (push) + - name: Cache compilation (push) if: github.event_name == 'push' uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -42,8 +39,7 @@ jobs: key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.sha }} restore-keys: | ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - - name: Cache Compilation (pull_request) + - name: Cache compilation (pull_request) if: github.event_name == 'pull_request' uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -51,88 +47,72 @@ jobs: key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.event.pull_request.base.sha }} restore-keys: | ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - - name: Get tmpdir path - if: github.event_name == 'push' - id: tmpdir-path - run: echo "::set-output name=TMPDIR_PATH::$TMPDIR" - - - name: Build + - name: Prepare build environment run: | set -e - cd frontends/concrete-python - + cd $GITHUB_WORKSPACE/frontends/concrete-python + # Setup pkg-config to find OpenBLAS (scipy need it) export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" - + rm -rf .venv python3.10 -m venv .venv - - . .venv/bin/activate - + + . $GITHUB_WORKSPACE/frontends/concrete-python/.venv/bin/activate + pip install -r requirements.dev.txt pip install -r requirements.txt - + - name: Build concrete-compiler python-bindings + run: | + $GITHUB_WORKSPACE/frontends/concrete-python .venv/bin/activate cd $GITHUB_WORKSPACE/compilers/concrete-compiler/compiler - - echo "Debug: ccache statistics (prior to the build):" - ccache -s - + ccache -z make Python3_EXECUTABLE=$(which python) python-bindings - - echo "Debug: ccache statistics (after the build):" ccache -s - - export COMPILER_BUILD_DIRECTORY=$(pwd)/build + - name: Create wheels + run: | + $GITHUB_WORKSPACE/frontends/concrete-python .venv/bin/activate cd $GITHUB_WORKSPACE/frontends/concrete-python - - rm -rf dist - mkdir -p dist - + + export COMPILER_BUILD_DIRECTORY=$GITHUB_WORKSPACE/compilers/concrete-compiler/compiler + rm -rf dist && mkdir -p dist pip wheel -v --no-deps -w dist . delocate-wheel -v dist/*macos*.whl - - deactivate + deactivate - name: Prepare test environment run: | set -e export TEST_TMP_DIR=$(mktemp -d) echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}" cd $TEST_TMP_DIR - + python3.10 -m venv .testenv . .testenv/bin/activate - + pip install $GITHUB_WORKSPACE/frontends/concrete-python/dist/*macos*.whl pip install -r $GITHUB_WORKSPACE/frontends/concrete-python/requirements.dev.txt # MacOS x86 have conflict between our OpenMP library, and one from torch # we fix it by using a single one (from torch) # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8 - + find .testenv/lib/python3.10/site-packages -not \( -path .testenv/lib/python3.10/site-packages/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1 ln -f -s $(pwd)/.testenv/lib/python3.10/site-packages/concrete/.dylibs/libomp.dylib cp -R $GITHUB_WORKSPACE/frontends/concrete-python/examples ./examples cp -R $GITHUB_WORKSPACE/frontends/concrete-python/tests ./tests - - cp $GITHUB_WORKSPACE/frontends/concrete-python/Makefile . - - name: Test + cp $GITHUB_WORKSPACE/frontends/concrete-python/Makefile . + - name: Run pytest-macos run: | - set -e - export TEST_TMP_DIR="testing_concrete_python" cd $TEST_TMP_DIR && . .testenv/bin/activate KEY_CACHE_DIRECTORY=./KeySetCache PYTEST_MARKERS="not dataflow and not graphviz" make pytest-macos - - - name: Test notebooks + - name: Run test-notebooks run: | set -e - export TEST_TMP_DIR="testing_concrete_python" cd $TEST_TMP_DIR && . .testenv/bin/activate make test-notebooks - - name: Cleanup host if: success() || failure() run: | diff --git a/.github/workflows/concrete_python_tests_linux.yml b/.github/workflows/concrete_python_tests_linux.yml index 6960926957..1832bd2c2a 100644 --- a/.github/workflows/concrete_python_tests_linux.yml +++ b/.github/workflows/concrete_python_tests_linux.yml @@ -1,69 +1,72 @@ -name: Concrete Python Tests (Linux) +name: concrete-python tests linux-cpu on: workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - -# concurrency: -# group: concrete_python_tests_linux-${{ github.ref }} -# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + pull_request: + paths: + - .github/workflows/concrete_python_tests_linux.yml + - frontends/concrete-python/** + push: + branches: + - 'main' + - 'release/*' env: DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler - CUDA_PATH: /usr/local/cuda-11.8 - GCC_VERSION: 11 - GLIB_VER: 2_28 + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_python_tests_linux_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - BuildConcreteCompilerAndTestConcretePythonInLinux: - name: Build Concrete Compiler and Test Concrete Python in Linux - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-test + + pre-commit-check: + runs-on: ubuntu-22.04 steps: - - name: Log instance configuration + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Install platform dependencies run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Set up GitHub environment + sudo apt install -y graphviz libgraphviz-dev + - name: Pre-commit Checks run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - - - name: Checkout + cd frontends/concrete-python + make venv + source .venv/bin/activate + make pcc + + build-python-bindings: + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Checkout concrete uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Create build directory + fetch-depth: 0 + - name: Create concrete build directory run: mkdir build - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Build bindings + - name: Build concrete-compiler python bindings uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} - id: build-compiler-bindings with: registry: ghcr.io image: ${{ env.DOCKER_IMAGE_TEST }} @@ -72,73 +75,58 @@ jobs: options: >- -v ${{ github.workspace }}:/concrete -v ${{ github.workspace }}/build:/build - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket shell: bash run: | - rustup toolchain install nightly-2024-09-30 - pip install mypy set -e - rm -rf /build/* + rustup toolchain install nightly-2024-09-30 dnf -y install graphviz graphviz-devel + #rm -rf /build/* + cd /concrete/frontends/concrete-python make venv source .venv/bin/activate - + cd /concrete/compilers/concrete-compiler/compiler make BUILD_DIR=/build DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$(which python3) python-bindings - + echo "Debug: ccache statistics (after the build):" ccache -s - - - name: Prepare test environment - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-python-bindings finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + - name: Create artifact archive + run: | + cd build + tar czvf artifacts.tgz lib/libConcretelangRuntime.so tools/concretelang/python_packages + - name: Upload concrete-compiler python-bindings + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - shell: bash - run: | - set -e - - dnf -y install graphviz graphviz-devel - - cd /concrete/frontends/concrete-python - make venv - - - name: Test - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} + name: concrete-compiler-python-bindings + include-hidden-files: true + retention-days: 3 + path: build/artifacts.tgz + + test-pytest: + needs: [setup-instance, build-python-bindings] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Download concrete-compiler python-bindings + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - shell: bash - run: | - set -e - - cd /concrete/frontends/concrete-python - source .venv/bin/activate - - export COMPILER_BUILD_DIRECTORY=/build - - mkdir ./KeySetCache - KEY_CACHE_DIRECTORY=./KeySetCache make pytest - - - name: Test notebooks + name: concrete-compiler-python-bindings + path: compiler-artifacts + - name: Extract artifacts archive + run: | + cd compiler-artifacts + tar xzvf artifacts.tgz + - name: Run pytest uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} with: registry: ghcr.io image: ${{ env.DOCKER_IMAGE_TEST }} @@ -146,53 +134,39 @@ jobs: password: ${{ secrets.GHCR_PASSWORD }} options: >- -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build + -v ${{ github.workspace }}/compiler-artifacts:/compiler-artifacts shell: bash run: | set -e - + export COMPILER_BUILD_DIRECTORY=/compiler-artifacts cd /concrete/frontends/concrete-python source .venv/bin/activate - - export COMPILER_BUILD_DIRECTORY=/build - - make test-notebooks - - - - name: Build bindings gpu - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ contains(inputs.instance_type, 'p3') }} - id: build-compiler-bindings-gpu + export KEY_CACHE_DIRECTORY=./key-set-cache + mkdir $KEY_CACHE_DIRECTORY + make pytest + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "test-pytest finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + test-notebooks: + needs: [setup-instance, build-python-bindings] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Download concrete-compiler python-bindings + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - --gpus all - shell: bash - run: | - set -e - rm -rf /build/* - - cd /concrete/frontends/concrete-python - make venv - source .venv/bin/activate - - cd /concrete/compilers/concrete-compiler/compiler - make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python3) CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} python-bindings - - echo "Debug: ccache statistics (after the build):" - ccache -s - - - name: Test gpu + name: concrete-compiler-python-bindings + path: compiler-artifacts + - name: Extract artifacts archive + run: | + cd compiler-artifacts + tar xzvf artifacts.tgz + - name: Run pytest uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ contains(inputs.instance_type, 'p3') }} with: registry: ghcr.io image: ${{ env.DOCKER_IMAGE_TEST }} @@ -200,19 +174,41 @@ jobs: password: ${{ secrets.GHCR_PASSWORD }} options: >- -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - --gpus all + -v ${{ github.workspace }}/compiler-artifacts:/compiler-artifacts shell: bash run: | set -e - + export COMPILER_BUILD_DIRECTORY=/compiler-artifacts cd /concrete/frontends/concrete-python - make venv source .venv/bin/activate - - export COMPILER_BUILD_DIRECTORY=/build - KEY_CACHE_DIRECTORY=/tmp/KeySetCache mkdir ./KeySetCache - make pytest-gpu - - chmod -R ugo+rwx /tmp/KeySetCache + make test-notebooks + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "test-notebooks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, test-pytest, test-notebooks ] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/docker-lint.yml b/.github/workflows/docker-lint.yml deleted file mode 100644 index 197f72a972..0000000000 --- a/.github/workflows/docker-lint.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Lint Dockerfiles - -on: - pull_request: - push: - branches: - - main - -jobs: - lint: - runs-on: ubuntu-latest - container: - image: hadolint/hadolint@sha256:27173fe25e062448490a32de410c08491c626a0bef360aa2ce5d5bdd9384b50d #2.12.0-debian - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - - name: Lint All Dockerfiles (except third_party) - run: hadolint -V `find -name "*Dockerfile*" -not -path "./third_party/*" |xargs ` diff --git a/.github/workflows/docker_compliance.yml b/.github/workflows/docker_compliance.yml new file mode 100644 index 0000000000..d5da1eda40 --- /dev/null +++ b/.github/workflows/docker_compliance.yml @@ -0,0 +1,29 @@ +name: check docker files compliance + +on: + pull_request: + paths: + - .github/workflows/docker_compliance.yml + - '**Dockerfile**' + push: + branches: + - main + - 'release/*' + +jobs: + lint: + runs-on: ubuntu-latest + container: + image: hadolint/hadolint@sha256:27173fe25e062448490a32de410c08491c626a0bef360aa2ce5d5bdd9384b50d #2.12.0-debian + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + + - name: Lint All Dockerfiles (except third_party) + run: hadolint -V `find -name "*Dockerfile*" -not -path "./third_party/*" |xargs ` + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "lint finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/linelint.yml b/.github/workflows/linelint.yml deleted file mode 100644 index 03b814c839..0000000000 --- a/.github/workflows/linelint.yml +++ /dev/null @@ -1,18 +0,0 @@ -# This job is the main jobs will dispatch build and test for every modules of our mono repo. -name: Linelint - -on: - pull_request: - push: - branches: - - 'main' - -jobs: - linelint: - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Linelint - uses: fernandrone/linelint@8136e0fa9997122d80f5f793e0bb9a45e678fbb1 # 0.0.4 - id: linelint diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e39d983fde..1a3471214f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,18 +2,18 @@ name: Main on: - pull_request: +# pull_request: push: branches: - 'main' - - 'release/*' - - 'force-docker-images' - - 'private_release/*' - tags: - - 'v[0-9]+.[0-9]+.[0-9]+*' - schedule: - # Nightly Release @ 3AM after each work day - - cron: "0 3 * * 2-6" +# - 'release/*' +# - 'force-docker-images' +# - 'private_release/*' +# tags: +# - 'v[0-9]+.[0-9]+.[0-9]+*' +# schedule: +# # Nightly Release @ 3AM after each work day +# - cron: "0 3 * * 2-6" jobs: # This jobs outputs for each modules of our mono-repo if it changed, diff --git a/.github/workflows/optimizer_setup/action.yml b/.github/workflows/optimizer_setup/action.yml index d5887aa7d7..2f4e97592e 100644 --- a/.github/workflows/optimizer_setup/action.yml +++ b/.github/workflows/optimizer_setup/action.yml @@ -1,8 +1,3 @@ -inputs: - ssh_private_key: - description: 'A ssh key to access private github repository' - required: true - runs: using: "composite" steps: @@ -10,6 +5,7 @@ runs: uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7 with: toolchain: stable + default: true - name: Download cargo cache uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 diff --git a/ci/ec2_products_cost.json b/ci/ec2_products_cost.json index 436cabb134..d43366be82 100644 --- a/ci/ec2_products_cost.json +++ b/ci/ec2_products_cost.json @@ -1,4 +1,5 @@ { + "hpc7a.96xlarge": 7.200, "m7i.48xlarge": 9.677, "m7i.metal-48xl": 9.677, "m6i.metal": 7.168, diff --git a/ci/slab.toml b/ci/slab.toml index 0a42840d02..c69399c4a1 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -1,3 +1,28 @@ +[backend.aws.cpu-test] +region = "eu-west-1" +image_id = "ami-002bdcd64b8472cf9" # Based on Ubuntu 22.4 +instance_type = "m7i.16xlarge" +security_group = ["sg-0e55cc31dfda0d8a7", ] + +[backend.aws.cpu-bench] +region = "eu-west-1" +image_id = "ami-002bdcd64b8472cf9" +instance_type = "hpc7a.96xlarge" + +[backend.aws.gpu-test] +region = "us-east-1" +image_id = "ami-0257c6ad39f902b5e" +instance_type = "p3.2xlarge" +subnet_id = "subnet-8123c9e7" +security_group= ["sg-017afab1f328af917", ] + +[backend.aws.slurm-cluster] +region = "eu-west-3" +image_id = "ami-0bb5bb9cb747b5ddd" +instance_id = "i-0e5ae2a14134d6275" +instance_type = "m6i.8xlarge" +security_group= ["sg-02dd8470fa845f31b", ] + # This is the new version of Slab that handles multi backend providers. [backend.aws.m7i-cpu-bench] region = "eu-west-1" diff --git a/compilers/concrete-compiler/compiler/Makefile b/compilers/concrete-compiler/compiler/Makefile index c67d5dd2d8..58f5de526c 100644 --- a/compilers/concrete-compiler/compiler/Makefile +++ b/compilers/concrete-compiler/compiler/Makefile @@ -479,33 +479,6 @@ else detected_OS := $(shell sh -c 'uname 2>/dev/null || echo Unknown') endif -PIP=$(Python3_EXECUTABLE) -m pip -PIP_WHEEL=$(PIP) wheel --no-deps -w $(BUILD_DIR)/wheels . -AUDIT_WHEEL_REPAIR=$(Python3_EXECUTABLE) -m auditwheel repair -w $(BUILD_DIR)/wheels - -linux-python-package: - $(PIP) install wheel auditwheel - # We need to run it twice: the first will generate the directories, so that - # the second run can find the packages via find_namespace_packages - $(PIP_WHEEL) - $(PIP_WHEEL) - GLIBC_VER=$(shell ldd --version | head -n 1 | grep -o '[^ ]*$$'|head|tr '.' '_'); \ - for PLATFORM in manylinux_$${GLIBC_VER}_x86_64 linux_x86_64; do \ - if $(AUDIT_WHEEL_REPAIR) $(BUILD_DIR)/wheels/*.whl --plat $$PLATFORM; then \ - echo Success for $$PLATFORM; \ - break; \ - else \ - echo No repair with $$PLATFORM; \ - fi \ - done - -darwin-python-package: - $(PIP) install wheel delocate - $(PIP_WHEEL) - delocate-wheel -v $(BUILD_DIR)/wheels/*macosx*.whl - -python-package: python-bindings $(OS)-python-package - @echo The python package is: $(BUILD_DIR)/wheels/*.whl install: concretecompiler install-deps $(info Install prefix set to $(INSTALL_PREFIX)) diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt b/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt index 021be4705c..199b6a5506 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt @@ -1,3 +1,4 @@ black==24.4.0 pylint==2.11.1 mypy==1.11.2 +numpy>=1.23,<2.0 diff --git a/frontends/concrete-python/Makefile b/frontends/concrete-python/Makefile index ef04545803..7322a5cf9f 100644 --- a/frontends/concrete-python/Makefile +++ b/frontends/concrete-python/Makefile @@ -30,9 +30,6 @@ CONCRETE_VERSION?="" # empty mean latest venv: $(PYTHON) -m venv .venv . .venv/bin/activate -ifeq (,$(wildcard ${RUNTIME_LIBRARY})) - $(PIP) install --extra-index-url https://pypi.zama.ai/cpu "concrete-python$(CONCRETE_VERSION)" -endif $(PIP) install -r requirements.dev.txt $(PIP) install -r requirements.extra-full.txt $(PIP) install -r requirements.txt @@ -66,6 +63,12 @@ tfhers-utils: pytest: pytest-default +debug: + echo "Printing environment" + env + echo "ls -R ${BINDINGS_DIRECTORY}" + ls -R ${BINDINGS_DIRECTORY} + pytest-default: tfhers-utils eval $(shell make silent_cp_activate) pytest tests -svv -n auto \ diff --git a/frontends/concrete-python/scripts/checks/checks.sh b/frontends/concrete-python/scripts/checks/checks.sh deleted file mode 100755 index 25a3bbb6c1..0000000000 --- a/frontends/concrete-python/scripts/checks/checks.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -ex - -cd frontends/concrete-python -make venv -source .venv/bin/activate -make pcc