diff --git a/.github/workflows/action-pin.yaml b/.github/workflows/action-pin.yaml deleted file mode 100644 index feb3cb4ba0..0000000000 --- a/.github/workflows/action-pin.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: Action Pin - -on: - pull_request: - push: - branches: - - main - -jobs: - check-action-pin: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - - name: Ensure SHA pinned actions - uses: zgosalvez/github-actions-ensure-sha-pinned-actions@0901cf7b71c7ea6261ec69a3dc2bd3f9264f893e # v3.0.12 - with: - allowlist: | - slsa-framework/slsa-github-generator diff --git a/.github/workflows/action_compliance.yaml b/.github/workflows/action_compliance.yaml new file mode 100644 index 0000000000..7fad8b0d98 --- /dev/null +++ b/.github/workflows/action_compliance.yaml @@ -0,0 +1,32 @@ +name: check action compliance + +on: + pull_request: + paths: + - .github/workflows/** + push: + branches: + - main + - 'release/*' + +jobs: + action-pin: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - name: Ensure SHA pinned actions + uses: zgosalvez/github-actions-ensure-sha-pinned-actions@0901cf7b71c7ea6261ec69a3dc2bd3f9264f893e # v3.0.12 + with: + allowlist: | + slsa-framework/slsa-github-generator + + action-lint: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - name: check-missing-teardown + run: .github/workflows/scripts/teardown-check.sh + - name: actionlint + uses: raven-actions/actionlint@01fce4f43a270a612932cb1c64d40505a029f821 # v2.0.0 diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml deleted file mode 100644 index fbd5087f24..0000000000 --- a/.github/workflows/actionlint.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Action Lint - -on: - pull_request: - push: - branches: - - main - -jobs: - action-lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - - name: actionlint - uses: raven-actions/actionlint@01fce4f43a270a612932cb1c64d40505a029f821 # v2.0.0 diff --git a/.github/workflows/check_commit_signature.yml b/.github/workflows/check_commit_signature.yml deleted file mode 100644 index ad2fa41191..0000000000 --- a/.github/workflows/check_commit_signature.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: Check Commit Signatures - -on: - pull_request: - -jobs: - check-commit-signatures: - runs-on: ubuntu-latest - steps: - - name: Check commit signatures - uses: 1Password/check-signed-commits-action@ed2885f3ed2577a4f5d3c3fe895432a557d23d52 diff --git a/.github/workflows/block_merge.yml b/.github/workflows/commit_compliance.yml similarity index 62% rename from .github/workflows/block_merge.yml rename to .github/workflows/commit_compliance.yml index 6500845837..ac14dbb27e 100644 --- a/.github/workflows/block_merge.yml +++ b/.github/workflows/commit_compliance.yml @@ -1,13 +1,10 @@ -# Check commit and PR compliance -name: Check commit message compliance +name: check commit compliance on: pull_request: - types: [opened, synchronize, reopened] jobs: - check-commit-pr: - name: Check commit and PR - runs-on: ubuntu-20.04 + format: + runs-on: ubuntu-latest steps: - name: Check first line uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee # v2.0.0 @@ -19,3 +16,13 @@ jobs: excludeTitle: 'true' # optional: this excludes the title of a pull request checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true + - name: checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: linelint + uses: fernandrone/linelint@8136e0fa9997122d80f5f793e0bb9a45e678fbb1 # 0.0.4 + id: linelint + - name: markdown-link-check + uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1 + with: + use-quiet-mode: 'yes' + use-verbose-mode: 'yes' diff --git a/.github/workflows/compiler_benchmark.yml b/.github/workflows/compiler_benchmark.yml deleted file mode 100644 index 0b17545144..0000000000 --- a/.github/workflows/compiler_benchmark.yml +++ /dev/null @@ -1,142 +0,0 @@ -# Run benchmarks on an AWS instance for compiler and return parsed results to Slab CI bot. -name: Compiler - Performance benchmarks - -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - -# concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - CUDA_PATH: /usr/local/cuda-11.8 - GCC_VERSION: 8 - -jobs: - run-benchmarks: - name: Execute end-to-end benchmarks in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Get benchmark date - run: | - echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" - - - name: Fetch submodules - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Export specific variables (CPU) - if: ${{ !startswith(inputs.instance_type, 'p3.') }} - run: | - echo "CUDA_SUPPORT=OFF" >> "${GITHUB_ENV}" - echo "BENCHMARK_TARGET=run-cpu-benchmarks" >> "${GITHUB_ENV}" - - - name: Export specific variables (GPU) - if: ${{ startswith(inputs.instance_type, 'p3.') }} - run: | - echo "CUDA_SUPPORT=ON" >> "${GITHUB_ENV}" - echo "BENCHMARK_TARGET=run-gpu-benchmarks" >> "${GITHUB_ENV}" - echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}" - echo "$CUDA_PATH/bin" >> "${GITHUB_PATH}" - echo "LD_LIBRARY_PATH=$CUDA_PATH/lib:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}" - echo "CC=/usr/bin/gcc-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" - echo "CXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" - echo "CUDAHOSTCXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}" - echo "CUDACXX=$CUDA_PATH/bin/nvcc" >> "${GITHUB_ENV}" - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Build compiler benchmarks - run: | - set -e - git config --global --add safe.directory '*' - cd compilers/concrete-compiler/compiler - make BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} build-benchmarks - - - name: Run end-to-end benchmarks - run: | - set -e - cd compilers/concrete-compiler/compiler - make ${{ env.BENCHMARK_TARGET }} - - - name: Upload raw results artifact - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: compiler_${{ github.sha }}_raw - path: compilers/concrete-compiler/compiler/benchmarks_results.json - - - name: Parse results - shell: bash - run: | - COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" - COMMIT_HASH="$(git describe --tags --dirty)" - python3 ./ci/benchmark_parser.py compilers/concrete-compiler/compiler/benchmarks_results.json ${{ env.RESULTS_FILENAME }} \ - --database compiler_benchmarks \ - --hardware ${{ inputs.instance_type }} \ - --project-version ${COMMIT_HASH} \ - --branch ${{ github.ref_name }} \ - --commit-date ${COMMIT_DATE} \ - --bench-date "${{ env.BENCH_DATE }}" \ - --throughput - - - name: Upload parsed results artifact - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: compiler_${{ github.sha }} - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - echo "Computing HMac on downloaded artifact" - SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" - echo "Sending results to Slab..." - curl -v -k \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: store_data" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @${{ env.RESULTS_FILENAME }} \ - ${{ secrets.SLAB_URL }} diff --git a/.github/workflows/compiler_build_and_test_cpu.yml b/.github/workflows/compiler_build_and_test_cpu.yml deleted file mode 100644 index e20cc664e0..0000000000 --- a/.github/workflows/compiler_build_and_test_cpu.yml +++ /dev/null @@ -1,158 +0,0 @@ -name: Compiler - Build and Test (CPU) - -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - matrix_item: - description: 'Build matrix item' - type: string - -# concurrency: -# group: compiler_build_and_test_cpu-${{ github.ref }} -# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -env: - DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler - GLIB_VER: 2_28 - -jobs: - BuildAndTest: - name: Build and test compiler in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "Matrix item: ${{ inputs.matrix_item }}" - - - name: Set up env - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - - - name: Fetch repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Create build dir - run: mkdir build - - - name: Build compiler - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - id: build-compiler - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - ${{ env.DOCKER_GPU_OPTION }} - shell: bash - run: | - rustup toolchain install nightly-2024-09-30 - pip install mypy - set -e - cd /concrete/compilers/concrete-compiler/compiler - rm -rf /build/* - make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all - echo "Debug: ccache statistics (after the build):" - ccache -s - - - name: Build compiler Dialects docs and check diff - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - id: build-compiler-docs - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - ${{ env.DOCKER_GPU_OPTION }} - shell: bash - run: | - set -e - cd /concrete/compilers/concrete-compiler/compiler - make BUILD_DIR=/build doc - cd /build/tools/concretelang/docs/concretelang/ - sed -i -e 's/\[TOC\]//' *Dialect.md - for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done; - - - name: Enable complete tests on push to main - if: github.ref == 'refs/heads/main' - run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV - - - name: Enable minimal tests otherwise - if: github.ref != 'refs/heads/main' - run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV - - - name: Test compiler - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - ${{ env.DOCKER_GPU_OPTION }} - shell: bash - run: | - set -e - rustup toolchain install nightly-2024-09-30 - cd /concrete/compilers/concrete-compiler/compiler - pip install pytest - pip install mypy - dnf install -y libzstd libzstd-devel - sed "s/pytest/python -m pytest/g" -i Makefile - mkdir -p /tmp/concrete_compiler/gpu_tests/ - make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests - chmod -R ugo+rwx /tmp/KeySetCache - - - name: Analyze logs - run: | - cd build/gtest-parallel-logs/passed - ls -1 | xargs grep -H "WARNING RETRY" | sed -e "s/.log.*//g" | uniq -c | sed -re "s/ *([0-9]*) (.*)/::warning ::Test \2 retried \1 times/g" | cat - - # - name: Archive python package - # uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - # with: - # name: concrete-compiler.whl - # path: build/wheels/concrete_compiler-*-manylinux_{{ env.GLIB_VER }}_x86_64.whl - # retention-days: 14 diff --git a/.github/workflows/compiler_build_and_test_cpu_distributed.yml b/.github/workflows/compiler_build_and_test_cpu_distributed.yml deleted file mode 100644 index 713653b4c8..0000000000 --- a/.github/workflows/compiler_build_and_test_cpu_distributed.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Compiler - Distributed Build and Test (CPU) - -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - matrix_item: - description: 'Build matrix item' - type: string - - -env: - GLIB_VER: 2_28 - -jobs: - BuildAndTest: - name: Build and test compiler on Slurm cluster in EC2 - runs-on: distributed-ci - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "ID: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "Matrix item: ${{ inputs.matrix_item }}" - - - name: Instance cleanup - run: | - sudo rm -rf /home/ubuntu/actions-runner/_work/concrete/concrete - mkdir -p /home/ubuntu/actions-runner/_work/concrete/concrete - docker system prune -af - - - name: Fetch repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/shared" >> "${GITHUB_ENV}" - - - name: Export specific variables (CPU) - if: ${{ !startswith(inputs.instance_type, 'p3.') }} - run: | - echo "CUDA_SUPPORT=OFF" >> "${GITHUB_ENV}" - echo "DATAFLOW_EXECUTION_ENABLED=ON" >> "${GITHUB_ENV}" - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Build compiler benchmarks - run: | - set -e - git config --global --add safe.directory '*' - cd compilers/concrete-compiler/compiler - rm -rf /shared/build - make HPX_DIR=/shared/hpx install-hpx-from-source - make HPX_DIR=/shared/hpx BUILD_DIR=/shared/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} build-end-to-end-tests - - - name: Run end-to-end benchmarks - run: | - set -e - cd compilers/concrete-compiler/compiler - rm -rf /shared/KeyCache - make BUILD_DIR=/shared/build KEY_CACHE_DIRECTORY=/shared/KeyCache run-end-to-end-distributed-tests - - - name: Instance cleanup - run: | - sudo rm -rf /home/ubuntu/actions-runner/_work/concrete/concrete/* - docker system prune -af diff --git a/.github/workflows/compiler_build_and_test_gpu.yml b/.github/workflows/compiler_build_and_test_gpu.yml deleted file mode 100644 index 9879d54764..0000000000 --- a/.github/workflows/compiler_build_and_test_gpu.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: Compiler - Build and Test (GPU) - -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - matrix_item: - description: 'Build matrix item' - type: string - -# concurrency: -# group: compiler_build_and_test_gpu-${{ github.ref }} -# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -env: - DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler - CUDA_PATH: /usr/local/cuda-11.8 - GCC_VERSION: 11 - -jobs: - BuildAndTest: - name: Build and test compiler in EC2 with CUDA support - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "Matrix item: ${{ inputs.matrix_item }}" - - - name: Set up env - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - - - name: Fetch repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Create build dir - run: mkdir build - - - name: Build and test compiler - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - id: build-compiler - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - --gpus all - shell: bash - run: | - rustup toolchain install nightly-2024-09-30 - pip install mypy - set -e - cd /concrete/compilers/concrete-compiler/compiler - rm -rf /build/* - mkdir -p /tmp/concrete_compiler/gpu_tests/ - make BINDINGS_PYTHON_ENABLED=OFF CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} run-end-to-end-tests-gpu - echo "Debug: ccache statistics (after the build):" - ccache -s diff --git a/.github/workflows/compiler_format_and_linting.yml b/.github/workflows/compiler_format_and_linting.yml deleted file mode 100644 index 4057fce6b5..0000000000 --- a/.github/workflows/compiler_format_and_linting.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Compiler - Compliance - -on: - workflow_call: - workflow_dispatch: - -jobs: - FormattingAndLinting: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Format with clang-format (Cpp) - run: | - sudo apt install moreutils - cd compilers/concrete-compiler/compiler - ./scripts/format_cpp.sh - - name: Format with cmake-format (Cmake) - run: | - pip3 install cmakelang - cd compilers/concrete-compiler/compiler - ./scripts/format_cmake.sh - - name: Format with black (Python) - run: | - cd compilers/concrete-compiler/compiler - pip install -r lib/Bindings/Python/requirements_dev.txt - make check-python-format - - name: Lint with pylint (Python) - run: | - cd compilers/concrete-compiler/compiler - # compiler requirements to lint - pip install numpy - make python-lint - - CheckLicense: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Check if sources include the license header - run: .github/workflows/scripts/check_for_license.sh diff --git a/.github/workflows/compiler_macos_build_and_test.yml b/.github/workflows/compiler_macos_build_and_test.yml deleted file mode 100644 index b44ec01481..0000000000 --- a/.github/workflows/compiler_macos_build_and_test.yml +++ /dev/null @@ -1,104 +0,0 @@ -# Perform a build on MacOS platform with M1 chip. -name: Compiler - Build and Test (MacOS) - -on: - workflow_call: - workflow_dispatch: - secrets: - CONCRETE_CI_SSH_PRIVATE: - required: true - CONCRETE_ACTIONS_TOKEN: - required: true - -concurrency: - group: compiler_macos_build_and_test-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - BuildAndTestMacOS: - strategy: - # if a failure happens, we want to know if it's specific - # to the architecture or the operating system - fail-fast: false - matrix: - runson: ["aws-mac1-metal", "aws-mac2-metal"] - runs-on: ${{ matrix.runson }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Install Deps - run: | - brew install ninja ccache - pip3.10 install numpy pybind11==2.8 wheel delocate - pip3.10 install pytest - pip3.10 install mypy - - - name: Cache compilation (push) - if: github.event_name == 'push' - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - path: /Users/runner/Library/Caches/ccache - key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - - name: Cache compilation (pull_request) - if: github.event_name == 'pull_request' - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - path: /Users/runner/Library/Caches/ccache - key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.event.pull_request.base.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - - name: Get tmpdir path - if: github.event_name == 'push' - id: tmpdir-path - run: echo "::set-output name=TMPDIR_PATH::$TMPDIR" - - # We do run run-check-tests as part of the build, as they aren't that costly - # and will at least give minimum confidence that the compiler works in PRs - - name: Build - run: | - set -e - cd compilers/concrete-compiler/compiler - echo "Debug: ccache statistics (prior to the build):" - ccache -s - make Python3_EXECUTABLE=$(which python3.10) all run-check-tests python-package - echo "Debug: ccache statistics (after the build):" - ccache -s - - - name: Enable complete tests on push to main - if: github.ref == 'refs/heads/main' - run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV - - - name: Enable minimal tests otherwise - if: github.ref != 'refs/heads/main' - run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV - - - name: Test - run: | - set -e - export KEY_CACHE_DIRECTORY=$(mktemp -d)/KeySetCache - echo "KEY_CACHE_DIRECTORY=$KEY_CACHE_DIRECTORY" >> "${GITHUB_ENV}" - mkdir $KEY_CACHE_DIRECTORY - - cd compilers/concrete-compiler/compiler - echo "Debug: ccache statistics (prior to the tests):" - ccache -s - export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF - pip3.10 install build/wheels/*macosx*.whl - make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE=$(which python3.10) run-tests - echo "Debug: ccache statistics (after the tests):" - ccache -s - - - name: Cleanup host - if: success() || failure() - run: | - rm -rf $KEY_CACHE_DIRECTORY diff --git a/.github/workflows/compiler_publish_docker_images.yml b/.github/workflows/compiler_publish_docker_images.yml deleted file mode 100644 index 30c015228b..0000000000 --- a/.github/workflows/compiler_publish_docker_images.yml +++ /dev/null @@ -1,191 +0,0 @@ -# Build and publish Docker images for different applications using AWS EC2. -name: Compiler - Docker images build & publish - -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - matrix_item: - description: 'Build matrix item' - type: string - -# concurrency: -# group: compiler_publish_docker_images-${{ github.ref }} -# cancel-in-progress: true - -env: - THIS_FILE: .github/workflows/compiler_publish_docker_images.yml - -jobs: - BuildAndPushDockerImages: - needs: [BuildAndPublishHPXDockerImage, BuildAndPublishCUDADockerImage] - name: Build & Publish Docker Images - runs-on: ${{ github.event.inputs.runner_name }} - strategy: - matrix: - include: - - name: test-env - image: ghcr.io/zama-ai/concrete-compiler - dockerfile: docker/Dockerfile.concrete-compiler-env - - steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Login to Registry - run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io - - # label was initially a need from the frontend CI - - name: Build Image - run: | - DOCKER_BUILDKIT=1 docker build --no-cache \ - --label "commit-sha=${{ github.sha }}" -t ${{ matrix.image }} -f ${{ matrix.dockerfile }} . - - # disabled because of https://github.com/aquasecurity/trivy/discussions/7668 - # - name: Run Trivy vulnerability scanner - # uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0 - # with: - # image-ref: '${{ matrix.image }}' - # format: 'table' - # exit-code: '1' - # ignore-unfixed: true - # vuln-type: 'os,library' - # severity: 'CRITICAL,HIGH' - - - name: Tag and Publish Image - run: | - docker image tag ${{ matrix.image }} ${{ matrix.image }}:${{ github.sha }} - docker image push ${{ matrix.image }}:latest - docker image push ${{ matrix.image }}:${{ github.sha }} - - - name: Tag and Publish Release Image - if: startsWith(github.ref, 'refs/tags/v') - run: | - docker image tag ${{ matrix.image }} ${{ matrix.image }}:${{ github.ref_name }} - docker image push ${{ matrix.image }}:${{ github.ref_name }} - - BuildAndPublishHPXDockerImage: - name: Build & Publish HPX Docker Image - runs-on: ${{ github.event.inputs.runner_name }} - env: - IMAGE: ghcr.io/zama-ai/hpx - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - name: Set up env - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24 - - - name: Login - id: login - if: contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.hpx-env') || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE) - run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io - - - name: Build - if: ${{ steps.login.conclusion != 'skipped' }} - run: docker build -t "${IMAGE}" -f docker/Dockerfile.hpx-env . - - # disabled because of https://github.com/aquasecurity/trivy/discussions/7668 - # - name: Run Trivy vulnerability scanner - # if: ${{ steps.login.conclusion != 'skipped' }} - # uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0 - # with: - # image-ref: '${{ env.IMAGE }}' - # format: 'table' - # exit-code: '1' - # ignore-unfixed: true - # vuln-type: 'os,library' - # severity: 'CRITICAL,HIGH' - - - name: Publish - if: ${{ steps.login.conclusion != 'skipped' }} - run: docker push "${IMAGE}:latest" - - BuildAndPublishCUDADockerImage: - name: Build & Publish CUDA Docker Image - runs-on: ${{ github.event.inputs.runner_name }} - env: - IMAGE: ghcr.io/zama-ai/cuda - strategy: - matrix: - include: - - name: cuda-12-3 - tag: 12-3 - dockerfile: docker/Dockerfile.cuda-123-env - - name: cuda-11-8 - tag: 11-8 - dockerfile: docker/Dockerfile.cuda-118-env - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - name: Set up env - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24 - - - name: Login - id: login - # from the docs: The jobs..if condition is evaluated before jobs..strategy.matrix is applied. So we can't just use matrix.dockerfile - # so we have to build both images if one of the two files change, or we will have to split this into two - # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idif - if: contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.cuda-118-env') || contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.cuda-123-env') || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE) - run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io - - - name: Build Tag and Publish - if: ${{ steps.login.conclusion != 'skipped' }} - run: | - docker build -t "${IMAGE}" -f ${{ matrix.dockerfile }} . - docker image tag "${IMAGE}" "${IMAGE}:${{ matrix.tag }}" - docker push "${IMAGE}:${{ matrix.tag }}" - - # disabled because of https://github.com/aquasecurity/trivy/discussions/7668 - # - name: Run Trivy vulnerability scanner - # if: ${{ steps.login.conclusion != 'skipped' }} - # uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0 - # with: - # image-ref: '${{ env.IMAGE }}' - # format: 'table' - # exit-code: '1' - # ignore-unfixed: true - # vuln-type: 'os,library' - # severity: 'CRITICAL,HIGH' - - - name: Push Latest Image - if: ${{ steps.login.conclusion != 'skipped' && matrix.tag == '11-8' }} - run: docker push "${IMAGE}:latest" diff --git a/.github/workflows/concrete_compiler_benchmark.yml b/.github/workflows/concrete_compiler_benchmark.yml new file mode 100644 index 0000000000..cd660a016f --- /dev/null +++ b/.github/workflows/concrete_compiler_benchmark.yml @@ -0,0 +1,167 @@ +name: concrete-compiler benchmark linux-cpu + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_benchmark.yml + - compilers/** + - backends/** + - tools/** + push: + branches: + - 'main' + - 'release/*' + +env: + DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_compiler_benchmark_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-bench + + build-and-run-benchmarks: + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + outputs: + bench_date: ${{ steps.benchmark-details.outputs.bench_date }} + commit_date: ${{ steps.benchmark-details.outputs.commit_date }} + commit_hash: ${{ steps.benchmark-details.outputs.commit_hash }} + steps: + - name: Checkout concrete + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + fetch-depth: 0 + - name: Ouput benchmark details + id: benchmark-details + run: | + echo "bench_date=$(date --iso-8601=seconds)" >> "$GITHUB_OUTPUT" + echo "commit_date=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" >> "$GITHUB_OUTPUT" + echo "commit_hash=$(git describe --tags --dirty)" >> "$GITHUB_OUTPUT" + - name: Set up home + # "Install rust" step require root user to have a HOME directory which is not set. + run: | + echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + - name: Setup rust toolchain for concrete-cpu + uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + - name: Build compiler benchmarks + run: | + set -e + git config --global --add safe.directory '*' + cd compilers/concrete-compiler/compiler + make BINDINGS_PYTHON_ENABLED=OFF build-benchmarks + - name: Run compiler benchmarks + run: | + set -e + cd compilers/concrete-compiler/compiler + make run-cpu-benchmarks + - name: Upload raw results artifact + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: compiler-benchmarks-result + path: compilers/concrete-compiler/compiler/benchmarks_results.json + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-run-benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + parse-and-send-results: + name: Parse and send results + needs: [setup-instance, build-and-run-benchmarks] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Download compiler-benchmarks-result + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: compiler-benchmarks-result + - name: Parse results + shell: bash + run: | + # TODO output setup-instance (https://github.com/zama-ai/slab-github-runner/issues/38) + python3 ./ci/benchmark_parser.py benchmarks_results.json parsed_benchmark_results.json \ + --database compiler_benchmarks \ + --hardware "hpc7a.96xlarge" \ + --project-version ${{ needs.build-and-run-benchmarks.outputs.commit_hash}} \ + --branch ${{ github.ref_name }} \ + --commit-date "${{ needs.build-and-run-benchmarks.outputs.commit_date }}" \ + --bench-date "${{ needs.build-and-run-benchmarks.outputs.bench_date }}" \ + --throughput + - name: Upload parsed results artifact + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: compiler-benchmarks-parsed-result + path: parsed_benchmark_results.json + - name: Checkout Slab repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: zama-ai/slab + path: slab + token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} + - name: Send data to Slab + shell: bash + run: | + echo "Computing HMac on downloaded artifact" + SIGNATURE="$(slab/scripts/hmac_calculator.sh $parsed_benchmark_results.json '${{ secrets.JOB_SECRET }}')" + echo "Sending results to Slab..." + curl -v -k \ + -H "Content-Type: application/json" \ + -H "X-Slab-Repository: ${{ github.repository }}" \ + -H "X-Slab-Command: store_data" \ + -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ + -d @parsed_benchmark_results.json \ + ${{ secrets.SLAB_URL }} + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "parse-and-send-results finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + name: Teardown instance + needs: [ setup-instance, parse-and-send-results ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_compiler_publish_docker_images.yml b/.github/workflows/concrete_compiler_publish_docker_images.yml new file mode 100644 index 0000000000..8e7c14a93e --- /dev/null +++ b/.github/workflows/concrete_compiler_publish_docker_images.yml @@ -0,0 +1,218 @@ +name: concrete-compiler publish docker images + +on: + workflow_dispatch: + push: + branches: + - 'main' + - 'force-docker-images' + +env: + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + THIS_FILE: .github/workflows/concrete_compiler_publish_docker_images.yml + +concurrency: + group: concrete_compiler_publish_docker_images + cancel-in-progress: true + +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-test + + hpx-image: + needs: [setup-instance] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + env: + image: ghcr.io/zama-ai/hpx + dockerfile: docker/Dockerfile.hpx-env + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24 + - name: Login + id: login + if: contains(steps.changed-files.outputs.modified_files, env.dockerfile) || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE) + run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io + - name: Build + if: ${{ steps.login.conclusion != 'skipped' }} + run: docker build -t "${{ env.image }}" -f ${{ env.dockerfile }} . + # disabled because of https://github.com/aquasecurity/trivy/discussions/7668 + # - name: Run Trivy vulnerability scanner + # if: ${{ steps.login.conclusion != 'skipped' }} + # uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0 + # with: + # image-ref: '${{ env.IMAGE }}' + # format: 'table' + # exit-code: '1' + # ignore-unfixed: true + # vuln-type: 'os,library' + # severity: 'CRITICAL,HIGH' + - name: Publish + if: ${{ steps.login.conclusion != 'skipped' }} + run: docker push "${{ env.image }}:latest" + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "hpx-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + cuda-image: + needs: [setup-instance] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + env: + image: ghcr.io/zama-ai/cuda + strategy: + matrix: + include: + - name: cuda-12-3 + tag: 12-3 + dockerfile: docker/Dockerfile.cuda-123-env + - name: cuda-11-8 + tag: 11-8 + dockerfile: docker/Dockerfile.cuda-118-env + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Set up env + run: | + echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24 + - name: Login + id: login + # from the docs: The jobs..if condition is evaluated before jobs..strategy.matrix is applied. So we can't just use matrix.dockerfile + # so we have to build both images if one of the two files change, or we will have to split this into two + # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idif + if: contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.cuda-118-env') || contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.cuda-123-env') || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE) + run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io + - name: Build Tag and Publish + if: ${{ steps.login.conclusion != 'skipped' }} + run: | + docker build -t "${{ env.image }}" -f ${{ matrix.dockerfile }} . + docker image tag "${{ env.image }}" "${{ env.image }}:${{ matrix.tag }}" + docker push "${{ env.image }}:${{ matrix.tag }}" + # disabled because of https://github.com/aquasecurity/trivy/discussions/7668 + # - name: Run Trivy vulnerability scanner + # if: ${{ steps.login.conclusion != 'skipped' }} + # uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0 + # with: + # image-ref: '${{ env.image }}' + # format: 'table' + # exit-code: '1' + # ignore-unfixed: true + # vuln-type: 'os,library' + # severity: 'CRITICAL,HIGH' + - name: Push Latest Image + if: ${{ steps.login.conclusion != 'skipped' && matrix.tag == '11-8' }} + run: docker push "${{ env.image }}:latest" + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "cuda-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + compiler-image: + needs: [setup-instance, hpx-image, cuda-image] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + env: + image: ghcr.io/zama-ai/concrete-compiler + dockerfile: docker/Dockerfile.concrete-compiler-env + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + submodules: recursive + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24 + with: + files: | + backends/** + compilers/** + third_party/** + tools/** + - name: Login to Registry + id: login + if: steps.changed-files.outputs.any_changed == 'true' + run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io + - name: Build Image + if: steps.login.conclusion != 'skipped' + run: | + DOCKER_BUILDKIT=1 docker build --no-cache \ + --label "commit-sha=${{ github.sha }}" -t ${{ env.image }} -f ${{ env.dockerfile }} . + # disabled because of https://github.com/aquasecurity/trivy/discussions/7668 + # - name: Run Trivy vulnerability scanner + # uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0 + # with: + # image-ref: '${{ matrix.image }}' + # format: 'table' + # exit-code: '1' + # ignore-unfixed: true + # vuln-type: 'os,library' + # severity: 'CRITICAL,HIGH' + - name: Tag and Publish Image + if: steps.login.conclusion != 'skipped' + run: | + docker image tag ${{ env.image }} ${{ env.image }}:${{ github.sha }} + docker image push ${{ env.image }}:latest + docker image push ${{ env.image }}:${{ github.sha }} + - name: Tag and Publish Release Image + if: steps.login.conclusion != 'skipped' && startsWith(github.ref, 'refs/tags/v') + run: | + docker image tag ${{ env.image }} ${{ env.image }}:${{ github.ref_name }} + docker image push ${{ env.image }}:${{ github.ref_name }} + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "compiler-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + name: Teardown instance + needs: [ setup-instance, compiler-image ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_compiler_test_cpu.yml b/.github/workflows/concrete_compiler_test_cpu.yml new file mode 100644 index 0000000000..ad2e658eaa --- /dev/null +++ b/.github/workflows/concrete_compiler_test_cpu.yml @@ -0,0 +1,181 @@ +name: concrete-compiler test linux-cpu + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_test_cpu.yml + - compilers/** + - backends/concrete-cpu/** + - tools/** + push: + branches: + - 'main' + - 'release/*' + +env: + DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_compiler_test_cpu_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-test + + format-and-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Format with clang-format (Cpp) + run: | + sudo apt install moreutils + cd compilers/concrete-compiler/compiler + ./scripts/format_cpp.sh + - name: Format with cmake-format (Cmake) + run: | + pip3 install cmakelang + cd compilers/concrete-compiler/compiler + ./scripts/format_cmake.sh + - name: Format with black (Python) + run: | + cd compilers/concrete-compiler/compiler + pip install -r lib/Bindings/Python/requirements_dev.txt + make check-python-format + - name: Lint with pylint (Python) + run: | + cd compilers/concrete-compiler/compiler + # compiler requirements to lint + pip install numpy + make python-lint + - name: Check if sources include the license header + run: .github/workflows/scripts/check_for_license.sh + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "format-and-lint finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + build-and-run-test: + needs: [ setup-instance ] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Fetch repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + fetch-depth: 0 + - name: Create build dir + run: mkdir build + - name: Build compiler + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + id: build-compiler + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + -v ${{ github.workspace }}/wheels:/wheels + shell: bash + run: | + set -e + cd /concrete/compilers/concrete-compiler/compiler + rm -rf /build/* + make DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all + echo "Debug: ccache statistics (after the build):" + ccache -s + - name: Check compiler dialects docs is up to date + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + id: build-compiler-docs + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + -v ${{ github.workspace }}/wheels:/wheels + shell: bash + run: | + set -e + cd /build/tools/concretelang/docs/concretelang/ + sed -i -e 's/\[TOC\]//' *Dialect.md + for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done; + - name: Enable complete tests on push to main + if: github.ref == 'refs/heads/main' + run: echo "MINIMAL_TESTS=OFF" >> ${GITHUB_ENV} + - name: Enable minimal tests otherwise + if: github.ref != 'refs/heads/main' + run: echo "MINIMAL_TESTS=ON" >> ${GITHUB_ENV} + - name: Run compiler tests + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + shell: bash + run: | + set -e + cd /concrete/compilers/concrete-compiler/compiler + mkdir -p /tmp/concrete_compiler/gpu_tests/ + pip install pytest + sed "s/pytest/python -m pytest/g" -i Makefile + make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + needs: [ setup-instance, build-and-run-test ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_compiler_test_cpu_distributed.yml b/.github/workflows/concrete_compiler_test_cpu_distributed.yml new file mode 100644 index 0000000000..f09ca686b6 --- /dev/null +++ b/.github/workflows/concrete_compiler_test_cpu_distributed.yml @@ -0,0 +1,109 @@ +name: concrete-compiler test linux-cpu-distributed + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_test_cpu_distributed.yml + - compilers/concrete-compiler/** + push: + branches: + - 'main' + - 'release/*' + +env: + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_compiler_test_cpu_distributed_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: slurm-cluster + + build-and-run-test: + # The distributed-ci runner is registered on the instance configured in the slurm-cluster profile. + # It's why we need to setup-instance + needs: setup-instance + runs-on: distributed-ci + steps: + - name: Instance cleanup + run: | + sudo rm -rf /home/ubuntu/actions-runner/_work/concrete/concrete + mkdir -p /home/ubuntu/actions-runner/_work/concrete/concrete + docker system prune -af + + - name: Fetch repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + submodules: recursive + + - name: Set up home + # "Install rust" step require root user to have a HOME directory which is not set. + run: | + echo "HOME=/shared" >> "${GITHUB_ENV}" + + - name: Setup rust toolchain for concrete-cpu + uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + + - name: Build end-to-end distributed test + run: | + cd compilers/concrete-compiler/compiler + rm -rf /shared/build + make HPX_DIR=/shared/hpx install-hpx-from-source + make HPX_DIR=/shared/hpx BUILD_DIR=/shared/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON BINDINGS_PYTHON_ENABLED=OFF build-end-to-end-tests + + - name: Run end-to-end distributed test + run: | + cd compilers/concrete-compiler/compiler + rm -rf /shared/KeyCache + make BUILD_DIR=/shared/build KEY_CACHE_DIRECTORY=/shared/KeyCache run-end-to-end-distributed-tests + + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + needs: [ setup-instance, build-and-run-test ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_compiler_test_gpu.yml b/.github/workflows/concrete_compiler_test_gpu.yml new file mode 100644 index 0000000000..0e674e293a --- /dev/null +++ b/.github/workflows/concrete_compiler_test_gpu.yml @@ -0,0 +1,106 @@ +name: concrete-compiler test linux-gpu + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_test_gpu.yml + - compilers/** + - backends/concrete-cuda/** + - tools/** + push: + branches: + - 'main' + - 'release/*' + +env: + DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + CUDA_PATH: /usr/local/cuda-11.8 + +concurrency: + group: concrete_compiler_test_gpu_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: gpu-test + + build-and-test: + needs: [ setup-instance ] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + if: ${{ !cancelled() }} + steps: + - name: Fetch repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + submodules: recursive + - name: Create build dir + run: mkdir build + - name: Build and test compiler + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + id: build-compiler + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + --gpus all + shell: bash + run: | + set -e + cd /concrete/compilers/concrete-compiler/compiler + rm -rf /build/* + mkdir -p /tmp/concrete_compiler/gpu_tests/ + make BINDINGS_PYTHON_ENABLED=OFF Python3_EXECUTABLE=$PYTHON_EXEC CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} run-end-to-end-tests-gpu + echo "Debug: ccache statistics (after the build):" + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + needs: [ setup-instance, build-and-test ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_compiler_test_macos_cpu.yml b/.github/workflows/concrete_compiler_test_macos_cpu.yml new file mode 100644 index 0000000000..b5d9c5c30e --- /dev/null +++ b/.github/workflows/concrete_compiler_test_macos_cpu.yml @@ -0,0 +1,89 @@ +name: concrete-compiler test macos-cpu + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_compiler_test_macos_cpu.yml + - compilers/** + - backends/** + - tools/** + push: + branches: + - 'main' + - 'release/*' + +env: + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_compiler_test_macos_cpu_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + build-and-test: + strategy: + # if a failure happens, we want to know if it's specific + # to the architecture or the operating system + fail-fast: false + matrix: + runson: ["aws-mac1-metal", "aws-mac2-metal"] + python-version: ["3.10"] + runs-on: ${{ matrix.runson }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + fetch-depth: 0 + - name: Setup rust toolchain for concrete-cpu + uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu + - name: Set python variables + run: | + set -e + echo "PIP=${{ format('pip{0}', matrix.python-version) }}" >> "${GITHUB_ENV}" + echo "PYTHON=${{ format('python{0}', matrix.python-version) }}" >> "${GITHUB_ENV}" + echo "PYTHON_EXEC=$(which ${{ format('python{0}', matrix.python-version) }})" >> "${GITHUB_ENV}" + - name: Install dependencies + run: | + brew install ninja ccache + ${PIP} install pytest + - name: Build compiler + run: | + set -e + cd compilers/concrete-compiler/compiler + echo "Debug: ccache statistics (prior to the build):" + ccache -s + make Python3_EXECUTABLE=$PYTHON_EXEC all + echo "Debug: ccache statistics (after the build):" + ccache -s + - name: Enable complete tests on push to main + if: github.ref == 'refs/heads/main' + run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV + - name: Enable minimal tests otherwise + if: github.ref != 'refs/heads/main' + run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV + - name: Create keyset cache directory + run: | + export KEY_CACHE_DIRECTORY=$(mktemp -d)/KeySetCache + echo "KEY_CACHE_DIRECTORY=$KEY_CACHE_DIRECTORY" >> "${GITHUB_ENV}" + mkdir $KEY_CACHE_DIRECTORY + - name: Test + run: | + set -e + cd compilers/concrete-compiler/compiler + export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF + make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE=$PYTHON_EXEC run-tests + - name: Cleanup host + if: success() || failure() + run: | + rm -rf $KEY_CACHE_DIRECTORY + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_cpu_test.yml b/.github/workflows/concrete_cpu_test.yml index 40a1ed2549..6e7241c023 100644 --- a/.github/workflows/concrete_cpu_test.yml +++ b/.github/workflows/concrete_cpu_test.yml @@ -1,17 +1,28 @@ -name: Concrete CPU - Tests +name: concrete-cpu test on: - workflow_call: workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_cpu_test.yml + - backends/concrete-cpu/** + push: + branches: + - 'main' + - 'release/*' concurrency: - group: concrete_cpu_test-${{ github.ref }} + group: concrete_cpu_test_${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: CARGO_TERM_COLOR: always jobs: tests-linux: + strategy: + fail-fast: false + matrix: + runson: ["ubuntu-20.04"] runs-on: ubuntu-20.04 env: RUSTFLAGS: -D warnings @@ -57,39 +68,3 @@ jobs: run: | cd backends/concrete-cpu/implementation cargo test --no-fail-fast --all-targets --features=nightly - - tests-mac_x86: - runs-on: macos-11 - env: - RUSTFLAGS: -D warnings - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Download cargo cache - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 - - - name: Tests - run: | - cd backends/concrete-cpu/implementation - cargo test --no-fail-fast --all-targets - - tests-mac-m1: - runs-on: "aws-mac2-metal" - env: - RUSTFLAGS: -D warnings - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Download cargo cache - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 - - - name: Tests - run: | - cd backends/concrete-cpu/implementation - cargo test --no-fail-fast --all-targets diff --git a/.github/workflows/concrete_ml_test.yml b/.github/workflows/concrete_ml_test.yml new file mode 100644 index 0000000000..26a8ea37e6 --- /dev/null +++ b/.github/workflows/concrete_ml_test.yml @@ -0,0 +1,138 @@ +name: concrete-ml test + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/concrete_ml_test.yml + - frontends/concrete-python/** + push: + branches: + - 'main' + - 'release/*' + +env: + DOCKER_IMAGE: ghcr.io/zama-ai/concrete-compiler + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_ml_test_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-test + + build-and-run-tests: + strategy: + matrix: + python-version: ["3.8"] + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - name: Set release version + run: echo "__version__ = \"$(date +"%Y.%m.%d")\"" >| frontends/concrete-python/version.txt + - name: Expose release version from Python + run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py + + - name: Create build directory + run: mkdir build + + - name: Build wheel + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + id: build-compiler-bindings + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}:/concrete + -v ${{ github.workspace }}/build:/build + shell: bash + run: | + set -e + rm -rf /build/* + + export PYTHON=${{ format('python{0}', matrix.python-version) }} + echo "Using $PYTHON" + + cd /concrete/frontends/concrete-python + make PYTHON=$PYTHON venv + source .venv/bin/activate + + cd /concrete/compilers/concrete-compiler/compiler + make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings + + echo "Debug: ccache statistics (after the build):" + ccache -s + + cd /concrete/frontends/concrete-python + + export COMPILER_BUILD_DIRECTORY="/build" + make whl + + deactivate + + - name: Setup Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: ${{ matrix.python-version }} + - name: ML Tests + run: | + export HOME="/home/ubuntu" + export CONCRETE_PYTHON_WHEEL=$(pwd)/frontends/concrete-python/dist/*manylinux*.whl + apt update + apt install git git-lfs -y + pip install poetry==1.7.1 + ./ci/scripts/test_cml.sh --use-wheel $CONCRETE_PYTHON_WHEEL --verbose + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-and-run-tests finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, build-and-run-tests ] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_ml_tests.yml b/.github/workflows/concrete_ml_tests.yml deleted file mode 100644 index 47f7e6e695..0000000000 --- a/.github/workflows/concrete_ml_tests.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: Concrete ML Tests -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - - -env: - DOCKER_IMAGE: ghcr.io/zama-ai/concrete-compiler - -jobs: - linux-x86: - strategy: - matrix: - python-version: ["3.8"] - - runs-on: ${{ github.event.inputs.runner_name }} - steps: - - name: Log instance configuration - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "User Inputs: ${{ inputs.user_inputs }}" - - - name: Set up GitHub environment - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Set release version - run: echo "__version__ = \"$(date +"%Y.%m.%d")\"" >| frontends/concrete-python/version.txt - - - name: Expose release version from Python - run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py - - - name: Create build directory - run: mkdir build - - - name: Build wheel - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - id: build-compiler-bindings - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - ${{ env.DOCKER_GPU_OPTION }} - shell: bash - run: | - set -e - rm -rf /build/* - - export PYTHON=${{ format('python{0}', matrix.python-version) }} - echo "Using $PYTHON" - - cd /concrete/frontends/concrete-python - make PYTHON=$PYTHON venv - source .venv/bin/activate - - cd /concrete/compilers/concrete-compiler/compiler - make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings - - echo "Debug: ccache statistics (after the build):" - ccache -s - - cd /concrete/frontends/concrete-python - - export COMPILER_BUILD_DIRECTORY="/build" - make whl - - deactivate - - - name: Setup Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 - with: - python-version: ${{ matrix.python-version }} - - - name: ML Tests - run: | - export CONCRETE_PYTHON_WHEEL=$(pwd)/frontends/concrete-python/dist/*manylinux*.whl - apt update - apt install git git-lfs -y - pip install poetry==1.7.1 - ./ci/scripts/test_cml.sh --use-wheel $CONCRETE_PYTHON_WHEEL --verbose diff --git a/.github/workflows/optimizer.yml b/.github/workflows/concrete_optimizer.yml similarity index 74% rename from .github/workflows/optimizer.yml rename to .github/workflows/concrete_optimizer.yml index 48e86b8a6a..f74229d1cd 100644 --- a/.github/workflows/optimizer.yml +++ b/.github/workflows/concrete_optimizer.yml @@ -1,73 +1,72 @@ -name: Optimizer - Tests +name: concrete-optimizer test on: - workflow_call: workflow_dispatch: - secrets: - CONCRETE_CI_SSH_PRIVATE: - required: true - CONCRETE_ACTIONS_TOKEN: - required: true + pull_request: + paths: + - .github/workflows/concrete_optimizer.yml + - compilers/concrete-optimizer/** + - backends/** + - tools/** + push: + branches: + - 'main' + - 'release/*' + +env: + CARGO_TERM_COLOR: always + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} concurrency: - group: optimizer-${{ github.ref }} + group: concrete_optimizer-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} -env: - CARGO_TERM_COLOR: always jobs: tests: strategy: matrix: - os: [ubuntu-20.04, macos-11] - runs-on: ${{ matrix.os }} + runson: ["ubuntu-latest", "aws-mac1-metal", "aws-mac2-metal"] + runs-on: ${{ matrix.runson }} env: RUSTFLAGS: -D warnings steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: "Setup" + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Setup uses: ./.github/workflows/optimizer_setup - with: - ssh_private_key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} - - name: Formatting run: | + cargo --version cd compilers/concrete-optimizer cargo fmt --check - - name: Build run: | cd compilers/concrete-optimizer cargo build --release --all-targets - - name: Lint run: | cd compilers/concrete-optimizer cargo clippy --release --all-targets - - name: Tests - if: matrix.os == 'ubuntu-20.04' run: | cd compilers/concrete-optimizer cargo test --release --no-fail-fast --all-targets make -C concrete-optimizer-cpp test-ci benchmarks: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: "Setup" + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Setup uses: ./.github/workflows/optimizer_setup - with: - ssh_private_key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }} - - name: Run benchmark run: | cd compilers/concrete-optimizer cargo bench -p v0-parameters -- --output-format bencher | tee bench_result.txt - - name: Download PR base benchmark data if: ${{ github.event_name == 'pull_request' }} # for artifacts restrictions see https://github.com/actions/download-artifact/issues/3 @@ -81,7 +80,6 @@ jobs: name: ${{ runner.os }}-benchmark if_no_artifact_found: warn path: ./benchmark - - name: Save benchmark result to file uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 # v1.20.3 with: @@ -94,7 +92,6 @@ jobs: comment-always: true # Enable Job Summary for PRs summary-always: true - - name: Upload benchmark data uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: diff --git a/.github/workflows/concrete_python_benchmark.yml b/.github/workflows/concrete_python_benchmark.yml index b62b99836b..5142d117c2 100644 --- a/.github/workflows/concrete_python_benchmark.yml +++ b/.github/workflows/concrete_python_benchmark.yml @@ -1,17 +1,25 @@ -name: Concrete Python Benchmark +name: concrete-python benchmark linux-cpu on: workflow_dispatch: schedule: - cron: "0 1 * * SAT" - + pull_request: + paths: + - .github/workflows/concrete_pyhon_benchmark.yml + push: + branches: + - 'main' + - 'release/*' env: DOCKER_IMAGE: ghcr.io/zama-ai/concrete-compiler - GLIB_VER: 2_28 + +concurrency: + group: concrete_python_benchmark_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: setup-instance: - name: Setup Instance runs-on: ubuntu-latest outputs: runner-name: ${{ steps.start-instance.outputs.label }} @@ -25,10 +33,9 @@ jobs: slab-url: ${{ secrets.SLAB_BASE_URL }} job-secret: ${{ secrets.JOB_SECRET }} backend: aws - profile: m7i-cpu-bench + profile: cpu-bench concrete-python-benchmarks: - name: Run Concrete Python Benchmarks needs: setup-instance runs-on: ${{ needs.setup-instance.outputs.runner-name }} steps: @@ -49,22 +56,14 @@ jobs: options: >- -v ${{ github.workspace }}:/concrete -v ${{ github.workspace }}/build:/build - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - ${{ env.DOCKER_GPU_OPTION }} shell: bash run: | set -e - - rustup toolchain install nightly-2024-09-30 - pip install mypy rm -rf /build/* export PYTHON=${{ format('python{0}', matrix.python-version) }} echo "Using $PYTHON" - dnf -y install graphviz graphviz-devel - cd /concrete/frontends/concrete-python make PYTHON=$PYTHON venv source .venv/bin/activate @@ -72,14 +71,12 @@ jobs: cd /concrete/compilers/concrete-compiler/compiler make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings - echo "Debug: ccache statistics (after the build):" - ccache -s - cd /concrete/frontends/concrete-python export COMPILER_BUILD_DIRECTORY="/build" - export PROGRESS_MACHINE_NAME="m7i.48xlarge" - + # TODO output setup-instance (https://github.com/zama-ai/slab-github-runner/issues/38) + export PROGRESS_MACHINE_NAME="hpc7a.96xlarge" + make benchmark make process-benchmark-results-for-grafana @@ -97,9 +94,9 @@ jobs: run: | echo "Computing HMac on results file" SIGNATURE="$(slab/scripts/hmac_calculator.sh frontends/concrete-python/progress.processed.json '${{ secrets.JOB_SECRET }}')" - + cd frontends/concrete-python - + echo "Sending results to Slab..." curl -v -k \ -H "Content-Type: application/json" \ @@ -108,9 +105,15 @@ jobs: -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ -d @progress.processed.json \ ${{ secrets.SLAB_URL }} + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "concrete-python-benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" teardown-instance: - name: Teardown Instance if: ${{ always() && needs.setup-instance.result != 'skipped' }} needs: [ setup-instance, concrete-python-benchmarks ] runs-on: ubuntu-latest @@ -124,3 +127,10 @@ jobs: slab-url: ${{ secrets.SLAB_BASE_URL }} job-secret: ${{ secrets.JOB_SECRET }} label: ${{ needs.setup-instance.outputs.runner-name }} + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_python_checks.yml b/.github/workflows/concrete_python_checks.yml deleted file mode 100644 index fa2f908b4d..0000000000 --- a/.github/workflows/concrete_python_checks.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Concrete Python Checks - -on: - workflow_call: - -jobs: - Checks: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Install Platform Dependencies - run: | - sudo apt install -y graphviz libgraphviz-dev - - name: Pre-Commit Checks - run: | - ./frontends/concrete-python/scripts/checks/checks.sh diff --git a/.github/workflows/concrete_python_finalize_release.yml b/.github/workflows/concrete_python_finalize_release.yml new file mode 100644 index 0000000000..482c6f60c6 --- /dev/null +++ b/.github/workflows/concrete_python_finalize_release.yml @@ -0,0 +1,79 @@ +# This workflows should be runned after that releases has been validated and ready to push to pypi.org and docker hub. +name: concrete-python finalize-release + +on: + workflow_dispatch: + inputs: + version: + description: 'version of concrete-python to push to pypi and docker hub' + required: true + type: string + +jobs: + publish-to-pypi: + runs-on: ubuntu-latest + steps: + - name: Pull wheels from S3 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }} + run: | + mkdir wheels + aws s3 cp s3://${S3_BUCKET_NAME}/cpu/concrete-python/ ./wheels/ --recursive --exclude "*" --include "concrete_python-${{ inputs.version }}-*" + echo "============== Downloaded wheels ===============" + ls -la ./wheels/ + - name: Push wheels to public PyPI (public) + run: | + pip install twine==4.0.2 + twine upload wheels/concrete_python-${{ inputs.version }}*.whl \ + -u "${{ secrets.PUBLIC_PYPI_USER }}" \ + -p "${{ secrets.PUBLIC_PYPI_PASSWORD }}" \ + -r pypi + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "test-linux-x86 (${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + publish-to-dockerhub: + runs-on: ubuntu-latest + env: + DOCKER_IMAGE_NAME: zamafhe/concrete-python + DOCKER_FILE: docker/Dockerfile.concrete-python + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Get version from tag + run: | + # remove leading 'v' and '-' from tag + export VERSION=`echo ${{ inputs.tag }} | sed "s/^v*//g" | sed "s/-//g"` + echo "VERSION=$VERSION" >> "${GITHUB_ENV}" + echo "NAME_TAG=${{ env.DOCKER_IMAGE_NAME }}:v$VERSION" >> "${GITHUB_ENV}" + - name: Build image + run: | + mkdir empty_context + docker image build -t ${{ env.NAME_TAG }} --build-arg version=${{ env.VERSION }} -f ${{ env.DOCKER_FILE }} empty_context + + # disabled because of https://github.com/aquasecurity/trivy/discussions/7668 + # - name: Run Trivy vulnerability scanner + # uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0 + # with: + # image-ref: '${{ env.NAME_TAG }}' + # format: 'table' + # exit-code: '1' + # ignore-unfixed: true + # vuln-type: 'os,library' + # severity: 'CRITICAL,HIGH' + + - name: Login to Docker Hub + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push image + run: docker image push ${{ env.NAME_TAG }} diff --git a/.github/workflows/concrete_python_push_docker_image.yml b/.github/workflows/concrete_python_push_docker_image.yml deleted file mode 100644 index f88248a464..0000000000 --- a/.github/workflows/concrete_python_push_docker_image.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Concrete Python Push Docker Image -on: - workflow_dispatch: - inputs: - tag: - description: 'tag to use for the docker image' - type: string - workflow_call: - inputs: - tag: - description: 'tag to use for the docker image' - type: string - -env: - DOCKER_IMAGE_NAME: zamafhe/concrete-python - DOCKER_FILE: docker/Dockerfile.concrete-python - -jobs: - build_and_push: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Get version from tag - run: | - # remove leading 'v' and '-' from tag - export VERSION=`echo ${{ inputs.tag }} | sed "s/^v*//g" | sed "s/-//g"` - echo "VERSION=$VERSION" >> "${GITHUB_ENV}" - echo "NAME_TAG=${{ env.DOCKER_IMAGE_NAME }}:v$VERSION" >> "${GITHUB_ENV}" - - - name: Build image - run: | - mkdir empty_context - docker image build -t ${{ env.NAME_TAG }} --build-arg version=${{ env.VERSION }} -f ${{ env.DOCKER_FILE }} empty_context - - # disabled because of https://github.com/aquasecurity/trivy/discussions/7668 - # - name: Run Trivy vulnerability scanner - # uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0 - # with: - # image-ref: '${{ env.NAME_TAG }}' - # format: 'table' - # exit-code: '1' - # ignore-unfixed: true - # vuln-type: 'os,library' - # severity: 'CRITICAL,HIGH' - - - name: Login to Docker Hub - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Push image - run: docker image push ${{ env.NAME_TAG }} diff --git a/.github/workflows/concrete_python_release.yml b/.github/workflows/concrete_python_release_cpu.yml similarity index 74% rename from .github/workflows/concrete_python_release.yml rename to .github/workflows/concrete_python_release_cpu.yml index 7282b50ee4..9922fc550e 100644 --- a/.github/workflows/concrete_python_release.yml +++ b/.github/workflows/concrete_python_release_cpu.yml @@ -1,37 +1,46 @@ -name: Concrete Python Release +name: concrete-python release-cpu + on: workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - user_inputs: - description: 'either "nightly" or "public" or "private" to specify the release type' - required: true - default: 'nightly' - type: string - + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+*' + schedule: + # Nightly Release @ 3AM after each work day + - cron: "0 3 * * 2-6" env: DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler - GLIB_VER: 2_28 - RELEASE_TYPE: ${{ inputs.user_inputs }} + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + RELEASE_TYPE: ${{ (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) && 'public' || 'nightly' }} + +concurrency: + group: concrete_python_release_cpu_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: release + release-checks: - runs-on: ${{ github.event.inputs.runner_name }} + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -40,32 +49,29 @@ jobs: fetch-depth: 0 - name: Check python api doc is up to date run: ci/scripts/make_apidocs.sh + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "release-checks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" build-linux-x86: strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - - runs-on: ${{ github.event.inputs.runner_name }} + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} steps: - - name: Log instance configuration - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "User Inputs: ${{ inputs.user_inputs }}" - - name: Set up GitHub environment run: | echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: recursive fetch-depth: 0 - - name: Set release version (nightly) if: ${{ env.RELEASE_TYPE == 'nightly' }} run: | @@ -75,14 +81,11 @@ jobs: echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt git tag nightly-$NIGHTLY_VERSION || true git push origin nightly-$NIGHTLY_VERSION || true - - name: Set release version (public) if: ${{ env.RELEASE_TYPE == 'public' }} run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt - - name: Expose release version from Python run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py - - name: Build wheel uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 id: build-compiler-bindings @@ -94,15 +97,10 @@ jobs: options: >- -v ${{ github.workspace }}:/concrete -v ${{ github.workspace }}/build:/build - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - ${{ env.DOCKER_GPU_OPTION }} shell: bash run: | set -e - rustup toolchain install nightly-2024-09-30 - pip install mypy rm -rf /build/* export PYTHON=${{ format('python{0}', matrix.python-version) }} @@ -126,20 +124,25 @@ jobs: make whl deactivate - - name: Upload wheel uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} path: frontends/concrete-python/dist/*manylinux*.whl retention-days: 3 + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-linux-x86 finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" build-macos: strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] runs-on: ["aws-mac1-metal", "aws-mac2-metal"] - runs-on: ${{ matrix.runs-on }} steps: - name: Checkout @@ -147,14 +150,11 @@ jobs: with: submodules: recursive fetch-depth: 0 - - name: Install OS Dependencies run: | brew install ninja ccache - - name: Setup rust toolchain for concrete-cpu uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - name: Set release version (nightly) if: ${{ env.RELEASE_TYPE == 'nightly' }} run: | @@ -162,14 +162,11 @@ jobs: NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d") LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o` echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt - - name: Set release version (public) if: ${{ env.RELEASE_TYPE == 'public' }} run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt - - name: Expose release version from Python run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py - - name: Build wheel run: | export CONCRETE_PYTHON=$(pwd)/frontends/concrete-python @@ -203,13 +200,19 @@ jobs: delocate-wheel -v dist/*macos*.whl deactivate - - name: Upload wheel uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} path: frontends/concrete-python/dist/*macos*.whl retention-days: 3 + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-macos finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" hash: # Generate hashes for the wheels, used later for provenance. @@ -279,28 +282,24 @@ jobs: aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/cpu/concrete-python # update indexes and invalidate cloudfront cache python .github/workflows/scripts/s3_update_html_indexes.py - - - name: Start pushing Docker images - if: ${{ env.RELEASE_TYPE == 'public' }} - run: | - export TAG=$(git describe --tags --abbrev=0) - curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/zama-ai/concrete/actions/workflows/concrete_python_push_docker_image.yml/dispatches \ - -d "{\"ref\": \"$TAG\", \"inputs\": {\"tag\":\"v$TAG\"}}" + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "push finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" test-linux-x86: - needs: [build-linux-x86] + needs: [setup-instance, build-linux-x86] continue-on-error: true strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - runs-on: ${{ github.event.inputs.runner_name }} + runs-on: ${{ needs.setup-instance.outputs.runner-name }} steps: - - uses: actions-rust-lang/setup-rust-toolchain@11df97af8e8102fd60b60a77dfbf58d40cd843b8 # v1.10.1 + - name: Install rust + uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7 - name: Setup Python uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: @@ -343,6 +342,36 @@ jobs: # Running tests make tfhers-utils pytest tests -svv -n auto + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "test-linux-x86 (${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + needs: [ setup-instance, test-linux-x86 ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" test-macos: needs: [build-macos] @@ -396,8 +425,14 @@ jobs: make tfhers-utils mkdir ./KeySetCache pytest tests -svv -n auto --key-cache "./KeySetCache" -m "not dataflow and not graphviz" - - name: Cleanup host if: success() || failure() run: | rm -rf $TEST_TMP_DIR + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "test-macos (${{matrix.runs-on}}/${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_python_release_gpu.yml b/.github/workflows/concrete_python_release_gpu.yml index 26411b5706..231ab27061 100644 --- a/.github/workflows/concrete_python_release_gpu.yml +++ b/.github/workflows/concrete_python_release_gpu.yml @@ -1,61 +1,58 @@ -name: Concrete Python Release (GPU) - +name: concrete-python release-gpu on: workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - user_inputs: - description: 'either "nightly" or "public" or "private" to specify the release type' - required: true - default: 'nightly' - type: string + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+*' + schedule: + # Nightly Release @ 3AM after each work day + - cron: "0 3 * * 2-6" env: DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler CUDA_PATH: /usr/local/cuda-11.8 - GCC_VERSION: 11 - RELEASE_TYPE: ${{ inputs.user_inputs }} + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + RELEASE_TYPE: ${{ (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) && 'public' || 'nightly' }} + +concurrency: + group: concrete_python_release_gpu_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: release + build-linux-x86: + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - - runs-on: ${{ github.event.inputs.runner_name }} steps: - - name: Log instance configuration - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "User Inputs: ${{ inputs.user_inputs }}" - - name: Set up GitHub environment run: | echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: recursive fetch-depth: 0 - - name: Set release version (nightly) if: ${{ env.RELEASE_TYPE == 'nightly' }} run: | @@ -65,14 +62,11 @@ jobs: echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt git tag nightly-$NIGHTLY_VERSION || true git push origin nightly-$NIGHTLY_VERSION || true - - name: Set release version (public) if: ${{ env.RELEASE_TYPE == 'public' }} run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt - - name: Expose release version from Python run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py - - name: Build wheel uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 id: build-compiler-bindings @@ -84,29 +78,23 @@ jobs: options: >- -v ${{ github.workspace }}:/concrete -v ${{ github.workspace }}/build:/build - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket shell: bash run: | set -e - rustup toolchain install nightly-2024-09-30 - pip install mypy rm -rf /build/* - + export PYTHON=${{ format('python{0}', matrix.python-version) }} echo "Using $PYTHON" - dnf -y install graphviz graphviz-devel - cd /concrete/frontends/concrete-python make PYTHON=$PYTHON venv source .venv/bin/activate - + cd /concrete/compilers/concrete-compiler/compiler make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=OFF Python3_EXECUTABLE=$(which python) \ CUDA_SUPPORT=ON TIMING_ENABLED=ON CUDA_PATH=${{ env.CUDA_PATH }} python-bindings - + echo "Debug: ccache statistics (after the build):" ccache -s @@ -114,37 +102,62 @@ jobs: export COMPILER_BUILD_DIRECTORY="/build" make whl - - deactivate + deactivate - name: Upload wheel uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} path: frontends/concrete-python/dist/*manylinux*.whl retention-days: 3 + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-linux-x86 (${{matrix.python-version}}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - push: + teardown-instance: + needs: [ setup-instance, build-linux-x86 ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + push-wheels: needs: [build-linux-x86] runs-on: ubuntu-latest outputs: wheel_version: ${{ steps.version.outputs.wheel_version }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: path: wheels merge-multiple: true - - - name: Install aws-cli if not present + - name: Install aws-cli run: | aws --version || (curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ unzip awscliv2.zip && \ sudo ./aws/install) - - name: Upload wheels to S3 - if: ${{ env.RELEASE_TYPE == 'public' || env.RELEASE_TYPE == 'nightly' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }} @@ -157,18 +170,92 @@ jobs: aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/gpu/concrete-python # update indexes and invalidate cloudfront cache python .github/workflows/scripts/s3_update_html_indexes.py - - name: Output Wheel Version id: version run: | export VERSION=`ls ./wheels/*manylinux* | head -n1 | cut -d "-" -f2` echo "VERSION=$VERSION" echo "wheel_version=$VERSION" >> "$GITHUB_OUTPUT" + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "push-wheels finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + setup-test-instance: + runs-on: ubuntu-latest + needs: [push-wheels] + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: gpu-test + + test-linux-x86: + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + fail-fast: false + needs: [setup-test-instance, push-wheels] + runs-on: ${{ needs.setup-test-instance.outputs.runner-name }} + steps: + - name: Setup Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: ${{ matrix.python-version }} - test-gpu-wheel: - needs: [push] - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-test-gpu-wheel - user_inputs: "${{ needs.push.outputs.wheel_version }}" + - name: Install concrete-python + run: pip install --pre --extra-index-url https://pypi.zama.ai/gpu/ "concrete-python==${{ needs.push-wheels.outputs.wheel_version }}" + + - name: Checkout the repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: repo + + - name: Test wheel + run: | + CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python + + # Install extra requirements for tests + sudo apt update -y + sudo apt install -y graphviz libgraphviz-dev + pip install -r $CONCRETE_PYTHON/requirements.extra-full.txt + pip install -r $CONCRETE_PYTHON/requirements.dev.txt + + # Running tests + cd $CONCRETE_PYTHON + make pytest-gpu + + + teardown-test-instance: + needs: [ setup-test-instance, test-linux-x86 ] + if: ${{ always() && needs.setup-test-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-test-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_python_test_macos.yml b/.github/workflows/concrete_python_test_macos.yml index 79e9409218..9c12d56386 100644 --- a/.github/workflows/concrete_python_test_macos.yml +++ b/.github/workflows/concrete_python_test_macos.yml @@ -1,40 +1,37 @@ -name: Concrete Python Tests (macOS) +name: concrete-python tests macos on: - workflow_call: workflow_dispatch: - secrets: - CONCRETE_CI_SSH_PRIVATE: - required: true - CONCRETE_ACTIONS_TOKEN: - required: true + pull_request: + paths: + - .github/workflows/concrete_python_tests_macos.yml + push: + branches: + - 'main' + - 'release/*' concurrency: - group: concrete_python_tests_macos-${{ github.ref }} + group: concrete_python_tests_macos_${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - BuildConcreteCompilerAndTestConcretePythonInMacOS: + concrete-python-test-pytest: strategy: fail-fast: false matrix: machine: ["aws-mac1-metal", "aws-mac2-metal"] - runs-on: ${{ matrix.machine }} steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Install OS Dependencies + fetch-depth: 0 + - name: Install build dependencies run: | brew install ninja ccache - - name: Setup rust toolchain for concrete-cpu uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Cache Compilation (push) + - name: Cache compilation (push) if: github.event_name == 'push' uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -42,8 +39,7 @@ jobs: key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.sha }} restore-keys: | ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - - name: Cache Compilation (pull_request) + - name: Cache compilation (pull_request) if: github.event_name == 'pull_request' uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -51,88 +47,72 @@ jobs: key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.event.pull_request.base.sha }} restore-keys: | ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - - name: Get tmpdir path - if: github.event_name == 'push' - id: tmpdir-path - run: echo "::set-output name=TMPDIR_PATH::$TMPDIR" - - - name: Build + - name: Prepare build environment run: | set -e - cd frontends/concrete-python - + cd $GITHUB_WORKSPACE/frontends/concrete-python + # Setup pkg-config to find OpenBLAS (scipy need it) export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" - + rm -rf .venv python3.10 -m venv .venv - - . .venv/bin/activate - + + . $GITHUB_WORKSPACE/frontends/concrete-python/.venv/bin/activate + pip install -r requirements.dev.txt pip install -r requirements.txt - + - name: Build concrete-compiler python-bindings + run: | + $GITHUB_WORKSPACE/frontends/concrete-python .venv/bin/activate cd $GITHUB_WORKSPACE/compilers/concrete-compiler/compiler - - echo "Debug: ccache statistics (prior to the build):" - ccache -s - + ccache -z make Python3_EXECUTABLE=$(which python) python-bindings - - echo "Debug: ccache statistics (after the build):" ccache -s - - export COMPILER_BUILD_DIRECTORY=$(pwd)/build + - name: Create wheels + run: | + $GITHUB_WORKSPACE/frontends/concrete-python .venv/bin/activate cd $GITHUB_WORKSPACE/frontends/concrete-python - - rm -rf dist - mkdir -p dist - + + export COMPILER_BUILD_DIRECTORY=$GITHUB_WORKSPACE/compilers/concrete-compiler/compiler + rm -rf dist && mkdir -p dist pip wheel -v --no-deps -w dist . delocate-wheel -v dist/*macos*.whl - - deactivate + deactivate - name: Prepare test environment run: | set -e export TEST_TMP_DIR=$(mktemp -d) echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}" cd $TEST_TMP_DIR - + python3.10 -m venv .testenv . .testenv/bin/activate - + pip install $GITHUB_WORKSPACE/frontends/concrete-python/dist/*macos*.whl pip install -r $GITHUB_WORKSPACE/frontends/concrete-python/requirements.dev.txt # MacOS x86 have conflict between our OpenMP library, and one from torch # we fix it by using a single one (from torch) # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8 - + find .testenv/lib/python3.10/site-packages -not \( -path .testenv/lib/python3.10/site-packages/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1 ln -f -s $(pwd)/.testenv/lib/python3.10/site-packages/concrete/.dylibs/libomp.dylib cp -R $GITHUB_WORKSPACE/frontends/concrete-python/examples ./examples cp -R $GITHUB_WORKSPACE/frontends/concrete-python/tests ./tests - - cp $GITHUB_WORKSPACE/frontends/concrete-python/Makefile . - - name: Test + cp $GITHUB_WORKSPACE/frontends/concrete-python/Makefile . + - name: Run pytest-macos run: | - set -e - export TEST_TMP_DIR="testing_concrete_python" cd $TEST_TMP_DIR && . .testenv/bin/activate KEY_CACHE_DIRECTORY=./KeySetCache PYTEST_MARKERS="not dataflow and not graphviz" make pytest-macos - - - name: Test notebooks + - name: Run test-notebooks run: | set -e - export TEST_TMP_DIR="testing_concrete_python" cd $TEST_TMP_DIR && . .testenv/bin/activate make test-notebooks - - name: Cleanup host if: success() || failure() run: | diff --git a/.github/workflows/concrete_python_tests_linux.yml b/.github/workflows/concrete_python_tests_linux.yml index 6960926957..3cbbc93407 100644 --- a/.github/workflows/concrete_python_tests_linux.yml +++ b/.github/workflows/concrete_python_tests_linux.yml @@ -1,69 +1,72 @@ -name: Concrete Python Tests (Linux) +name: concrete-python tests linux-cpu on: workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - -# concurrency: -# group: concrete_python_tests_linux-${{ github.ref }} -# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + pull_request: + paths: + - .github/workflows/concrete_python_tests_linux.yml + - frontends/concrete-python/** + push: + branches: + - 'main' + - 'release/*' env: DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler - CUDA_PATH: /usr/local/cuda-11.8 - GCC_VERSION: 11 - GLIB_VER: 2_28 + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +concurrency: + group: concrete_python_tests_linux_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - BuildConcreteCompilerAndTestConcretePythonInLinux: - name: Build Concrete Compiler and Test Concrete Python in Linux - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-test + + pre-commit-check: + runs-on: ubuntu-22.04 steps: - - name: Log instance configuration + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Install platform dependencies run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Set up GitHub environment + sudo apt install -y graphviz libgraphviz-dev + - name: Pre-commit Checks run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - - - name: Checkout + cd frontends/concrete-python + make venv + source .venv/bin/activate + make pcc + + build-python-bindings: + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Checkout concrete uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Create build directory + fetch-depth: 0 + - name: Create concrete build directory run: mkdir build - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Build bindings + - name: Build concrete-compiler python bindings uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} - id: build-compiler-bindings with: registry: ghcr.io image: ${{ env.DOCKER_IMAGE_TEST }} @@ -72,73 +75,56 @@ jobs: options: >- -v ${{ github.workspace }}:/concrete -v ${{ github.workspace }}/build:/build - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket shell: bash run: | - rustup toolchain install nightly-2024-09-30 - pip install mypy set -e - rm -rf /build/* + rustup toolchain install nightly-2024-09-30 dnf -y install graphviz graphviz-devel cd /concrete/frontends/concrete-python make venv source .venv/bin/activate - + cd /concrete/compilers/concrete-compiler/compiler make BUILD_DIR=/build DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$(which python3) python-bindings - + echo "Debug: ccache statistics (after the build):" ccache -s - - - name: Prepare test environment - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} + - name: Create artifact archive + run: | + cd build + tar czvf artifacts.tgz lib/libConcretelangRuntime.so tools/concretelang/python_packages + - name: Upload concrete-compiler python-bindings + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - shell: bash - run: | - set -e - - dnf -y install graphviz graphviz-devel - - cd /concrete/frontends/concrete-python - make venv - - - name: Test - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} + name: concrete-compiler-python-bindings + include-hidden-files: true + retention-days: 3 + path: build/artifacts.tgz + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-python-bindings finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + test-pytest: + needs: [setup-instance, build-python-bindings] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Download concrete-compiler python-bindings + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - shell: bash - run: | - set -e - - cd /concrete/frontends/concrete-python - source .venv/bin/activate - - export COMPILER_BUILD_DIRECTORY=/build - - mkdir ./KeySetCache - KEY_CACHE_DIRECTORY=./KeySetCache make pytest - - - name: Test notebooks + name: concrete-compiler-python-bindings + path: compiler-artifacts + - name: Extract artifacts archive + run: | + cd compiler-artifacts + tar xzvf artifacts.tgz + - name: Run pytest uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} with: registry: ghcr.io image: ${{ env.DOCKER_IMAGE_TEST }} @@ -146,53 +132,39 @@ jobs: password: ${{ secrets.GHCR_PASSWORD }} options: >- -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build + -v ${{ github.workspace }}/compiler-artifacts:/compiler-artifacts shell: bash run: | set -e - + export COMPILER_BUILD_DIRECTORY=/compiler-artifacts cd /concrete/frontends/concrete-python source .venv/bin/activate - - export COMPILER_BUILD_DIRECTORY=/build - - make test-notebooks - - - - name: Build bindings gpu - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ contains(inputs.instance_type, 'p3') }} - id: build-compiler-bindings-gpu + export KEY_CACHE_DIRECTORY=./key-set-cache + mkdir $KEY_CACHE_DIRECTORY + make pytest + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "test-pytest finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + test-notebooks: + needs: [setup-instance, build-python-bindings] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Download concrete-compiler python-bindings + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - --gpus all - shell: bash - run: | - set -e - rm -rf /build/* - - cd /concrete/frontends/concrete-python - make venv - source .venv/bin/activate - - cd /concrete/compilers/concrete-compiler/compiler - make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python3) CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} python-bindings - - echo "Debug: ccache statistics (after the build):" - ccache -s - - - name: Test gpu + name: concrete-compiler-python-bindings + path: compiler-artifacts + - name: Extract artifacts archive + run: | + cd compiler-artifacts + tar xzvf artifacts.tgz + - name: Run pytest uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ contains(inputs.instance_type, 'p3') }} with: registry: ghcr.io image: ${{ env.DOCKER_IMAGE_TEST }} @@ -200,19 +172,41 @@ jobs: password: ${{ secrets.GHCR_PASSWORD }} options: >- -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - --gpus all + -v ${{ github.workspace }}/compiler-artifacts:/compiler-artifacts shell: bash run: | set -e - + export COMPILER_BUILD_DIRECTORY=/compiler-artifacts cd /concrete/frontends/concrete-python - make venv source .venv/bin/activate - - export COMPILER_BUILD_DIRECTORY=/build - KEY_CACHE_DIRECTORY=/tmp/KeySetCache mkdir ./KeySetCache - make pytest-gpu - - chmod -R ugo+rwx /tmp/KeySetCache + make test-notebooks + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "test-notebooks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, test-pytest, test-notebooks ] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/docker-lint.yml b/.github/workflows/docker-lint.yml deleted file mode 100644 index 197f72a972..0000000000 --- a/.github/workflows/docker-lint.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Lint Dockerfiles - -on: - pull_request: - push: - branches: - - main - -jobs: - lint: - runs-on: ubuntu-latest - container: - image: hadolint/hadolint@sha256:27173fe25e062448490a32de410c08491c626a0bef360aa2ce5d5bdd9384b50d #2.12.0-debian - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - - name: Lint All Dockerfiles (except third_party) - run: hadolint -V `find -name "*Dockerfile*" -not -path "./third_party/*" |xargs ` diff --git a/.github/workflows/docker_compliance.yml b/.github/workflows/docker_compliance.yml new file mode 100644 index 0000000000..d5da1eda40 --- /dev/null +++ b/.github/workflows/docker_compliance.yml @@ -0,0 +1,29 @@ +name: check docker files compliance + +on: + pull_request: + paths: + - .github/workflows/docker_compliance.yml + - '**Dockerfile**' + push: + branches: + - main + - 'release/*' + +jobs: + lint: + runs-on: ubuntu-latest + container: + image: hadolint/hadolint@sha256:27173fe25e062448490a32de410c08491c626a0bef360aa2ce5d5bdd9384b50d #2.12.0-debian + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + + - name: Lint All Dockerfiles (except third_party) + run: hadolint -V `find -name "*Dockerfile*" -not -path "./third_party/*" |xargs ` + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "lint finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/linelint.yml b/.github/workflows/linelint.yml deleted file mode 100644 index 03b814c839..0000000000 --- a/.github/workflows/linelint.yml +++ /dev/null @@ -1,18 +0,0 @@ -# This job is the main jobs will dispatch build and test for every modules of our mono repo. -name: Linelint - -on: - pull_request: - push: - branches: - - 'main' - -jobs: - linelint: - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Linelint - uses: fernandrone/linelint@8136e0fa9997122d80f5f793e0bb9a45e678fbb1 # 0.0.4 - id: linelint diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index e39d983fde..0000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,303 +0,0 @@ -# This job is the main jobs will dispatch build and test for every modules of our mono repo. -name: Main - -on: - pull_request: - push: - branches: - - 'main' - - 'release/*' - - 'force-docker-images' - - 'private_release/*' - tags: - - 'v[0-9]+.[0-9]+.[0-9]+*' - schedule: - # Nightly Release @ 3AM after each work day - - cron: "0 3 * * 2-6" - -jobs: - # This jobs outputs for each modules of our mono-repo if it changed, - # in order to launch jobs only for the changed modules - file-change: - if: ${{ github.event_name != 'schedule' }} - runs-on: ubuntu-latest - outputs: - compiler: ${{ steps.compiler.outputs.any_changed }} - optimizer: ${{ steps.optimizer.outputs.any_changed }} - concrete-cpu: ${{ steps.concrete-cpu.outputs.any_changed }} - concrete-cpu-api: ${{ steps.concrete-cpu-api.outputs.any_changed }} - concrete-cuda-api: ${{ steps.concrete-cuda-api.outputs.any_changed }} - concrete-python: ${{ steps.concrete-python.outputs.any_changed }} - concrete-compiler-cpu-workflow: ${{ steps.concrete-compiler-cpu-workflow.outputs.any_changed }} - concrete-compiler-gpu-workflow: ${{ steps.concrete-compiler-gpu-workflow.outputs.any_changed }} - concrete-compiler-format-and-linting-workflow: ${{ steps.concrete-compiler-format-and-linting-workflow.outputs.any_changed }} - concrete-compiler-macos-workflow: ${{ steps.concrete-compiler-macos-workflow.outputs.any_changed }} - concrete-compiler-docker-images-workflow: ${{ steps.concrete-compiler-docker-images-workflow.outputs.any_changed }} - concrete-cpu-workflow: ${{ steps.concrete-cpu-workflow.outputs.any_changed }} - concrete-python-workflow: ${{ steps.concrete-python-workflow.outputs.any_changed }} - concrete-optimizer-workflow: ${{ steps.concrete-optimizer-workflow.outputs.any_changed }} - push-main: ${{ steps.github.outputs.push-main }} - steps: - - name: Checkout the repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Get changed files in the concrete-compiler directory - id: compiler - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./compilers/concrete-compiler/** - - - name: Get changed files for concrete-optimizer - id: optimizer - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: | - ./tools/parameter-curves/concrete-security-curves-rust/** - ./compilers/concrete-optimizer/** - ./.github/workflows/optimizer.yml - - - name: Get changed files in the concrete-cpu directory - id: concrete-cpu - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./backends/concrete-cpu/implementation/** - - - name: Get changed files in the concrete-python directory - id: concrete-python - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./frontends/concrete-python/** - - - name: Check if compiler_build_and_test_cpu workflow has changed - id: concrete-compiler-cpu-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/compiler_build_and_test_cpu.yml - - - name: Check if compiler_build_and_test_gpu workflow has changed - id: concrete-compiler-gpu-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/compiler_build_and_test_gpu.yml - - - name: Check if compiler_format_and_linting.yml workflow has changed - id: concrete-compiler-format-and-linting-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/compiler_format_and_linting.yml - - - name: Check if compiler_macos_build_and_test workflow has changed - id: concrete-compiler-macos-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/compiler_macos_build_and_test.yml - - - name: Check if compiler_publish_docker_images workflow has changed - id: concrete-compiler-docker-images-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: | - ./.github/workflows/compiler_publish_docker_images.yml - ./docker/** - - - name: Check if concrete_cpu_test workflow has changed - id: concrete-cpu-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/concrete_cpu_test.yml - - - name: Check if concrete_python_checks workflow has changed - id: concrete-python-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/concrete_python_checks.yml - - - name: Check if optimizer workflow has changed - id: concrete-optimizer-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/optimizer.yml - - - name: Get changed files in the concrete-cpu directory - id: concrete-cpu-api - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./backends/concrete-cpu/implementation/include/** - - - name: Get changed files in the concrete-cuda directory - id: concrete-cuda-api - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./backends/concrete-cuda/implementation/include/** - - - name: Set some github event outputs - id: github - if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) - run: echo "push-main=true" >> "$GITHUB_OUTPUT" - -################################################# -# Compiler jobs ################################# - compiler-compliance: - needs: file-change - if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-format-and-linting-workflow == 'true' || needs.file-change.outputs.push-main == 'true' - uses: ./.github/workflows/compiler_format_and_linting.yml - - compiler-cpu-build: - needs: file-change - if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cpu-api == 'true'|| needs.file-change.outputs.concrete-compiler-cpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true' - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: compiler-cpu-build - - compiler-cpu-build-distributed: - needs: file-change - if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cpu-api == 'true'|| needs.file-change.outputs.concrete-compiler-cpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true' - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: compiler-cpu-build-distributed - - compiler-gpu-build: - needs: file-change - if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cuda-api == 'true' || needs.file-change.outputs.concrete-compiler-gpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true' - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: compiler-gpu-build - - compiler-macos-tests: - needs: file-change - if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-macos-workflow == 'true' || needs.file-change.outputs.push-main == 'true' - uses: ./.github/workflows/compiler_macos_build_and_test.yml - secrets: inherit - - compiler-publish-docker-images: - needs: file-change - if: (needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-docker-images-workflow == 'true') && (needs.file-change.outputs.push-main == 'true' || contains(github.ref, 'refs/heads/force-docker-images')) - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: compiler-publish-docker-images - - compiler-cpu-benchmark: - needs: file-change - if: needs.file-change.outputs.push-main == 'true' - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: compiler-cpu-benchmark - - # compiler-gpu-benchmark: - # needs: file-change - # if: needs.file-change.outputs.push-main == 'true' - # uses: ./.github/workflows/start_slab.yml - # secrets: inherit - # with: - # command: compiler-gpu-benchmark - -################################################# -# Optimizer jobs ################################ - optimizer: - needs: file-change - if: | - needs.file-change.outputs.parameters-curves == 'true' || - needs.file-change.outputs.concrete-cpu == 'true' || - needs.file-change.outputs.optimizer == 'true'|| - needs.file-change.outputs.push-main - uses: ./.github/workflows/optimizer.yml - secrets: inherit - -################################################# -# ConcreteCPU jobs ############################## - concrete-cpu: - needs: file-change - if: needs.file-change.outputs.concrete-cpu == 'true' || needs.file-change.outputs.concrete-cpu-workflow == 'true' || needs.file-change.outputs.push-main - uses: ./.github/workflows/concrete_cpu_test.yml - secrets: inherit - -################################################# -# Concrete Python jobs ########################## - concrete-python: - needs: file-change - if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.concrete-python-workflow == 'true' || needs.file-change.outputs.push-main - uses: ./.github/workflows/concrete_python_checks.yml - secrets: inherit - - concrete-python-tests-linux: - needs: file-change - if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.push-main - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-tests-linux - - concrete-python-tests-linux-gpu: - needs: file-change - if: needs.file-change.outputs.concrete-python == 'true' && needs.file-change.outputs.push-main - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-tests-linux-gpu - - concrete-python-tests-macos: - needs: file-change - if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.push-main - uses: ./.github/workflows/concrete_python_test_macos.yml - secrets: inherit - -################################################# -# Concrete-ML tests ############################# - concrete-ml-tests-linux: - needs: file-change - if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.push-main - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: ml-test - -################################################# -# Release jobs ################################# - concrete-python-nightly-release: - if: ${{ github.event_name == 'schedule' }} - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-release - user_inputs: 'nightly' - - concrete-python-nightly-release-gpu: - if: ${{ github.event_name == 'schedule' }} - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-release-gpu - user_inputs: 'nightly' - - concrete-python-public-release: -# needs: [compiler-cpu-build, compiler-macos-tests, compiler-publish-docker-images, concrete-python-tests-linux, concrete-python-tests-macos] - if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-release - user_inputs: 'public' - - concrete-python-public-release-gpu: - if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-release-gpu - user_inputs: 'public' - - concrete-python-private-release: - if: github.event_name == 'push' && contains(github.ref, 'refs/heads/private_release/') - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-release - user_inputs: 'private' diff --git a/.github/workflows/markdown_link_check.yml b/.github/workflows/markdown_link_check.yml deleted file mode 100644 index b2c6ffb849..0000000000 --- a/.github/workflows/markdown_link_check.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Check Markdown links - -on: - pull_request: - paths: - - '**.md' - - .github/workflows/markdown_link_check.yml - push: - branches: - - main - -jobs: - markdown-link-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1 - with: - use-quiet-mode: 'yes' - use-verbose-mode: 'yes' diff --git a/.github/workflows/optimizer_setup/action.yml b/.github/workflows/optimizer_setup/action.yml index d5887aa7d7..b4048c82d2 100644 --- a/.github/workflows/optimizer_setup/action.yml +++ b/.github/workflows/optimizer_setup/action.yml @@ -1,8 +1,3 @@ -inputs: - ssh_private_key: - description: 'A ssh key to access private github repository' - required: true - runs: using: "composite" steps: @@ -10,6 +5,8 @@ runs: uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7 with: toolchain: stable + default: true + override: true - name: Download cargo cache uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 diff --git a/.github/workflows/push_wheels_to_public_pypi.yml b/.github/workflows/push_wheels_to_public_pypi.yml deleted file mode 100644 index 3f1ff7d80d..0000000000 --- a/.github/workflows/push_wheels_to_public_pypi.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Push Wheels to Public PyPI - -on: - workflow_dispatch: - inputs: - version: - description: 'version of concrete-python to pull from Zama PyPI and push to public PyPI. Use the version as it appears in the wheel file (e.g. 2.7.0rc1)' - required: true - type: string - - -jobs: - pull_and_push: - runs-on: ubuntu-latest - steps: - - name: Pull wheels from S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} - S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }} - run: | - mkdir wheels - aws s3 cp s3://${S3_BUCKET_NAME}/cpu/concrete-python/ ./wheels/ --recursive --exclude "*" --include "concrete_python-${{ inputs.version }}-*" - echo "============== Downloaded wheels ===============" - ls -la ./wheels/ - - - name: Push wheels to public PyPI (public) - run: | - pip install twine==4.0.2 - twine upload wheels/concrete_python-${{ inputs.version }}*.whl \ - -u "${{ secrets.PUBLIC_PYPI_USER }}" \ - -p "${{ secrets.PUBLIC_PYPI_PASSWORD }}" \ - -r pypi - diff --git a/.github/workflows/scripts/teardown-check.sh b/.github/workflows/scripts/teardown-check.sh new file mode 100755 index 0000000000..bff9254ea7 --- /dev/null +++ b/.github/workflows/scripts/teardown-check.sh @@ -0,0 +1,10 @@ +#!/bin/bash -e + +grep setup-instance -Rl .github/workflows/ | xargs grep -L teardown-instance &> missing-teardown.txt + +if [ -s missing-teardown.txt ]; then + echo "There are missing teardown-instance jobs in following jobs:" + echo + cat missing-teardown.txt + exit 1 +fi diff --git a/.github/workflows/start_slab.yml b/.github/workflows/start_slab.yml deleted file mode 100644 index 0bdd7961c5..0000000000 --- a/.github/workflows/start_slab.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Start job on Slab CI bot given by input command. -name: Start AWS job - -on: - workflow_call: - inputs: - command: - required: true - type: string - user_inputs: - required: false - type: string - workflow_dispatch: - inputs: - command: - required: true - type: string - user_inputs: - description: 'user inputs to be forwarded to the called workflow' - required: false - type: string - -env: - GIT_REF: ${{ github.head_ref }} - -jobs: - sl: - runs-on: ubuntu-latest - steps: - - name: Checkout concrete - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - name: Get git ref - # github.head_ref is only available from a Pull Request - if: env.GIT_REF == '' - run: | - echo "GIT_REF=${{ github.ref_name }}" >> $GITHUB_ENV - - - name: Checkout Slab repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Start AWS job in Slab - shell: bash - run: | - GIT_SHA="$(git --no-pager show -s --format="%H" origin/${{ env.GIT_REF }})" || GIT_SHA=${{ github.sha }} - echo -n '{"command": "${{ inputs.command }}", "git_ref": "${{ env.GIT_REF }}", "sha":"'${GIT_SHA}'", "user_inputs": "${{ inputs.user_inputs }}"}' > command.json - cat command.json - SIGNATURE="$(slab/scripts/hmac_calculator.sh command.json '${{ secrets.JOB_SECRET }}')" - curl -v -k \ - --fail-with-body \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: start_aws" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @command.json \ - ${{ secrets.SLAB_URL }} diff --git a/ci/ec2_products_cost.json b/ci/ec2_products_cost.json index 436cabb134..d43366be82 100644 --- a/ci/ec2_products_cost.json +++ b/ci/ec2_products_cost.json @@ -1,4 +1,5 @@ { + "hpc7a.96xlarge": 7.200, "m7i.48xlarge": 9.677, "m7i.metal-48xl": 9.677, "m6i.metal": 7.168, diff --git a/ci/slab.toml b/ci/slab.toml index 0a42840d02..07e1a293e9 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -1,136 +1,29 @@ -# This is the new version of Slab that handles multi backend providers. -[backend.aws.m7i-cpu-bench] +[backend.aws.cpu-test] region = "eu-west-1" image_id = "ami-002bdcd64b8472cf9" # Based on Ubuntu 22.4 -instance_type = "m7i.48xlarge" -security_group = ["sg-0e55cc31dfda0d8a7", ] - -[profile.m7i-cpu-bench] -region = "eu-west-1" -image_id = "ami-002bdcd64b8472cf9" # Based on Ubuntu 22.4 -instance_type = "m7i.48xlarge" -security_group= ["sg-0e55cc31dfda0d8a7", ] - -[profile.m7i-cpu-test] -region = "eu-west-1" -image_id = "ami-002bdcd64b8472cf9" instance_type = "m7i.16xlarge" -security_group= ["sg-0e55cc31dfda0d8a7", ] +security_group = ["sg-0e55cc31dfda0d8a7", ] -[profile.m7i-metal] +[backend.aws.cpu-bench] region = "eu-west-1" image_id = "ami-002bdcd64b8472cf9" -instance_type = "m7i.metal-24xl" -security_group= ["sg-0e55cc31dfda0d8a7", ] - -[profile.gpu-bench] -region = "us-east-1" -image_id = "ami-08e27480d79e82238" -instance_type = "p3.2xlarge" -subnet_id = "subnet-8123c9e7" -security_group= ["sg-017afab1f328af917", ] +instance_type = "hpc7a.96xlarge" -# Docker is well configured for test inside docker in this AMI -[profile.gpu-test] +[backend.aws.gpu-test] region = "us-east-1" image_id = "ami-0257c6ad39f902b5e" instance_type = "p3.2xlarge" subnet_id = "subnet-8123c9e7" security_group= ["sg-017afab1f328af917", ] -# It has CUDA Driver (<=12.5) and Docker installed -[profile.gpu-test-ubuntu22] -region = "us-east-1" -image_id = "ami-05385e0c3c574621f" -instance_type = "p3.2xlarge" -subnet_id = "subnet-8123c9e7" -security_group= ["sg-017afab1f328af917", ] - -[profile.slurm-cluster] +[backend.aws.slurm-cluster] region = "eu-west-3" image_id = "ami-0bb5bb9cb747b5ddd" instance_id = "i-0e5ae2a14134d6275" instance_type = "m6i.8xlarge" security_group= ["sg-02dd8470fa845f31b", ] -################################################# -# Compiler commands -################################################# - -[command.compiler-cpu-build] -workflow = "compiler_build_and_test_cpu.yml" -profile = "m7i-cpu-test" -check_run_name = "Compiler Build and Test (CPU)" - -[command.compiler-cpu-build-distributed] -workflow = "compiler_build_and_test_cpu_distributed.yml" -profile = "slurm-cluster" -check_run_name = "Compiler Distributed Build and Test (CPU)" -runner_name = "distributed-ci" - -[command.compiler-gpu-build] -workflow = "compiler_build_and_test_gpu.yml" -profile = "gpu-test" -check_run_name = "Compiler Build and Test (GPU)" - -[command.compiler-cpu-benchmark] -workflow = "compiler_benchmark.yml" -profile = "m7i-cpu-bench" -check_run_name = "Compiler Performances Benchmarks (CPU)" - -[command.compiler-gpu-benchmark] -workflow = "compiler_benchmark.yml" -profile = "gpu-bench" -check_run_name = "Compiler Performances Benchmarks (GPU)" - -# Trigger Docker images build -[command.compiler-publish-docker-images] -workflow = "compiler_publish_docker_images.yml" -profile = "m7i-cpu-test" -check_run_name = "Compiler - Docker images build & publish" - -# Trigger ML benchmarks by running each use cases subset in parallel. -[command.ml-bench] -workflow = "ml_benchmark_subset.yml" -profile = "m7i-cpu-bench" -matrix = [0,1,2,3,4,5,6,7,8,9,10] -max_parallel_jobs = 2 - -# Trigger ML tests with latest CP -[command.ml-test] -workflow = "concrete_ml_tests.yml" -profile = "m7i-cpu-test" -check_run_name = "Concrete ML Tests" - -################################################# -# Concrete Python Commands -################################################# - -[command.concrete-python-tests-linux] -workflow = "concrete_python_tests_linux.yml" -profile = "m7i-cpu-test" -check_run_name = "Concrete Python Tests (Linux)" - -[command.concrete-python-tests-linux-gpu] -workflow = "concrete_python_tests_linux.yml" -profile = "gpu-test" -check_run_name = "Concrete Python Tests (Linux Gpu)" - -################################################# -# Release Commands -################################################# - -[command.concrete-python-release] -workflow = "concrete_python_release.yml" -profile = "m7i-cpu-test" -check_run_name = "Concrete Python Release" - -[command.concrete-python-release-gpu] -workflow = "concrete_python_release_gpu.yml" -profile = "m7i-cpu-test" -check_run_name = "Concrete Python Release (GPU)" - -[command.concrete-python-test-gpu-wheel] -workflow = "concrete_python_test_gpu_wheel.yml" -profile = "gpu-test" -check_run_name = "Concrete Python Test GPU Wheel" +[backend.aws.release] +region = "eu-west-1" +image_id = "ami-002bdcd64b8472cf9" +instance_type = "hpc7a.96xlarge" diff --git a/compilers/concrete-compiler/compiler/Makefile b/compilers/concrete-compiler/compiler/Makefile index c67d5dd2d8..58f5de526c 100644 --- a/compilers/concrete-compiler/compiler/Makefile +++ b/compilers/concrete-compiler/compiler/Makefile @@ -479,33 +479,6 @@ else detected_OS := $(shell sh -c 'uname 2>/dev/null || echo Unknown') endif -PIP=$(Python3_EXECUTABLE) -m pip -PIP_WHEEL=$(PIP) wheel --no-deps -w $(BUILD_DIR)/wheels . -AUDIT_WHEEL_REPAIR=$(Python3_EXECUTABLE) -m auditwheel repair -w $(BUILD_DIR)/wheels - -linux-python-package: - $(PIP) install wheel auditwheel - # We need to run it twice: the first will generate the directories, so that - # the second run can find the packages via find_namespace_packages - $(PIP_WHEEL) - $(PIP_WHEEL) - GLIBC_VER=$(shell ldd --version | head -n 1 | grep -o '[^ ]*$$'|head|tr '.' '_'); \ - for PLATFORM in manylinux_$${GLIBC_VER}_x86_64 linux_x86_64; do \ - if $(AUDIT_WHEEL_REPAIR) $(BUILD_DIR)/wheels/*.whl --plat $$PLATFORM; then \ - echo Success for $$PLATFORM; \ - break; \ - else \ - echo No repair with $$PLATFORM; \ - fi \ - done - -darwin-python-package: - $(PIP) install wheel delocate - $(PIP_WHEEL) - delocate-wheel -v $(BUILD_DIR)/wheels/*macosx*.whl - -python-package: python-bindings $(OS)-python-package - @echo The python package is: $(BUILD_DIR)/wheels/*.whl install: concretecompiler install-deps $(info Install prefix set to $(INSTALL_PREFIX)) diff --git a/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt b/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt index 021be4705c..199b6a5506 100644 --- a/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt +++ b/compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt @@ -1,3 +1,4 @@ black==24.4.0 pylint==2.11.1 mypy==1.11.2 +numpy>=1.23,<2.0 diff --git a/frontends/concrete-python/Makefile b/frontends/concrete-python/Makefile index ef04545803..7b7ace237f 100644 --- a/frontends/concrete-python/Makefile +++ b/frontends/concrete-python/Makefile @@ -30,9 +30,6 @@ CONCRETE_VERSION?="" # empty mean latest venv: $(PYTHON) -m venv .venv . .venv/bin/activate -ifeq (,$(wildcard ${RUNTIME_LIBRARY})) - $(PIP) install --extra-index-url https://pypi.zama.ai/cpu "concrete-python$(CONCRETE_VERSION)" -endif $(PIP) install -r requirements.dev.txt $(PIP) install -r requirements.extra-full.txt $(PIP) install -r requirements.txt diff --git a/frontends/concrete-python/scripts/checks/checks.sh b/frontends/concrete-python/scripts/checks/checks.sh deleted file mode 100755 index 25a3bbb6c1..0000000000 --- a/frontends/concrete-python/scripts/checks/checks.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -ex - -cd frontends/concrete-python -make venv -source .venv/bin/activate -make pcc