From da3b9f9a80885df9d959fc2e225b903a7946a063 Mon Sep 17 00:00:00 2001 From: Bourgerie Quentin Date: Fri, 18 Oct 2024 18:35:32 +0200 Subject: [PATCH] refactor ci --- .github/workflows/check_commit_signature.yml | 11 - ...{block_merge.yml => commit_compliance.yml} | 24 +- .../workflows/concrete_python_test_macos.yml | 98 +++---- .../workflows/concrete_python_tests_linux.yml | 254 +++++++----------- .github/workflows/linelint.yml | 18 -- ci/slab.toml | 14 + 6 files changed, 172 insertions(+), 247 deletions(-) delete mode 100644 .github/workflows/check_commit_signature.yml rename .github/workflows/{block_merge.yml => commit_compliance.yml} (63%) delete mode 100644 .github/workflows/linelint.yml diff --git a/.github/workflows/check_commit_signature.yml b/.github/workflows/check_commit_signature.yml deleted file mode 100644 index ad2fa41191..0000000000 --- a/.github/workflows/check_commit_signature.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: Check Commit Signatures - -on: - pull_request: - -jobs: - check-commit-signatures: - runs-on: ubuntu-latest - steps: - - name: Check commit signatures - uses: 1Password/check-signed-commits-action@ed2885f3ed2577a4f5d3c3fe895432a557d23d52 diff --git a/.github/workflows/block_merge.yml b/.github/workflows/commit_compliance.yml similarity index 63% rename from .github/workflows/block_merge.yml rename to .github/workflows/commit_compliance.yml index 6500845837..361b24e199 100644 --- a/.github/workflows/block_merge.yml +++ b/.github/workflows/commit_compliance.yml @@ -1,13 +1,10 @@ -# Check commit and PR compliance -name: Check commit message compliance +name: check commit compliance on: pull_request: - types: [opened, synchronize, reopened] jobs: - check-commit-pr: - name: Check commit and PR - runs-on: ubuntu-20.04 + format: + runs-on: ubuntu-latest steps: - name: Check first line uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee # v2.0.0 @@ -19,3 +16,18 @@ jobs: excludeTitle: 'true' # optional: this excludes the title of a pull request checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true + + signatures: + runs-on: ubuntu-latest + steps: + - name: Check commit signatures + uses: 1Password/check-signed-commits-action@ed2885f3ed2577a4f5d3c3fe895432a557d23d52 + + linelint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Linelint + uses: fernandrone/linelint@0.0.4 + id: linelint \ No newline at end of file diff --git a/.github/workflows/concrete_python_test_macos.yml b/.github/workflows/concrete_python_test_macos.yml index 33a3977c36..14e22d48bf 100644 --- a/.github/workflows/concrete_python_test_macos.yml +++ b/.github/workflows/concrete_python_test_macos.yml @@ -1,40 +1,37 @@ -name: Concrete Python Tests (macOS) +name: concrete-python tests macos on: - workflow_call: workflow_dispatch: - secrets: - CONCRETE_CI_SSH_PRIVATE: - required: true - CONCRETE_ACTIONS_TOKEN: - required: true + pull_request: + path: + - .github/workflows/concrete_python_tests_macos.yml + push: + branches: + - 'main' + - 'release/*' concurrency: - group: concrete_python_tests_macos-${{ github.ref }} + group: concrete_python_tests_macos_${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - BuildConcreteCompilerAndTestConcretePythonInMacOS: + concrete-python-test-pytest: strategy: fail-fast: false matrix: machine: ["aws-mac1-metal", "aws-mac2-metal"] - runs-on: ${{ matrix.machine }} steps: - - name: Checkout + - name: Checkout concrete uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Install OS Dependencies + fetch-depth: 0 + - name: Install build dependencies run: | brew install ninja ccache - - name: Setup rust toolchain for concrete-cpu uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Cache Compilation (push) + - name: Cache compilation (push) if: github.event_name == 'push' uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: @@ -42,8 +39,7 @@ jobs: key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.sha }} restore-keys: | ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - - name: Cache Compilation (pull_request) + - name: Cache compilation (pull_request) if: github.event_name == 'pull_request' uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: @@ -51,88 +47,72 @@ jobs: key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.event.pull_request.base.sha }} restore-keys: | ${{ runner.os }}-${{ runner.arch }}-compilation-cache- - - - name: Get tmpdir path - if: github.event_name == 'push' - id: tmpdir-path - run: echo "::set-output name=TMPDIR_PATH::$TMPDIR" - - - name: Build + - name: Prepare build environment run: | set -e - cd frontends/concrete-python - + cd $GITHUB_WORKSPACE/frontends/concrete-python + # Setup pkg-config to find OpenBLAS (scipy need it) export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" - + rm -rf .venv python3.10 -m venv .venv - - . .venv/bin/activate - + + . $GITHUB_WORKSPACE/frontends/concrete-python/.venv/bin/activate + pip install -r requirements.dev.txt pip install -r requirements.txt - + - name: Build concrete-compiler python-bindings + run: | + $GITHUB_WORKSPACE/frontends/concrete-python .venv/bin/activate cd $GITHUB_WORKSPACE/compilers/concrete-compiler/compiler - - echo "Debug: ccache statistics (prior to the build):" - ccache -s - + ccache -z make Python3_EXECUTABLE=$(which python) python-bindings - - echo "Debug: ccache statistics (after the build):" ccache -s - - export COMPILER_BUILD_DIRECTORY=$(pwd)/build + - name: Create wheels + run: | + $GITHUB_WORKSPACE/frontends/concrete-python .venv/bin/activate cd $GITHUB_WORKSPACE/frontends/concrete-python - - rm -rf dist - mkdir -p dist - + + export COMPILER_BUILD_DIRECTORY=$GITHUB_WORKSPACE/compilers/concrete-compiler/compiler + rm -rf dist && mkdir -p dist pip wheel -v --no-deps -w dist . delocate-wheel -v dist/*macos*.whl - - deactivate + deactivate - name: Prepare test environment run: | set -e export TEST_TMP_DIR=$(mktemp -d) echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}" cd $TEST_TMP_DIR - + python3.10 -m venv .testenv . .testenv/bin/activate - + pip install $GITHUB_WORKSPACE/frontends/concrete-python/dist/*macos*.whl pip install -r $GITHUB_WORKSPACE/frontends/concrete-python/requirements.dev.txt # MacOS x86 have conflict between our OpenMP library, and one from torch # we fix it by using a single one (from torch) # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8 - + find .testenv/lib/python3.10/site-packages -not \( -path .testenv/lib/python3.10/site-packages/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1 ln -f -s $(pwd)/.testenv/lib/python3.10/site-packages/concrete/.dylibs/libomp.dylib cp -R $GITHUB_WORKSPACE/frontends/concrete-python/examples ./examples cp -R $GITHUB_WORKSPACE/frontends/concrete-python/tests ./tests - - cp $GITHUB_WORKSPACE/frontends/concrete-python/Makefile . - - name: Test + cp $GITHUB_WORKSPACE/frontends/concrete-python/Makefile . + - name: Run pytest-macos run: | - set -e - export TEST_TMP_DIR="testing_concrete_python" cd $TEST_TMP_DIR && . .testenv/bin/activate KEY_CACHE_DIRECTORY=./KeySetCache PYTEST_MARKERS="not dataflow and not graphviz" make pytest-macos - - - name: Test notebooks + - name: Run test-notebooks run: | set -e - export TEST_TMP_DIR="testing_concrete_python" cd $TEST_TMP_DIR && . .testenv/bin/activate make test-notebooks - - name: Cleanup host if: success() || failure() run: | diff --git a/.github/workflows/concrete_python_tests_linux.yml b/.github/workflows/concrete_python_tests_linux.yml index 7090bc891f..69d79c9088 100644 --- a/.github/workflows/concrete_python_tests_linux.yml +++ b/.github/workflows/concrete_python_tests_linux.yml @@ -1,69 +1,52 @@ -name: Concrete Python Tests (Linux) +name: concrete-python tests linux-cpu on: workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - -# concurrency: -# group: concrete_python_tests_linux-${{ github.ref }} -# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - + pull_request: + path: + - .github/workflows/concrete_python_tests_linux.yml + - frontends/concrete-python/** + push: + branches: + - 'main' + - 'release/*' env: DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler - CUDA_PATH: /usr/local/cuda-11.8 - GCC_VERSION: 11 - GLIB_VER: 2_28 - +concurrency: + group: concrete_python_tests_linux_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - BuildConcreteCompilerAndTestConcretePythonInLinux: - name: Build Concrete Compiler and Test Concrete Python in Linux - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} + setup-instance: + name: Setup Instance + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} steps: - - name: Log instance configuration - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Set up GitHub environment - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - #echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - - - name: Checkout + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-test + + build-python-bindings: + name: Build concrete-compiler python bindings + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Checkout concrete uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: recursive - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Create build directory + fetch-depth: 0 + - name: Create concrete build directory run: mkdir build - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Build bindings + - name: Build concrete-compiler python bindings uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} - id: build-compiler-bindings with: registry: ghcr.io image: ${{ env.DOCKER_IMAGE_TEST }} @@ -72,97 +55,45 @@ jobs: options: >- -v ${{ github.workspace }}:/concrete -v ${{ github.workspace }}/build:/build - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket shell: bash run: | - rustup toolchain install nightly-2024-09-30 - pip install mypy set -e - rm -rf /build/* + rustup toolchain install nightly-2024-09-30 dnf -y install graphviz graphviz-devel + rm -rf /build/* + cd /concrete/frontends/concrete-python make venv source .venv/bin/activate - + cd /concrete/compilers/concrete-compiler/compiler make BUILD_DIR=/build DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$(which python3) python-bindings - + echo "Debug: ccache statistics (after the build):" ccache -s - - - name: Prepare test environment - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} + - name: Upload concrete-compiler python-bindings + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - shell: bash - run: | - set -e - - dnf -y install graphviz graphviz-devel - - cd /concrete/frontends/concrete-python - make venv - - - name: Test - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - shell: bash - run: | - set -e - - cd /concrete/frontends/concrete-python - source .venv/bin/activate - - export COMPILER_BUILD_DIRECTORY=/build - - mkdir ./KeySetCache - KEY_CACHE_DIRECTORY=./KeySetCache make pytest - - - name: Test notebooks - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ !contains(inputs.instance_type, 'p3') }} + name: concrete-compiler-python-bindings + retention-days: 3 + path: | + build/lib/libConcretelangRuntime.so + build/tools/concretelang/python_packages/concretelang_core/ + + test-pytest: + name: Test concrete-python pytest + needs: [setup-instance, build-python-bindings] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Download concrete-compiler python-bindings + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - shell: bash - run: | - set -e - - cd /concrete/frontends/concrete-python - source .venv/bin/activate - - export COMPILER_BUILD_DIRECTORY=/build - - make test-notebooks - - - - name: Build bindings gpu + name: concrete-compiler-python-bindings + path: /compiler-artifacts-pytest + - name: Run pytest uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ contains(inputs.instance_type, 'p3') }} - id: build-compiler-bindings-gpu with: registry: ghcr.io image: ${{ env.DOCKER_IMAGE_TEST }} @@ -171,28 +102,29 @@ jobs: options: >- -v ${{ github.workspace }}:/concrete -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - --gpus all shell: bash run: | set -e - rm -rf /build/* - + export COMPILER_BUILD_DIRECTORY=/compiler-artifacts-pytest cd /concrete/frontends/concrete-python make venv source .venv/bin/activate - - cd /concrete/compilers/concrete-compiler/compiler - make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python3) CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} python-bindings - - echo "Debug: ccache statistics (after the build):" - ccache -s - - - name: Test gpu + export KEY_CACHE_DIRECTORY=./key-set-cache + mkdir $KEY_CACHE_DIRECTORY + make pytest + + test-notebooks: + name: Test concrete-python notebooks + needs: [setup-instance, build-python-bindings] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Download concrete-compiler python-bindings + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: concrete-compiler-python-bindings + path: /compiler-artifacts-notebooks + - name: Run pytest uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - if: ${{ contains(inputs.instance_type, 'p3') }} with: registry: ghcr.io image: ${{ env.DOCKER_IMAGE_TEST }} @@ -201,18 +133,34 @@ jobs: options: >- -v ${{ github.workspace }}:/concrete -v ${{ github.workspace }}/build:/build - -v ${{ github.workspace }}/wheels:/wheels - --gpus all shell: bash run: | set -e - + export COMPILER_BUILD_DIRECTORY=/compiler-artifacts-notebooks cd /concrete/frontends/concrete-python - make venv source .venv/bin/activate + make test-notebooks - export COMPILER_BUILD_DIRECTORY=/build - KEY_CACHE_DIRECTORY=/tmp/KeySetCache mkdir ./KeySetCache - make pytest-gpu - - chmod -R ugo+rwx /tmp/KeySetCache + teardown-instance: + name: Teardown instance + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, test-pytest, test-notebooks ] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" \ No newline at end of file diff --git a/.github/workflows/linelint.yml b/.github/workflows/linelint.yml deleted file mode 100644 index 616f04d407..0000000000 --- a/.github/workflows/linelint.yml +++ /dev/null @@ -1,18 +0,0 @@ -# This job is the main jobs will dispatch build and test for every modules of our mono repo. -name: Linelint - -on: - pull_request: - push: - branches: - - 'main' - -jobs: - linelint: - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - name: Linelint - uses: fernandrone/linelint@0.0.4 - id: linelint diff --git a/ci/slab.toml b/ci/slab.toml index 0a42840d02..64b4489c41 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -1,3 +1,17 @@ +[backend.aws.cpu-test] +region = "eu-west-1" +image_id = "ami-002bdcd64b8472cf9" # Based on Ubuntu 22.4 +instance_type = "m7i.16xlarge" +security_group = ["sg-0e55cc31dfda0d8a7", ] + +[backend.aws.gpu-test] +region = "us-east-1" +image_id = "ami-0257c6ad39f902b5e" +instance_type = "p3.2xlarge" +subnet_id = "subnet-8123c9e7" +security_group= ["sg-017afab1f328af917", ] + + # This is the new version of Slab that handles multi backend providers. [backend.aws.m7i-cpu-bench] region = "eu-west-1"