From 72f63acb169153981f4a0570c2e0f8efde109416 Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Thu, 7 Nov 2024 11:10:02 +0100 Subject: [PATCH] chore(ci): Remove release-checks for pushing releases on private pypi --- .github/workflows/action_compliance.yaml | 9 +- .github/workflows/commit_compliance.yml | 18 +- ...oncrete_compiler_publish_docker_images.yml | 23 + .../workflows/concrete_compiler_test_gpu.yml | 23 + .../workflows/concrete_python_benchmark.yml | 18 +- .../concrete_python_push_docker_image.yml | 55 --- .github/workflows/concrete_python_release.yml | 403 ------------------ .../workflows/concrete_python_release_gpu.yml | 217 +++++++--- .github/workflows/main.yml | 303 ------------- .github/workflows/markdown_link_check.yml | 20 - .../workflows/push_wheels_to_public_pypi.yml | 35 -- .github/workflows/scripts/teardown-check.sh | 10 + .github/workflows/start_slab.yml | 62 --- ci/slab.toml | 136 +----- 14 files changed, 245 insertions(+), 1087 deletions(-) delete mode 100644 .github/workflows/concrete_python_push_docker_image.yml delete mode 100644 .github/workflows/concrete_python_release.yml delete mode 100644 .github/workflows/main.yml delete mode 100644 .github/workflows/markdown_link_check.yml delete mode 100644 .github/workflows/push_wheels_to_public_pypi.yml create mode 100755 .github/workflows/scripts/teardown-check.sh delete mode 100644 .github/workflows/start_slab.yml diff --git a/.github/workflows/action_compliance.yaml b/.github/workflows/action_compliance.yaml index f24c6b508d..df44d2e1a1 100644 --- a/.github/workflows/action_compliance.yaml +++ b/.github/workflows/action_compliance.yaml @@ -13,7 +13,8 @@ jobs: action-pin: runs-on: ubuntu-latest steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - name: checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - name: Ensure SHA pinned actions uses: zgosalvez/github-actions-ensure-sha-pinned-actions@0901cf7b71c7ea6261ec69a3dc2bd3f9264f893e # v3.0.12 with: @@ -23,7 +24,9 @@ jobs: action-lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - + - name: checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - name: check-missing-teardown + run: ./workflows/scripts/teardown-check.sh - name: actionlint uses: raven-actions/actionlint@01fce4f43a270a612932cb1c64d40505a029f821 # v2.0.0 diff --git a/.github/workflows/commit_compliance.yml b/.github/workflows/commit_compliance.yml index fe0f13e8de..8c5ae9d9d3 100644 --- a/.github/workflows/commit_compliance.yml +++ b/.github/workflows/commit_compliance.yml @@ -27,11 +27,23 @@ jobs: - name: Check commit signatures uses: 1Password/check-signed-commits-action@ed2885f3ed2577a4f5d3c3fe895432a557d23d52 - linelint: + files-lint: runs-on: ubuntu-latest steps: - - name: Checkout + - name: checkout uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - name: Linelint + - name: linelint uses: fernandrone/linelint@8136e0fa9997122d80f5f793e0bb9a45e678fbb1 # 0.0.4 id: linelint + - name: markdown-link-check + uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1 + with: + use-quiet-mode: 'yes' + use-verbose-mode: 'yes' + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "files-lint finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_compiler_publish_docker_images.yml b/.github/workflows/concrete_compiler_publish_docker_images.yml index 5209190fb8..be21a7ae37 100644 --- a/.github/workflows/concrete_compiler_publish_docker_images.yml +++ b/.github/workflows/concrete_compiler_publish_docker_images.yml @@ -185,3 +185,26 @@ jobs: env: SLACK_COLOR: ${{ job.status }} SLACK_MESSAGE: "cuda-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instane: + name: Teardown instance + needs: [ setup-instance, compiler-image ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" \ No newline at end of file diff --git a/.github/workflows/concrete_compiler_test_gpu.yml b/.github/workflows/concrete_compiler_test_gpu.yml index 58f4d8d551..0e674e293a 100644 --- a/.github/workflows/concrete_compiler_test_gpu.yml +++ b/.github/workflows/concrete_compiler_test_gpu.yml @@ -81,3 +81,26 @@ jobs: env: SLACK_COLOR: ${{ job.status }} SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + needs: [ setup-instance, build-and-test ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/concrete_python_benchmark.yml b/.github/workflows/concrete_python_benchmark.yml index 34fcf6a7df..5142d117c2 100644 --- a/.github/workflows/concrete_python_benchmark.yml +++ b/.github/workflows/concrete_python_benchmark.yml @@ -13,7 +13,10 @@ on: - 'release/*' env: DOCKER_IMAGE: ghcr.io/zama-ai/concrete-compiler - GLIB_VER: 2_28 + +concurrency: + group: concrete_python_benchmark_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: setup-instance: @@ -73,7 +76,7 @@ jobs: export COMPILER_BUILD_DIRECTORY="/build" # TODO output setup-instance (https://github.com/zama-ai/slab-github-runner/issues/38) export PROGRESS_MACHINE_NAME="hpc7a.96xlarge" - + make benchmark make process-benchmark-results-for-grafana @@ -91,9 +94,9 @@ jobs: run: | echo "Computing HMac on results file" SIGNATURE="$(slab/scripts/hmac_calculator.sh frontends/concrete-python/progress.processed.json '${{ secrets.JOB_SECRET }}')" - + cd frontends/concrete-python - + echo "Sending results to Slab..." curl -v -k \ -H "Content-Type: application/json" \ @@ -102,6 +105,13 @@ jobs: -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ -d @progress.processed.json \ ${{ secrets.SLAB_URL }} + - name: Slack Notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "concrete-python-benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" teardown-instance: if: ${{ always() && needs.setup-instance.result != 'skipped' }} diff --git a/.github/workflows/concrete_python_push_docker_image.yml b/.github/workflows/concrete_python_push_docker_image.yml deleted file mode 100644 index f88248a464..0000000000 --- a/.github/workflows/concrete_python_push_docker_image.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Concrete Python Push Docker Image -on: - workflow_dispatch: - inputs: - tag: - description: 'tag to use for the docker image' - type: string - workflow_call: - inputs: - tag: - description: 'tag to use for the docker image' - type: string - -env: - DOCKER_IMAGE_NAME: zamafhe/concrete-python - DOCKER_FILE: docker/Dockerfile.concrete-python - -jobs: - build_and_push: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Get version from tag - run: | - # remove leading 'v' and '-' from tag - export VERSION=`echo ${{ inputs.tag }} | sed "s/^v*//g" | sed "s/-//g"` - echo "VERSION=$VERSION" >> "${GITHUB_ENV}" - echo "NAME_TAG=${{ env.DOCKER_IMAGE_NAME }}:v$VERSION" >> "${GITHUB_ENV}" - - - name: Build image - run: | - mkdir empty_context - docker image build -t ${{ env.NAME_TAG }} --build-arg version=${{ env.VERSION }} -f ${{ env.DOCKER_FILE }} empty_context - - # disabled because of https://github.com/aquasecurity/trivy/discussions/7668 - # - name: Run Trivy vulnerability scanner - # uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0 - # with: - # image-ref: '${{ env.NAME_TAG }}' - # format: 'table' - # exit-code: '1' - # ignore-unfixed: true - # vuln-type: 'os,library' - # severity: 'CRITICAL,HIGH' - - - name: Login to Docker Hub - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Push image - run: docker image push ${{ env.NAME_TAG }} diff --git a/.github/workflows/concrete_python_release.yml b/.github/workflows/concrete_python_release.yml deleted file mode 100644 index 5c4a3c82ab..0000000000 --- a/.github/workflows/concrete_python_release.yml +++ /dev/null @@ -1,403 +0,0 @@ -name: Concrete Python Release -on: - workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - user_inputs: - description: 'either "nightly" or "public" or "private" to specify the release type' - required: true - default: 'nightly' - type: string - - -env: - DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler - GLIB_VER: 2_28 - RELEASE_TYPE: ${{ inputs.user_inputs }} - -jobs: - release-checks: - runs-on: ${{ github.event.inputs.runner_name }} - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - fetch-depth: 0 - - name: Check python api doc is up to date - run: ci/scripts/make_apidocs.sh - - build-linux-x86: - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - - runs-on: ${{ github.event.inputs.runner_name }} - steps: - - name: Log instance configuration - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "User Inputs: ${{ inputs.user_inputs }}" - - - name: Set up GitHub environment - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - fetch-depth: 0 - - - name: Set release version (nightly) - if: ${{ env.RELEASE_TYPE == 'nightly' }} - run: | - NIGHTLY_VERSION=$(date +"%Y.%m.%d") - NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d") - LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o` - echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt - git tag nightly-$NIGHTLY_VERSION || true - git push origin nightly-$NIGHTLY_VERSION || true - - - name: Set release version (public) - if: ${{ env.RELEASE_TYPE == 'public' }} - run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt - - - name: Expose release version from Python - run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py - - - name: Build wheel - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - id: build-compiler-bindings - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}:/concrete - -v ${{ github.workspace }}/build:/build - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - ${{ env.DOCKER_GPU_OPTION }} - shell: bash - run: | - set -e - - rustup toolchain install nightly-2024-09-30 - pip install mypy - rm -rf /build/* - - export PYTHON=${{ format('python{0}', matrix.python-version) }} - echo "Using $PYTHON" - - dnf -y install graphviz graphviz-devel - - cd /concrete/frontends/concrete-python - make PYTHON=$PYTHON venv - source .venv/bin/activate - - cd /concrete/compilers/concrete-compiler/compiler - make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings - - echo "Debug: ccache statistics (after the build):" - ccache -s - - cd /concrete/frontends/concrete-python - - export COMPILER_BUILD_DIRECTORY="/build" - make whl - - deactivate - - - name: Upload wheel - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} - path: frontends/concrete-python/dist/*manylinux*.whl - retention-days: 3 - - build-macos: - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - runs-on: ["aws-mac1-metal", "aws-mac2-metal"] - - runs-on: ${{ matrix.runs-on }} - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - fetch-depth: 0 - - - name: Install OS Dependencies - run: | - brew install ninja ccache - - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu - - - name: Set release version (nightly) - if: ${{ env.RELEASE_TYPE == 'nightly' }} - run: | - NIGHTLY_VERSION=$(date +"%Y.%m.%d") - NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d") - LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o` - echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt - - - name: Set release version (public) - if: ${{ env.RELEASE_TYPE == 'public' }} - run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt - - - name: Expose release version from Python - run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py - - - name: Build wheel - run: | - export CONCRETE_PYTHON=$(pwd)/frontends/concrete-python - export CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler - export COMPILER_BUILD_DIRECTORY=$CONCRETE_COMPILER/build - export PYTHON=${{ format('python{0}', matrix.python-version) }} - echo "Using $PYTHON" - - # Setup pkg-config to find OpenBLAS (scipy need it) - export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" - - # Setup vitual environment - rm -rf .venv - $PYTHON -m venv .venv && . .venv/bin/activate - - # Install requirements - pip install -r $CONCRETE_PYTHON/requirements.txt - pip install -r $CONCRETE_PYTHON/requirements.dev.txt - - # Build python bindings of concrete compiler - cd $CONCRETE_COMPILER - echo "Debug: ccache statistics (prior to the build):" && ccache -s - make Python3_EXECUTABLE=$(which python) python-bindings - echo "Debug: ccache statistics (after the build):" && ccache -s - - # Build wheel - cd $CONCRETE_PYTHON - rm -rf dist - mkdir -p dist - pip wheel -v --no-deps -w dist . - delocate-wheel -v dist/*macos*.whl - - deactivate - - - name: Upload wheel - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} - path: frontends/concrete-python/dist/*macos*.whl - retention-days: 3 - - hash: - # Generate hashes for the wheels, used later for provenance. - needs: [build-linux-x86, build-macos] - runs-on: ubuntu-latest - outputs: - hash: ${{ steps.hash.outputs.hash }} - steps: - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - path: frontends/concrete-python/dist - pattern: wheel-* - merge-multiple: true - - name: generate hash - id: hash - run: cd frontends/concrete-python/dist && echo "hash=$(sha256sum *.whl | base64 -w0)" >> $GITHUB_OUTPUT - - provenance: - needs: [hash] - permissions: - actions: read - id-token: write - contents: write - # Can't pin with hash due to how this workflow works. - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 - with: - base64-subjects: ${{ needs.hash.outputs.hash }} - - push: - needs: [release-checks, build-linux-x86, build-macos, provenance] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - path: wheels - pattern: 'wheel-*' - merge-multiple: true - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - pattern: '*.intoto.jsonl' - # When building a new public tag, create a new draft release. - - name: create draft release - if: ${{ env.RELEASE_TYPE == 'public'}} - run: | - export TAG=$(git describe --tags --abbrev=0) - echo $TAG - gh release create --draft --repo ${{ github.repository }} \ - --verify-tag $TAG \ - --title $TAG \ - wheels/* *.intoto.jsonl/* - env: - GH_TOKEN: ${{ github.token }} - - name: Upload wheels to S3 - if: ${{ env.RELEASE_TYPE == 'public' || env.RELEASE_TYPE == 'nightly' }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} - S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }} - CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.AWS_CLOUDFRONT_PYPI_DISTRIBUTION_ID }} - run: | - pip install boto3 bigtree - # upload wheels - aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/cpu/concrete-python - # update indexes and invalidate cloudfront cache - python .github/workflows/scripts/s3_update_html_indexes.py - - - name: Start pushing Docker images - if: ${{ env.RELEASE_TYPE == 'public' }} - run: | - export TAG=$(git describe --tags --abbrev=0) - curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/zama-ai/concrete/actions/workflows/concrete_python_push_docker_image.yml/dispatches \ - -d "{\"ref\": \"$TAG\", \"inputs\": {\"tag\":\"v$TAG\"}}" - - test-linux-x86: - needs: [build-linux-x86] - continue-on-error: true - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - runs-on: ${{ github.event.inputs.runner_name }} - steps: - - uses: actions-rust-lang/setup-rust-toolchain@11df97af8e8102fd60b60a77dfbf58d40cd843b8 # v1.10.1 - - name: Setup Python - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 - with: - python-version: ${{ matrix.python-version }} - - name: Download wheels - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} - path: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} - - name: Checkout the repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - path: repo - - name: Test wheel - run: | - WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-linux-x86', matrix.python-version) }} - CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python - - # Initialize an empty test environment - cd $(mktemp -d) - python -m venv .testenv && source .testenv/bin/activate - - # Install the concrete-python wheel - pip install $WHEEL_DIR/*.whl - - # Install extra requirements for tests - sudo apt update -y - sudo apt install -y graphviz libgraphviz-dev - pip install -r $CONCRETE_PYTHON/requirements.extra-full.txt - pip install -r $CONCRETE_PYTHON/requirements.dev.txt - - # TODO - check for version - - # Copy test files - cp -R $CONCRETE_PYTHON/tests . - cp -R $CONCRETE_PYTHON/pytest.ini . - cp $CONCRETE_PYTHON/Makefile . - - # Running tests - make tfhers-utils - pytest tests -svv -n auto - - test-macos: - needs: [build-macos] - continue-on-error: true - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - runs-on: ["aws-mac1-metal", "aws-mac2-metal"] - runs-on: ${{ matrix.runs-on }} - steps: - - name: Download wheels - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} - path: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} - - name: Checkout the repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - path: repo - - name: Test wheel - run: | - WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} - CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python - PYTHON=${{ format('python{0}', matrix.python-version) }} - - # Initialize an empty test environment - export TEST_TMP_DIR=$(mktemp -d) - echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}" - cd $TEST_TMP_DIR - - # Activate virtual environment - $PYTHON -m venv .testenv && source .testenv/bin/activate - - # Install extra requirements for tests - pip install $WHEEL_DIR/*macos*.whl - pip install -r $CONCRETE_PYTHON/requirements.dev.txt - - # MacOS x86 have conflict between our OpenMP library, and one from torch - # we fix it by using a single one (from torch) - # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8 - export SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON_VERSION/site-packages/ - find $SITE_PACKAGES -not \( -path $SITE_PACKAGES/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1 ln -f -s $SITE_PACKAGES/concrete/.dylibs/libomp.dylib - - # Copy test files - cp -R $CONCRETE_PYTHON/tests . - cp -R $CONCRETE_PYTHON/pytest.ini . - cp $CONCRETE_PYTHON/Makefile . - - # Running tests - make tfhers-utils - mkdir ./KeySetCache - pytest tests -svv -n auto --key-cache "./KeySetCache" -m "not dataflow and not graphviz" - - - name: Cleanup host - if: success() || failure() - run: | - rm -rf $TEST_TMP_DIR diff --git a/.github/workflows/concrete_python_release_gpu.yml b/.github/workflows/concrete_python_release_gpu.yml index 26411b5706..e42754be90 100644 --- a/.github/workflows/concrete_python_release_gpu.yml +++ b/.github/workflows/concrete_python_release_gpu.yml @@ -1,61 +1,58 @@ -name: Concrete Python Release (GPU) - +name: concrete-python release-gpu on: workflow_dispatch: - inputs: - instance_id: - description: 'Instance ID' - type: string - instance_image_id: - description: 'Instance AMI ID' - type: string - instance_type: - description: 'Instance product type' - type: string - runner_name: - description: 'Action runner name' - type: string - request_id: - description: 'Slab request ID' - type: string - user_inputs: - description: 'either "nightly" or "public" or "private" to specify the release type' - required: true - default: 'nightly' - type: string + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+*' + schedule: + # Nightly Release @ 3AM after each work day + - cron: "0 3 * * 2-6" env: DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler CUDA_PATH: /usr/local/cuda-11.8 - GCC_VERSION: 11 - RELEASE_TYPE: ${{ inputs.user_inputs }} + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + RELEASE_TYPE: ${{ (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) && 'public' || 'nightly' }} + +concurrency: + group: concrete_python_release_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: + setup-instance: + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-release + build-linux-x86: + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - - runs-on: ${{ github.event.inputs.runner_name }} steps: - - name: Log instance configuration - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - echo "User Inputs: ${{ inputs.user_inputs }}" - - name: Set up GitHub environment run: | echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: recursive fetch-depth: 0 - - name: Set release version (nightly) if: ${{ env.RELEASE_TYPE == 'nightly' }} run: | @@ -65,14 +62,11 @@ jobs: echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt git tag nightly-$NIGHTLY_VERSION || true git push origin nightly-$NIGHTLY_VERSION || true - - name: Set release version (public) if: ${{ env.RELEASE_TYPE == 'public' }} run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt - - name: Expose release version from Python run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py - - name: Build wheel uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 id: build-compiler-bindings @@ -84,29 +78,23 @@ jobs: options: >- -v ${{ github.workspace }}:/concrete -v ${{ github.workspace }}/build:/build - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket shell: bash run: | set -e - rustup toolchain install nightly-2024-09-30 - pip install mypy rm -rf /build/* - + export PYTHON=${{ format('python{0}', matrix.python-version) }} echo "Using $PYTHON" - dnf -y install graphviz graphviz-devel - cd /concrete/frontends/concrete-python make PYTHON=$PYTHON venv source .venv/bin/activate - + cd /concrete/compilers/concrete-compiler/compiler make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=OFF Python3_EXECUTABLE=$(which python) \ CUDA_SUPPORT=ON TIMING_ENABLED=ON CUDA_PATH=${{ env.CUDA_PATH }} python-bindings - + echo "Debug: ccache statistics (after the build):" ccache -s @@ -114,37 +102,62 @@ jobs: export COMPILER_BUILD_DIRECTORY="/build" make whl - - deactivate + deactivate - name: Upload wheel uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} path: frontends/concrete-python/dist/*manylinux*.whl retention-days: 3 + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "build-linux-x86 (${{matrix.python-version}}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - push: + teardown-instance: + needs: [ setup-instance, build-linux-x86 ] + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + push-wheels: needs: [build-linux-x86] runs-on: ubuntu-latest outputs: wheel_version: ${{ steps.version.outputs.wheel_version }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: path: wheels merge-multiple: true - - - name: Install aws-cli if not present + - name: Install aws-cli run: | aws --version || (curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ unzip awscliv2.zip && \ sudo ./aws/install) - - name: Upload wheels to S3 - if: ${{ env.RELEASE_TYPE == 'public' || env.RELEASE_TYPE == 'nightly' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }} @@ -157,18 +170,92 @@ jobs: aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/gpu/concrete-python # update indexes and invalidate cloudfront cache python .github/workflows/scripts/s3_update_html_indexes.py - - name: Output Wheel Version id: version run: | export VERSION=`ls ./wheels/*manylinux* | head -n1 | cut -d "-" -f2` echo "VERSION=$VERSION" echo "wheel_version=$VERSION" >> "$GITHUB_OUTPUT" + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "push-wheels finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + setup-test-instance: + runs-on: ubuntu-latest + needs: [push-wheels] + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: gpu-test + + test-linux-x86: + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + fail-fast: false + needs: [setup-test-instance, push-wheels] + runs-on: ${{ needs.setup-test-instance.outputs.runner-name }} + steps: + - name: Setup Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: ${{ matrix.python-version }} - test-gpu-wheel: - needs: [push] - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-test-gpu-wheel - user_inputs: "${{ needs.push.outputs.wheel_version }}" + - name: Install CP + run: pip install --pre --extra-index-url https://pypi.zama.ai/gpu/ "concrete-python==${{ env.CP_VERSION }}" + + - name: Checkout the repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: repo + + - name: Test wheel + run: | + CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python + + # Install extra requirements for tests + sudo apt update -y + sudo apt install -y graphviz libgraphviz-dev + pip install -r $CONCRETE_PYTHON/requirements.extra-full.txt + pip install -r $CONCRETE_PYTHON/requirements.dev.txt + + # Running tests + cd $CONCRETE_PYTHON + make pytest-gpu + + + teardown-test-instance: + needs: [ setup-test-instance, test-linux-x86 ] + if: ${{ always() && needs.setup-test-instance.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 1a3471214f..0000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,303 +0,0 @@ -# This job is the main jobs will dispatch build and test for every modules of our mono repo. -name: Main - -on: -# pull_request: - push: - branches: - - 'main' -# - 'release/*' -# - 'force-docker-images' -# - 'private_release/*' -# tags: -# - 'v[0-9]+.[0-9]+.[0-9]+*' -# schedule: -# # Nightly Release @ 3AM after each work day -# - cron: "0 3 * * 2-6" - -jobs: - # This jobs outputs for each modules of our mono-repo if it changed, - # in order to launch jobs only for the changed modules - file-change: - if: ${{ github.event_name != 'schedule' }} - runs-on: ubuntu-latest - outputs: - compiler: ${{ steps.compiler.outputs.any_changed }} - optimizer: ${{ steps.optimizer.outputs.any_changed }} - concrete-cpu: ${{ steps.concrete-cpu.outputs.any_changed }} - concrete-cpu-api: ${{ steps.concrete-cpu-api.outputs.any_changed }} - concrete-cuda-api: ${{ steps.concrete-cuda-api.outputs.any_changed }} - concrete-python: ${{ steps.concrete-python.outputs.any_changed }} - concrete-compiler-cpu-workflow: ${{ steps.concrete-compiler-cpu-workflow.outputs.any_changed }} - concrete-compiler-gpu-workflow: ${{ steps.concrete-compiler-gpu-workflow.outputs.any_changed }} - concrete-compiler-format-and-linting-workflow: ${{ steps.concrete-compiler-format-and-linting-workflow.outputs.any_changed }} - concrete-compiler-macos-workflow: ${{ steps.concrete-compiler-macos-workflow.outputs.any_changed }} - concrete-compiler-docker-images-workflow: ${{ steps.concrete-compiler-docker-images-workflow.outputs.any_changed }} - concrete-cpu-workflow: ${{ steps.concrete-cpu-workflow.outputs.any_changed }} - concrete-python-workflow: ${{ steps.concrete-python-workflow.outputs.any_changed }} - concrete-optimizer-workflow: ${{ steps.concrete-optimizer-workflow.outputs.any_changed }} - push-main: ${{ steps.github.outputs.push-main }} - steps: - - name: Checkout the repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Get changed files in the concrete-compiler directory - id: compiler - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./compilers/concrete-compiler/** - - - name: Get changed files for concrete-optimizer - id: optimizer - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: | - ./tools/parameter-curves/concrete-security-curves-rust/** - ./compilers/concrete-optimizer/** - ./.github/workflows/optimizer.yml - - - name: Get changed files in the concrete-cpu directory - id: concrete-cpu - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./backends/concrete-cpu/implementation/** - - - name: Get changed files in the concrete-python directory - id: concrete-python - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./frontends/concrete-python/** - - - name: Check if compiler_build_and_test_cpu workflow has changed - id: concrete-compiler-cpu-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/compiler_build_and_test_cpu.yml - - - name: Check if compiler_build_and_test_gpu workflow has changed - id: concrete-compiler-gpu-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/compiler_build_and_test_gpu.yml - - - name: Check if compiler_format_and_linting.yml workflow has changed - id: concrete-compiler-format-and-linting-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/compiler_format_and_linting.yml - - - name: Check if compiler_macos_build_and_test workflow has changed - id: concrete-compiler-macos-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/compiler_macos_build_and_test.yml - - - name: Check if compiler_publish_docker_images workflow has changed - id: concrete-compiler-docker-images-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: | - ./.github/workflows/compiler_publish_docker_images.yml - ./docker/** - - - name: Check if concrete_cpu_test workflow has changed - id: concrete-cpu-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/concrete_cpu_test.yml - - - name: Check if concrete_python_checks workflow has changed - id: concrete-python-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/concrete_python_checks.yml - - - name: Check if optimizer workflow has changed - id: concrete-optimizer-workflow - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./.github/workflows/optimizer.yml - - - name: Get changed files in the concrete-cpu directory - id: concrete-cpu-api - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./backends/concrete-cpu/implementation/include/** - - - name: Get changed files in the concrete-cuda directory - id: concrete-cuda-api - uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 - with: - files: ./backends/concrete-cuda/implementation/include/** - - - name: Set some github event outputs - id: github - if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) - run: echo "push-main=true" >> "$GITHUB_OUTPUT" - -################################################# -# Compiler jobs ################################# - compiler-compliance: - needs: file-change - if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-format-and-linting-workflow == 'true' || needs.file-change.outputs.push-main == 'true' - uses: ./.github/workflows/compiler_format_and_linting.yml - - compiler-cpu-build: - needs: file-change - if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cpu-api == 'true'|| needs.file-change.outputs.concrete-compiler-cpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true' - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: compiler-cpu-build - - compiler-cpu-build-distributed: - needs: file-change - if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cpu-api == 'true'|| needs.file-change.outputs.concrete-compiler-cpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true' - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: compiler-cpu-build-distributed - - compiler-gpu-build: - needs: file-change - if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cuda-api == 'true' || needs.file-change.outputs.concrete-compiler-gpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true' - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: compiler-gpu-build - - compiler-macos-tests: - needs: file-change - if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-macos-workflow == 'true' || needs.file-change.outputs.push-main == 'true' - uses: ./.github/workflows/compiler_macos_build_and_test.yml - secrets: inherit - - compiler-publish-docker-images: - needs: file-change - if: (needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-docker-images-workflow == 'true') && (needs.file-change.outputs.push-main == 'true' || contains(github.ref, 'refs/heads/force-docker-images')) - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: compiler-publish-docker-images - - compiler-cpu-benchmark: - needs: file-change - if: needs.file-change.outputs.push-main == 'true' - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: compiler-cpu-benchmark - - # compiler-gpu-benchmark: - # needs: file-change - # if: needs.file-change.outputs.push-main == 'true' - # uses: ./.github/workflows/start_slab.yml - # secrets: inherit - # with: - # command: compiler-gpu-benchmark - -################################################# -# Optimizer jobs ################################ - optimizer: - needs: file-change - if: | - needs.file-change.outputs.parameters-curves == 'true' || - needs.file-change.outputs.concrete-cpu == 'true' || - needs.file-change.outputs.optimizer == 'true'|| - needs.file-change.outputs.push-main - uses: ./.github/workflows/optimizer.yml - secrets: inherit - -################################################# -# ConcreteCPU jobs ############################## - concrete-cpu: - needs: file-change - if: needs.file-change.outputs.concrete-cpu == 'true' || needs.file-change.outputs.concrete-cpu-workflow == 'true' || needs.file-change.outputs.push-main - uses: ./.github/workflows/concrete_cpu_test.yml - secrets: inherit - -################################################# -# Concrete Python jobs ########################## - concrete-python: - needs: file-change - if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.concrete-python-workflow == 'true' || needs.file-change.outputs.push-main - uses: ./.github/workflows/concrete_python_checks.yml - secrets: inherit - - concrete-python-tests-linux: - needs: file-change - if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.push-main - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-tests-linux - - concrete-python-tests-linux-gpu: - needs: file-change - if: needs.file-change.outputs.concrete-python == 'true' && needs.file-change.outputs.push-main - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-tests-linux-gpu - - concrete-python-tests-macos: - needs: file-change - if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.push-main - uses: ./.github/workflows/concrete_python_test_macos.yml - secrets: inherit - -################################################# -# Concrete-ML tests ############################# - concrete-ml-tests-linux: - needs: file-change - if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.push-main - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: ml-test - -################################################# -# Release jobs ################################# - concrete-python-nightly-release: - if: ${{ github.event_name == 'schedule' }} - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-release - user_inputs: 'nightly' - - concrete-python-nightly-release-gpu: - if: ${{ github.event_name == 'schedule' }} - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-release-gpu - user_inputs: 'nightly' - - concrete-python-public-release: -# needs: [compiler-cpu-build, compiler-macos-tests, compiler-publish-docker-images, concrete-python-tests-linux, concrete-python-tests-macos] - if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-release - user_inputs: 'public' - - concrete-python-public-release-gpu: - if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-release-gpu - user_inputs: 'public' - - concrete-python-private-release: - if: github.event_name == 'push' && contains(github.ref, 'refs/heads/private_release/') - uses: ./.github/workflows/start_slab.yml - secrets: inherit - with: - command: concrete-python-release - user_inputs: 'private' diff --git a/.github/workflows/markdown_link_check.yml b/.github/workflows/markdown_link_check.yml deleted file mode 100644 index b2c6ffb849..0000000000 --- a/.github/workflows/markdown_link_check.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Check Markdown links - -on: - pull_request: - paths: - - '**.md' - - .github/workflows/markdown_link_check.yml - push: - branches: - - main - -jobs: - markdown-link-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1 - with: - use-quiet-mode: 'yes' - use-verbose-mode: 'yes' diff --git a/.github/workflows/push_wheels_to_public_pypi.yml b/.github/workflows/push_wheels_to_public_pypi.yml deleted file mode 100644 index 3f1ff7d80d..0000000000 --- a/.github/workflows/push_wheels_to_public_pypi.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Push Wheels to Public PyPI - -on: - workflow_dispatch: - inputs: - version: - description: 'version of concrete-python to pull from Zama PyPI and push to public PyPI. Use the version as it appears in the wheel file (e.g. 2.7.0rc1)' - required: true - type: string - - -jobs: - pull_and_push: - runs-on: ubuntu-latest - steps: - - name: Pull wheels from S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} - S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }} - run: | - mkdir wheels - aws s3 cp s3://${S3_BUCKET_NAME}/cpu/concrete-python/ ./wheels/ --recursive --exclude "*" --include "concrete_python-${{ inputs.version }}-*" - echo "============== Downloaded wheels ===============" - ls -la ./wheels/ - - - name: Push wheels to public PyPI (public) - run: | - pip install twine==4.0.2 - twine upload wheels/concrete_python-${{ inputs.version }}*.whl \ - -u "${{ secrets.PUBLIC_PYPI_USER }}" \ - -p "${{ secrets.PUBLIC_PYPI_PASSWORD }}" \ - -r pypi - diff --git a/.github/workflows/scripts/teardown-check.sh b/.github/workflows/scripts/teardown-check.sh new file mode 100755 index 0000000000..9c690cb7ae --- /dev/null +++ b/.github/workflows/scripts/teardown-check.sh @@ -0,0 +1,10 @@ +#!/bin/bash -e + +grep setup-instance -Rl .github/workflows/ | xargs grep -L teardown-instance &> missing-teardown.txt + +if [ -s missing-teardown.txt ]; then + echo "There are missing teardown-instance jobs in following jobs:" + echo + cat missing-teardown.txt + exit 1 +fi \ No newline at end of file diff --git a/.github/workflows/start_slab.yml b/.github/workflows/start_slab.yml deleted file mode 100644 index 0bdd7961c5..0000000000 --- a/.github/workflows/start_slab.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Start job on Slab CI bot given by input command. -name: Start AWS job - -on: - workflow_call: - inputs: - command: - required: true - type: string - user_inputs: - required: false - type: string - workflow_dispatch: - inputs: - command: - required: true - type: string - user_inputs: - description: 'user inputs to be forwarded to the called workflow' - required: false - type: string - -env: - GIT_REF: ${{ github.head_ref }} - -jobs: - sl: - runs-on: ubuntu-latest - steps: - - name: Checkout concrete - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - name: Get git ref - # github.head_ref is only available from a Pull Request - if: env.GIT_REF == '' - run: | - echo "GIT_REF=${{ github.ref_name }}" >> $GITHUB_ENV - - - name: Checkout Slab repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} - - - name: Start AWS job in Slab - shell: bash - run: | - GIT_SHA="$(git --no-pager show -s --format="%H" origin/${{ env.GIT_REF }})" || GIT_SHA=${{ github.sha }} - echo -n '{"command": "${{ inputs.command }}", "git_ref": "${{ env.GIT_REF }}", "sha":"'${GIT_SHA}'", "user_inputs": "${{ inputs.user_inputs }}"}' > command.json - cat command.json - SIGNATURE="$(slab/scripts/hmac_calculator.sh command.json '${{ secrets.JOB_SECRET }}')" - curl -v -k \ - --fail-with-body \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: start_aws" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @command.json \ - ${{ secrets.SLAB_URL }} diff --git a/ci/slab.toml b/ci/slab.toml index c69399c4a1..bccab4e1a4 100644 --- a/ci/slab.toml +++ b/ci/slab.toml @@ -23,139 +23,7 @@ instance_id = "i-0e5ae2a14134d6275" instance_type = "m6i.8xlarge" security_group= ["sg-02dd8470fa845f31b", ] -# This is the new version of Slab that handles multi backend providers. -[backend.aws.m7i-cpu-bench] -region = "eu-west-1" -image_id = "ami-002bdcd64b8472cf9" # Based on Ubuntu 22.4 -instance_type = "m7i.48xlarge" -security_group = ["sg-0e55cc31dfda0d8a7", ] - -[profile.m7i-cpu-bench] -region = "eu-west-1" -image_id = "ami-002bdcd64b8472cf9" # Based on Ubuntu 22.4 -instance_type = "m7i.48xlarge" -security_group= ["sg-0e55cc31dfda0d8a7", ] - -[profile.m7i-cpu-test] +[backend.aws.release] region = "eu-west-1" image_id = "ami-002bdcd64b8472cf9" -instance_type = "m7i.16xlarge" -security_group= ["sg-0e55cc31dfda0d8a7", ] - -[profile.m7i-metal] -region = "eu-west-1" -image_id = "ami-002bdcd64b8472cf9" -instance_type = "m7i.metal-24xl" -security_group= ["sg-0e55cc31dfda0d8a7", ] - -[profile.gpu-bench] -region = "us-east-1" -image_id = "ami-08e27480d79e82238" -instance_type = "p3.2xlarge" -subnet_id = "subnet-8123c9e7" -security_group= ["sg-017afab1f328af917", ] - -# Docker is well configured for test inside docker in this AMI -[profile.gpu-test] -region = "us-east-1" -image_id = "ami-0257c6ad39f902b5e" -instance_type = "p3.2xlarge" -subnet_id = "subnet-8123c9e7" -security_group= ["sg-017afab1f328af917", ] - -# It has CUDA Driver (<=12.5) and Docker installed -[profile.gpu-test-ubuntu22] -region = "us-east-1" -image_id = "ami-05385e0c3c574621f" -instance_type = "p3.2xlarge" -subnet_id = "subnet-8123c9e7" -security_group= ["sg-017afab1f328af917", ] - -[profile.slurm-cluster] -region = "eu-west-3" -image_id = "ami-0bb5bb9cb747b5ddd" -instance_id = "i-0e5ae2a14134d6275" -instance_type = "m6i.8xlarge" -security_group= ["sg-02dd8470fa845f31b", ] - -################################################# -# Compiler commands -################################################# - -[command.compiler-cpu-build] -workflow = "compiler_build_and_test_cpu.yml" -profile = "m7i-cpu-test" -check_run_name = "Compiler Build and Test (CPU)" - -[command.compiler-cpu-build-distributed] -workflow = "compiler_build_and_test_cpu_distributed.yml" -profile = "slurm-cluster" -check_run_name = "Compiler Distributed Build and Test (CPU)" -runner_name = "distributed-ci" - -[command.compiler-gpu-build] -workflow = "compiler_build_and_test_gpu.yml" -profile = "gpu-test" -check_run_name = "Compiler Build and Test (GPU)" - -[command.compiler-cpu-benchmark] -workflow = "compiler_benchmark.yml" -profile = "m7i-cpu-bench" -check_run_name = "Compiler Performances Benchmarks (CPU)" - -[command.compiler-gpu-benchmark] -workflow = "compiler_benchmark.yml" -profile = "gpu-bench" -check_run_name = "Compiler Performances Benchmarks (GPU)" - -# Trigger Docker images build -[command.compiler-publish-docker-images] -workflow = "compiler_publish_docker_images.yml" -profile = "m7i-cpu-test" -check_run_name = "Compiler - Docker images build & publish" - -# Trigger ML benchmarks by running each use cases subset in parallel. -[command.ml-bench] -workflow = "ml_benchmark_subset.yml" -profile = "m7i-cpu-bench" -matrix = [0,1,2,3,4,5,6,7,8,9,10] -max_parallel_jobs = 2 - -# Trigger ML tests with latest CP -[command.ml-test] -workflow = "concrete_ml_tests.yml" -profile = "m7i-cpu-test" -check_run_name = "Concrete ML Tests" - -################################################# -# Concrete Python Commands -################################################# - -[command.concrete-python-tests-linux] -workflow = "concrete_python_tests_linux.yml" -profile = "m7i-cpu-test" -check_run_name = "Concrete Python Tests (Linux)" - -[command.concrete-python-tests-linux-gpu] -workflow = "concrete_python_tests_linux.yml" -profile = "gpu-test" -check_run_name = "Concrete Python Tests (Linux Gpu)" - -################################################# -# Release Commands -################################################# - -[command.concrete-python-release] -workflow = "concrete_python_release.yml" -profile = "m7i-cpu-test" -check_run_name = "Concrete Python Release" - -[command.concrete-python-release-gpu] -workflow = "concrete_python_release_gpu.yml" -profile = "m7i-cpu-test" -check_run_name = "Concrete Python Release (GPU)" - -[command.concrete-python-test-gpu-wheel] -workflow = "concrete_python_test_gpu_wheel.yml" -profile = "gpu-test" -check_run_name = "Concrete Python Test GPU Wheel" +instance_type = "hpc7a.96xlarge" \ No newline at end of file