Skip to content

Commit

Permalink
Merge branch 'dev' into yash/remote-debugging
Browse files Browse the repository at this point in the history
  • Loading branch information
tcp authored Feb 19, 2024
2 parents f72aa7b + 8fffebe commit 5b76183
Show file tree
Hide file tree
Showing 77 changed files with 1,194 additions and 450 deletions.
24 changes: 18 additions & 6 deletions .bumpversion.cfg
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[bumpversion]
current_version = 0.8.4-beta.22
current_version = 0.8.4-beta.30
tag = False
tag_name = {new_version}
commit = True
Expand All @@ -26,16 +26,22 @@ values =
[bumpversion:part:prenum]
first_value = 1

# Root

[bumpversion:file:VERSION]

[bumpversion:file:packages/grid/devspace.yaml]
# Syft

[bumpversion:file:packages/syft/setup.cfg]

[bumpversion:file:packages/syft/src/syft/__init__.py]

[bumpversion:file:packages/syft/src/syft/VERSION]

# Grid

[bumpversion:file:packages/grid/devspace.yaml]

[bumpversion:file:packages/grid/VERSION]

[bumpversion:file:packages/grid/backend/worker_cpu.dockerfile]
Expand All @@ -44,14 +50,20 @@ first_value = 1

[bumpversion:file:packages/grid/helm/syft/Chart.yaml]

[bumpversion:file:packages/grid/podman/podman-kube/podman-syft-kube.yaml]

[bumpversion:file:packages/grid/podman/podman-kube/podman-syft-kube-config.yaml]

[bumpversion:file:packages/grid/helm/syft/values.yaml]

# Hagrid

[bumpversion:file:packages/hagrid/hagrid/manifest_template.yml]

[bumpversion:file:packages/hagrid/hagrid/deps.py]

[bumpversion:file:packages/grid/podman/podman-kube/podman-syft-kube.yaml]

[bumpversion:file:packages/grid/podman/podman-kube/podman-syft-kube-config.yaml]
# Syft CLI

[bumpversion:file:packages/syftcli/manifest.yml]

[bumpversion:file:packages/grid/helm/syft/values.yaml]

25 changes: 25 additions & 0 deletions .bumpversion_stable.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
[bumpversion]
current_version = 0.8.3
tag = False
tag_name = {new_version}
commit = True
parse =
(?P<major>\d+)
\.
(?P<minor>\d+)
\.
(?P<patch>\d+)
serialize =
{major}.{minor}.{patch}

# Syft

[bumpversion:file:packages/syft/src/syft/stable_version.py]

# Hagrid

[bumpversion:file:packages/hagrid/hagrid/stable_version.py]

[bumpversion:file:packages/hagrid/hagrid/cache.py]


6 changes: 6 additions & 0 deletions .github/workflows/cd-hagrid.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,12 @@ on:
required: false
default: "false"

# Prevents concurrent runs of the same workflow
# while the previous run is still in progress
concurrency:
group: "CD - Hagrid"
cancel-in-progress: false

jobs:
call-pr-tests-linting:
if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
Expand Down
189 changes: 189 additions & 0 deletions .github/workflows/cd-post-release-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,189 @@
name: CD - Post Release Tests

on:
workflow_dispatch:
inputs:
syft_version:
description: "Syft version to test"
required: true
type: string

workflow_call:
inputs:
syft_version:
description: "Syft version to test"
required: true
type: string

jobs:
notebook-test-hagrid:
strategy:
max-parallel: 99
matrix:
os: [ubuntu-latest]
python-version: ["3.11"]

runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4

# free 10GB of space
- name: Remove unnecessary files
if: matrix.os == 'ubuntu-latest'
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
docker image prune --all --force
docker builder prune --all --force
docker system prune --all --force
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5

with:
python-version: ${{ matrix.python-version }}

- name: Upgrade pip
run: |
python -m pip install --upgrade --user pip
- name: Get pip cache dir
id: pip-cache
shell: bash
run: |
echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
- name: pip cache
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
restore-keys: |
${{ runner.os }}-pip-py${{ matrix.python-version }}-
- name: Install Hagrid and tox
run: |
pip install -U hagrid
pip install tox
- name: Hagrid Version
run: |
hagrid version
- name: Launch Domain
run: |
hagrid launch test-domain-1 to docker:8081 --tag=${{ inputs.syft_version }} --low-side
- name: Run tests
env:
NODE_PORT: "8081"
SYFT_VERSION: ${{ inputs.syft_version }}
EXCLUDE_NOTEBOOKS: "not 11-container-images-k8s.ipynb"
run: |
tox -e e2e.test.notebook
syft-install-check:
strategy:
max-parallel: 99
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
# Re-enable macos-14 when it's available when we remove python 3.9 from syft
# os: [ubuntu-latest, macos-latest, windows-latest, macos-14]
python-version: ["3.11", "3.10", "3.9"]

runs-on: ${{ matrix.os }}
steps:
- name: System Architecture
run: |
echo "System Architecture: $(uname -m)"
echo "System Version: $(uname -a)"
- uses: actions/checkout@v4

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5

with:
python-version: ${{ matrix.python-version }}

- name: Upgrade pip
run: |
python -m pip install --upgrade --user pip
- name: Get pip cache dir
id: pip-cache
shell: bash
run: |
echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
- name: pip cache
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
restore-keys: |
${{ runner.os }}-pip-py${{ matrix.python-version }}-
- name: Install Syft
run: |
pip install syft==${{ inputs.syft_version }}
- name: Check Syft version
run: |
python -c "import syft; print(syft.__version__)"
notebook-test-k8s-k3d:
strategy:
max-parallel: 99
matrix:
os: [ubuntu-latest]
python-version: ["3.11"]

runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4

# free 10GB of space
- name: Remove unnecessary files
if: matrix.os == 'ubuntu-latest'
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
docker image prune --all --force
docker builder prune --all --force
docker system prune --all --force
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5

with:
python-version: ${{ matrix.python-version }}

- name: Install K3d
run: |
K3D_VERSION=v5.6.0
wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=$K3D_VERSION bash
- name: Get pip cache dir
id: pip-cache
shell: bash
run: |
echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
- name: pip cache
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
restore-keys: |
${{ runner.os }}-pip-py${{ matrix.python-version }}-
- name: Install tox
run: |
pip install tox
- name: Run K8s tests
env:
SYFT_VERSION: ${{ inputs.syft_version }}
run: |
tox -e syft.test.helm
40 changes: 2 additions & 38 deletions .github/workflows/cd-syft-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,20 +70,10 @@ jobs:
id: buildx
uses: docker/setup-buildx-action@v3

- name: Install Azure CLI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
az version
- name: Login to Azure CLI
uses: azure/login@v1
with:
creds: ${{ secrets.AZURE_CREDENTIALS_GITHUB_CI }}

- name: Login to Azure Container Registry
uses: azure/docker-login@v1
uses: docker/login-action@v3
with:
login-server: ${{ secrets.ACR_SERVER }}
registry: ${{ secrets.ACR_SERVER }}
username: ${{ secrets.ACR_USERNAME }}
password: ${{ secrets.ACR_PASSWORD }}

Expand Down Expand Up @@ -178,29 +168,3 @@ jobs:
add: "."
push: "origin main"
cwd: "./infrastructure/"

- name: Cleanup Azure Container Registry
run: |
ACR_REGISTRY_NAME=${{ secrets.ACR_REGISTRY_NAME }}
echo ">> Fetching repo list.."
REPO_LIST=$(az acr repository list -n $ACR_REGISTRY_NAME -o tsv)
KEEP_PREV_VERSIONS=5
TAIL_FROM_LINE=$(($KEEP_PREV_VERSIONS + 1))
for repo in $REPO_LIST
do
echo "Cleaning up '$repo'"
az acr repository show-tags --name $ACR_REGISTRY_NAME --repository $repo --orderby time_desc --output tsv \
| grep dev- \
| tail -n +$TAIL_FROM_LINE \
| xargs -r -I% az acr repository untag --name $ACR_REGISTRY_NAME --image $repo:%
done
- name: Logout and cleanup Azure account
if: always()
run: |
az logout
az cache purge
az account clear
19 changes: 19 additions & 0 deletions .github/workflows/cd-syft.yml
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,10 @@ jobs:
runs-on: ubuntu-latest
permissions:
contents: write # For tag and release notes.

outputs:
syft_version: ${{ steps.release_checks.outputs.syft_version }}

steps:
- name: Permission to home directory
run: |
Expand Down Expand Up @@ -354,6 +358,7 @@ jobs:
run: |
echo "github_release_version=$(python packages/grid/VERSION | sed 's/-beta./b/')" >> $GITHUB_OUTPUT
echo "future_stable_version=$(python packages/grid/VERSION | sed 's/-beta.*//')" >> $GITHUB_OUTPUT
echo "syft_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
if [[ ${{ needs.merge-docker-images.outputs.release_tag}} == "beta" ]]; then
echo "is_beta_release=true" >> $GITHUB_OUTPUT
else
Expand Down Expand Up @@ -514,3 +519,17 @@ jobs:
add: "helm/"
push: "origin gh-pages"
cwd: "./ghpages/"

# Since the post-release tests are dependent on the release being available on PyPI
# we need to wait for PyPI to update before running the post-release tests
- name: Wait for PyPI to update
run: |
sleep 60
# Can we remove the always flag from the below job?
call-cd-post-release-tests:
needs: [deploy-syft]
if: always() && github.repository == 'OpenMined/PySyft' && needs.deploy-syft.result == 'success' # don't run on forks
uses: OpenMined/PySyft/.github/workflows/cd-post-release-tests.yml@dev
with:
syft_version: ${{ needs.deploy-syft.outputs.syft_version }}
Loading

0 comments on commit 5b76183

Please sign in to comment.